1f5166768STheodore Ts'o // SPDX-License-Identifier: GPL-2.0 2c9de560dSAlex Tomas /* 3c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 4c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 5c9de560dSAlex Tomas */ 6c9de560dSAlex Tomas 7c9de560dSAlex Tomas 8c9de560dSAlex Tomas /* 9c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 10c9de560dSAlex Tomas */ 11c9de560dSAlex Tomas 1218aadd47SBobi Jam #include "ext4_jbd2.h" 138f6e39a7SMingming Cao #include "mballoc.h" 1428623c2fSTheodore Ts'o #include <linux/log2.h> 15a0b30c12STheodore Ts'o #include <linux/module.h> 165a0e3ad6STejun Heo #include <linux/slab.h> 171a5d5e5dSJeremy Cline #include <linux/nospec.h> 1866114cadSTejun Heo #include <linux/backing-dev.h> 199bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 209bffad1eSTheodore Ts'o 21c9de560dSAlex Tomas /* 22c9de560dSAlex Tomas * MUSTDO: 23c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 24c9de560dSAlex Tomas * - search for metadata in few groups 25c9de560dSAlex Tomas * 26c9de560dSAlex Tomas * TODO v4: 27c9de560dSAlex Tomas * - normalization should take into account whether file is still open 28c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 29c9de560dSAlex Tomas * - don't normalize tails 30c9de560dSAlex Tomas * - quota 31c9de560dSAlex Tomas * - reservation for superuser 32c9de560dSAlex Tomas * 33c9de560dSAlex Tomas * TODO v3: 34c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 35c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 36c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 37c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 38c9de560dSAlex Tomas * - error handling 39c9de560dSAlex Tomas */ 40c9de560dSAlex Tomas 41c9de560dSAlex Tomas /* 42c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 43c9de560dSAlex Tomas * near to the goal(block) value specified. 44c9de560dSAlex Tomas * 45b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 46b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 47b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 48b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 49b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 50b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 51b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 52b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 53b713a5ecSTheodore Ts'o * terms of number of blocks. 54c9de560dSAlex Tomas * 55c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 56b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 57c9de560dSAlex Tomas * 58b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 59b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 60b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 61b713a5ecSTheodore Ts'o * represented as: 62c9de560dSAlex Tomas * 63c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 64c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 6553accfa9STheodore Ts'o * pa_len -> length for this prealloc space (in clusters) 6653accfa9STheodore Ts'o * pa_free -> free space available in this prealloc space (in clusters) 67c9de560dSAlex Tomas * 68c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 69c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 70caaf7a29STao Ma * space we will consume the particular prealloc space. This makes sure that 71caaf7a29STao Ma * we have contiguous physical blocks representing the file blocks 72c9de560dSAlex Tomas * 73c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 74c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 75c9de560dSAlex Tomas * pa_free. 76c9de560dSAlex Tomas * 77c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 78c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 79caaf7a29STao Ma * prealloc space. These are per CPU prealloc list represented as 80c9de560dSAlex Tomas * 81c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 82c9de560dSAlex Tomas * 83c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 84c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 85c9de560dSAlex Tomas * 86c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 8725985edcSLucas De Marchi * enough free space (pa_free) within the prealloc space. 88c9de560dSAlex Tomas * 89c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 90c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 91c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 92c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 93c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 94c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 95c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 96c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 97c9de560dSAlex Tomas * inode as: 98c9de560dSAlex Tomas * 99c9de560dSAlex Tomas * { page } 100c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 101c9de560dSAlex Tomas * 102c9de560dSAlex Tomas * 103c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 104ea1754a0SKirill A. Shutemov * take up 2 blocks. A page can contain blocks_per_page (PAGE_SIZE / 105c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 106c9de560dSAlex Tomas * which is blocks_per_page/2 107c9de560dSAlex Tomas * 108c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 109c9de560dSAlex Tomas * away when the filesystem is unmounted. 110c9de560dSAlex Tomas * 111c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 112c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 113c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 114c9de560dSAlex Tomas * 115c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 116c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 117c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 118c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 119c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 120c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 12127baebb8STheodore Ts'o * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is 12227baebb8STheodore Ts'o * dependent on the cluster size; for non-bigalloc file systems, it is 123c9de560dSAlex Tomas * 512 blocks. This can be tuned via 124d7a1fee1SDan Ehrenberg * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in 125c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 126c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 127b483bb77SRandy Dunlap * smallest multiple of the stripe value (sbi->s_stripe) which is 128d7a1fee1SDan Ehrenberg * greater than the default mb_group_prealloc. 129c9de560dSAlex Tomas * 130196e402aSHarshad Shirwadkar * If "mb_optimize_scan" mount option is set, we maintain in memory group info 131196e402aSHarshad Shirwadkar * structures in two data structures: 132196e402aSHarshad Shirwadkar * 133196e402aSHarshad Shirwadkar * 1) Array of largest free order lists (sbi->s_mb_largest_free_orders) 134196e402aSHarshad Shirwadkar * 135196e402aSHarshad Shirwadkar * Locking: sbi->s_mb_largest_free_orders_locks(array of rw locks) 136196e402aSHarshad Shirwadkar * 137196e402aSHarshad Shirwadkar * This is an array of lists where the index in the array represents the 138196e402aSHarshad Shirwadkar * largest free order in the buddy bitmap of the participating group infos of 139196e402aSHarshad Shirwadkar * that list. So, there are exactly MB_NUM_ORDERS(sb) (which means total 140196e402aSHarshad Shirwadkar * number of buddy bitmap orders possible) number of lists. Group-infos are 141196e402aSHarshad Shirwadkar * placed in appropriate lists. 142196e402aSHarshad Shirwadkar * 14383e80a6eSJan Kara * 2) Average fragment size lists (sbi->s_mb_avg_fragment_size) 144196e402aSHarshad Shirwadkar * 14583e80a6eSJan Kara * Locking: sbi->s_mb_avg_fragment_size_locks(array of rw locks) 146196e402aSHarshad Shirwadkar * 14783e80a6eSJan Kara * This is an array of lists where in the i-th list there are groups with 14883e80a6eSJan Kara * average fragment size >= 2^i and < 2^(i+1). The average fragment size 14983e80a6eSJan Kara * is computed as ext4_group_info->bb_free / ext4_group_info->bb_fragments. 15083e80a6eSJan Kara * Note that we don't bother with a special list for completely empty groups 15183e80a6eSJan Kara * so we only have MB_NUM_ORDERS(sb) lists. 152196e402aSHarshad Shirwadkar * 153196e402aSHarshad Shirwadkar * When "mb_optimize_scan" mount option is set, mballoc consults the above data 154196e402aSHarshad Shirwadkar * structures to decide the order in which groups are to be traversed for 155196e402aSHarshad Shirwadkar * fulfilling an allocation request. 156196e402aSHarshad Shirwadkar * 157f52f3d2bSOjaswin Mujoo * At CR_POWER2_ALIGNED , we look for groups which have the largest_free_order 158f52f3d2bSOjaswin Mujoo * >= the order of the request. We directly look at the largest free order list 159f52f3d2bSOjaswin Mujoo * in the data structure (1) above where largest_free_order = order of the 160f52f3d2bSOjaswin Mujoo * request. If that list is empty, we look at remaining list in the increasing 161f52f3d2bSOjaswin Mujoo * order of largest_free_order. This allows us to perform CR_POWER2_ALIGNED 162f52f3d2bSOjaswin Mujoo * lookup in O(1) time. 163196e402aSHarshad Shirwadkar * 164f52f3d2bSOjaswin Mujoo * At CR_GOAL_LEN_FAST, we only consider groups where 165f52f3d2bSOjaswin Mujoo * average fragment size > request size. So, we lookup a group which has average 166f52f3d2bSOjaswin Mujoo * fragment size just above or equal to request size using our average fragment 167f52f3d2bSOjaswin Mujoo * size group lists (data structure 2) in O(1) time. 168196e402aSHarshad Shirwadkar * 169f52f3d2bSOjaswin Mujoo * At CR_BEST_AVAIL_LEN, we aim to optimize allocations which can't be satisfied 170f52f3d2bSOjaswin Mujoo * in CR_GOAL_LEN_FAST. The fact that we couldn't find a group in 171f52f3d2bSOjaswin Mujoo * CR_GOAL_LEN_FAST suggests that there is no BG that has avg 172f52f3d2bSOjaswin Mujoo * fragment size > goal length. So before falling to the slower 173f52f3d2bSOjaswin Mujoo * CR_GOAL_LEN_SLOW, in CR_BEST_AVAIL_LEN we proactively trim goal length and 174f52f3d2bSOjaswin Mujoo * then use the same fragment lists as CR_GOAL_LEN_FAST to find a BG with a big 175f52f3d2bSOjaswin Mujoo * enough average fragment size. This increases the chances of finding a 176f52f3d2bSOjaswin Mujoo * suitable block group in O(1) time and results in faster allocation at the 177f52f3d2bSOjaswin Mujoo * cost of reduced size of allocation. 1787e170922SOjaswin Mujoo * 179196e402aSHarshad Shirwadkar * If "mb_optimize_scan" mount option is not set, mballoc traverses groups in 180f52f3d2bSOjaswin Mujoo * linear order which requires O(N) search time for each CR_POWER2_ALIGNED and 181f52f3d2bSOjaswin Mujoo * CR_GOAL_LEN_FAST phase. 182196e402aSHarshad Shirwadkar * 183d7a1fee1SDan Ehrenberg * The regular allocator (using the buddy cache) supports a few tunables. 184c9de560dSAlex Tomas * 185b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 186b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 187b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 188196e402aSHarshad Shirwadkar * /sys/fs/ext4/<partition>/mb_linear_limit 189c9de560dSAlex Tomas * 190b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 191c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 192c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 193b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 194af901ca1SAndré Goddard Rosa * stripe size (sbi->s_stripe), we try to search for contiguous block in 195b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 196b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 197b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 198c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 199b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 200c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 201c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 202c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 203caaf7a29STao Ma * can be used for allocation. ext4_mb_good_group explains how the groups are 204c9de560dSAlex Tomas * checked. 205c9de560dSAlex Tomas * 206196e402aSHarshad Shirwadkar * When "mb_optimize_scan" is turned on, as mentioned above, the groups may not 207196e402aSHarshad Shirwadkar * get traversed linearly. That may result in subsequent allocations being not 208196e402aSHarshad Shirwadkar * close to each other. And so, the underlying device may get filled up in a 209196e402aSHarshad Shirwadkar * non-linear fashion. While that may not matter on non-rotational devices, for 210196e402aSHarshad Shirwadkar * rotational devices that may result in higher seek times. "mb_linear_limit" 211196e402aSHarshad Shirwadkar * tells mballoc how many groups mballoc should search linearly before 212196e402aSHarshad Shirwadkar * performing consulting above data structures for more efficient lookups. For 213196e402aSHarshad Shirwadkar * non rotational devices, this value defaults to 0 and for rotational devices 214196e402aSHarshad Shirwadkar * this is set to MB_DEFAULT_LINEAR_LIMIT. 215196e402aSHarshad Shirwadkar * 216c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 217c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 218c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 219c9de560dSAlex Tomas * subsequent request. 220c9de560dSAlex Tomas */ 221c9de560dSAlex Tomas 222c9de560dSAlex Tomas /* 223c9de560dSAlex Tomas * mballoc operates on the following data: 224c9de560dSAlex Tomas * - on-disk bitmap 225c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 226c9de560dSAlex Tomas * - preallocation descriptors (PAs) 227c9de560dSAlex Tomas * 228c9de560dSAlex Tomas * there are two types of preallocations: 229c9de560dSAlex Tomas * - inode 230c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 231c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 232c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 233c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 234c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 235c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 236c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 237c9de560dSAlex Tomas * must discard all preallocated blocks. 238c9de560dSAlex Tomas * - locality group 239c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 240c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 241c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 242c9de560dSAlex Tomas * it's consumed from the beginning to the end. 243c9de560dSAlex Tomas * 244c9de560dSAlex Tomas * relation between them can be expressed as: 245c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 246c9de560dSAlex Tomas * 247c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 248c9de560dSAlex Tomas * - allocated blocks (persistent) 249c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 250c9de560dSAlex Tomas * 251c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 252c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 253c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 254c9de560dSAlex Tomas * 255c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 256c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 257c9de560dSAlex Tomas * 258c9de560dSAlex Tomas * all operations can be expressed as: 259c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 260c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 261c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 262c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 263c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 264c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 265c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 266c9de560dSAlex Tomas * is used in real operation because we can't know actual used 267c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 268c9de560dSAlex Tomas * 269c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 270c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 271c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 272c9de560dSAlex Tomas * the following knowledge: 273c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 274c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 275c9de560dSAlex Tomas * nobody can re-allocate that block 276c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 277c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 278c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 279c9de560dSAlex Tomas * block 280c9de560dSAlex Tomas * 281c9de560dSAlex Tomas * so, now we're building a concurrency table: 282c9de560dSAlex Tomas * - init buddy vs. 283c9de560dSAlex Tomas * - new PA 284c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 285c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 286c9de560dSAlex Tomas * - use inode PA 287c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 288c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 289c9de560dSAlex Tomas * - discard inode PA 290c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 291c9de560dSAlex Tomas * - use locality group PA 292c9de560dSAlex Tomas * again PA-=N must be serialized with init 293c9de560dSAlex Tomas * - discard locality group PA 294c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 295c9de560dSAlex Tomas * - new PA vs. 296c9de560dSAlex Tomas * - use inode PA 297c9de560dSAlex Tomas * i_data_sem serializes them 298c9de560dSAlex Tomas * - discard inode PA 299c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 300c9de560dSAlex Tomas * - use locality group PA 301c9de560dSAlex Tomas * some mutex should serialize them 302c9de560dSAlex Tomas * - discard locality group PA 303c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 304c9de560dSAlex Tomas * - use inode PA 305c9de560dSAlex Tomas * - use inode PA 306c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 307c9de560dSAlex Tomas * - discard inode PA 308c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 309c9de560dSAlex Tomas * - use locality group PA 310c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 311c9de560dSAlex Tomas * - discard locality group PA 312c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 313c9de560dSAlex Tomas * 314c9de560dSAlex Tomas * now we're ready to make few consequences: 315c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 316c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 317c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 318c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 319c9de560dSAlex Tomas * any discard or they're serialized somehow 320c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 321c9de560dSAlex Tomas * 322c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 323c9de560dSAlex Tomas * in this case, but we should care about concurrent init 324c9de560dSAlex Tomas * 325c9de560dSAlex Tomas */ 326c9de560dSAlex Tomas 327c9de560dSAlex Tomas /* 328c9de560dSAlex Tomas * Logic in few words: 329c9de560dSAlex Tomas * 330c9de560dSAlex Tomas * - allocation: 331c9de560dSAlex Tomas * load group 332c9de560dSAlex Tomas * find blocks 333c9de560dSAlex Tomas * mark bits in on-disk bitmap 334c9de560dSAlex Tomas * release group 335c9de560dSAlex Tomas * 336c9de560dSAlex Tomas * - use preallocation: 337c9de560dSAlex Tomas * find proper PA (per-inode or group) 338c9de560dSAlex Tomas * load group 339c9de560dSAlex Tomas * mark bits in on-disk bitmap 340c9de560dSAlex Tomas * release group 341c9de560dSAlex Tomas * release PA 342c9de560dSAlex Tomas * 343c9de560dSAlex Tomas * - free: 344c9de560dSAlex Tomas * load group 345c9de560dSAlex Tomas * mark bits in on-disk bitmap 346c9de560dSAlex Tomas * release group 347c9de560dSAlex Tomas * 348c9de560dSAlex Tomas * - discard preallocations in group: 349c9de560dSAlex Tomas * mark PAs deleted 350c9de560dSAlex Tomas * move them onto local list 351c9de560dSAlex Tomas * load on-disk bitmap 352c9de560dSAlex Tomas * load group 353c9de560dSAlex Tomas * remove PA from object (inode or locality group) 354c9de560dSAlex Tomas * mark free blocks in-core 355c9de560dSAlex Tomas * 356c9de560dSAlex Tomas * - discard inode's preallocations: 357c9de560dSAlex Tomas */ 358c9de560dSAlex Tomas 359c9de560dSAlex Tomas /* 360c9de560dSAlex Tomas * Locking rules 361c9de560dSAlex Tomas * 362c9de560dSAlex Tomas * Locks: 363c9de560dSAlex Tomas * - bitlock on a group (group) 364c9de560dSAlex Tomas * - object (inode/locality) (object) 365c9de560dSAlex Tomas * - per-pa lock (pa) 366f52f3d2bSOjaswin Mujoo * - cr_power2_aligned lists lock (cr_power2_aligned) 367f52f3d2bSOjaswin Mujoo * - cr_goal_len_fast lists lock (cr_goal_len_fast) 368c9de560dSAlex Tomas * 369c9de560dSAlex Tomas * Paths: 370c9de560dSAlex Tomas * - new pa 371c9de560dSAlex Tomas * object 372c9de560dSAlex Tomas * group 373c9de560dSAlex Tomas * 374c9de560dSAlex Tomas * - find and use pa: 375c9de560dSAlex Tomas * pa 376c9de560dSAlex Tomas * 377c9de560dSAlex Tomas * - release consumed pa: 378c9de560dSAlex Tomas * pa 379c9de560dSAlex Tomas * group 380c9de560dSAlex Tomas * object 381c9de560dSAlex Tomas * 382c9de560dSAlex Tomas * - generate in-core bitmap: 383c9de560dSAlex Tomas * group 384c9de560dSAlex Tomas * pa 385c9de560dSAlex Tomas * 386c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 387c9de560dSAlex Tomas * object 388c9de560dSAlex Tomas * pa 389c9de560dSAlex Tomas * group 390c9de560dSAlex Tomas * 391c9de560dSAlex Tomas * - discard all for given group: 392c9de560dSAlex Tomas * group 393c9de560dSAlex Tomas * pa 394c9de560dSAlex Tomas * group 395c9de560dSAlex Tomas * object 396c9de560dSAlex Tomas * 397196e402aSHarshad Shirwadkar * - allocation path (ext4_mb_regular_allocator) 398196e402aSHarshad Shirwadkar * group 399f52f3d2bSOjaswin Mujoo * cr_power2_aligned/cr_goal_len_fast 400c9de560dSAlex Tomas */ 401c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 402c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 40318aadd47SBobi Jam static struct kmem_cache *ext4_free_data_cachep; 404fb1813f4SCurt Wohlgemuth 405fb1813f4SCurt Wohlgemuth /* We create slab caches for groupinfo data structures based on the 406fb1813f4SCurt Wohlgemuth * superblock block size. There will be one per mounted filesystem for 407fb1813f4SCurt Wohlgemuth * each unique s_blocksize_bits */ 4082892c15dSEric Sandeen #define NR_GRPINFO_CACHES 8 409fb1813f4SCurt Wohlgemuth static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; 410fb1813f4SCurt Wohlgemuth 411d6006186SEric Biggers static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { 4122892c15dSEric Sandeen "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", 4132892c15dSEric Sandeen "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", 4142892c15dSEric Sandeen "ext4_groupinfo_64k", "ext4_groupinfo_128k" 4152892c15dSEric Sandeen }; 4162892c15dSEric Sandeen 417c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 418c3a326a6SAneesh Kumar K.V ext4_group_t group); 4197a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 4207a2fcbf7SAneesh Kumar K.V ext4_group_t group); 42153f86b17SRitesh Harjani static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac); 422c3a326a6SAneesh Kumar K.V 423196e402aSHarshad Shirwadkar static bool ext4_mb_good_group(struct ext4_allocation_context *ac, 4244eb7a4a1SOjaswin Mujoo ext4_group_t group, enum criteria cr); 425196e402aSHarshad Shirwadkar 42655cdd0afSWang Jianchao static int ext4_try_to_trim_range(struct super_block *sb, 42755cdd0afSWang Jianchao struct ext4_buddy *e4b, ext4_grpblk_t start, 42855cdd0afSWang Jianchao ext4_grpblk_t max, ext4_grpblk_t minblocks); 42955cdd0afSWang Jianchao 43007b5b8e1SRitesh Harjani /* 43107b5b8e1SRitesh Harjani * The algorithm using this percpu seq counter goes below: 43207b5b8e1SRitesh Harjani * 1. We sample the percpu discard_pa_seq counter before trying for block 43307b5b8e1SRitesh Harjani * allocation in ext4_mb_new_blocks(). 43407b5b8e1SRitesh Harjani * 2. We increment this percpu discard_pa_seq counter when we either allocate 43507b5b8e1SRitesh Harjani * or free these blocks i.e. while marking those blocks as used/free in 43607b5b8e1SRitesh Harjani * mb_mark_used()/mb_free_blocks(). 43707b5b8e1SRitesh Harjani * 3. We also increment this percpu seq counter when we successfully identify 43807b5b8e1SRitesh Harjani * that the bb_prealloc_list is not empty and hence proceed for discarding 43907b5b8e1SRitesh Harjani * of those PAs inside ext4_mb_discard_group_preallocations(). 44007b5b8e1SRitesh Harjani * 44107b5b8e1SRitesh Harjani * Now to make sure that the regular fast path of block allocation is not 44207b5b8e1SRitesh Harjani * affected, as a small optimization we only sample the percpu seq counter 44307b5b8e1SRitesh Harjani * on that cpu. Only when the block allocation fails and when freed blocks 44407b5b8e1SRitesh Harjani * found were 0, that is when we sample percpu seq counter for all cpus using 44507b5b8e1SRitesh Harjani * below function ext4_get_discard_pa_seq_sum(). This happens after making 44607b5b8e1SRitesh Harjani * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty. 44707b5b8e1SRitesh Harjani */ 44807b5b8e1SRitesh Harjani static DEFINE_PER_CPU(u64, discard_pa_seq); 44907b5b8e1SRitesh Harjani static inline u64 ext4_get_discard_pa_seq_sum(void) 45007b5b8e1SRitesh Harjani { 45107b5b8e1SRitesh Harjani int __cpu; 45207b5b8e1SRitesh Harjani u64 __seq = 0; 45307b5b8e1SRitesh Harjani 45407b5b8e1SRitesh Harjani for_each_possible_cpu(__cpu) 45507b5b8e1SRitesh Harjani __seq += per_cpu(discard_pa_seq, __cpu); 45607b5b8e1SRitesh Harjani return __seq; 45707b5b8e1SRitesh Harjani } 45807b5b8e1SRitesh Harjani 459ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 460ffad0a44SAneesh Kumar K.V { 461c9de560dSAlex Tomas #if BITS_PER_LONG == 64 462ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 463ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 464c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 465ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 466ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 467c9de560dSAlex Tomas #else 468c9de560dSAlex Tomas #error "how many bits you are?!" 469c9de560dSAlex Tomas #endif 470ffad0a44SAneesh Kumar K.V return addr; 471ffad0a44SAneesh Kumar K.V } 472c9de560dSAlex Tomas 473c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 474c9de560dSAlex Tomas { 475c9de560dSAlex Tomas /* 476c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 477c9de560dSAlex Tomas * needs unsigned long aligned address 478c9de560dSAlex Tomas */ 479ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 480c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 481c9de560dSAlex Tomas } 482c9de560dSAlex Tomas 483c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 484c9de560dSAlex Tomas { 485ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 486c9de560dSAlex Tomas ext4_set_bit(bit, addr); 487c9de560dSAlex Tomas } 488c9de560dSAlex Tomas 489c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 490c9de560dSAlex Tomas { 491ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 492c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 493c9de560dSAlex Tomas } 494c9de560dSAlex Tomas 495eabe0444SAndrey Sidorov static inline int mb_test_and_clear_bit(int bit, void *addr) 496eabe0444SAndrey Sidorov { 497eabe0444SAndrey Sidorov addr = mb_correct_addr_and_bit(&bit, addr); 498eabe0444SAndrey Sidorov return ext4_test_and_clear_bit(bit, addr); 499eabe0444SAndrey Sidorov } 500eabe0444SAndrey Sidorov 501ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 502ffad0a44SAneesh Kumar K.V { 503e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 504ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 505e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 506ffad0a44SAneesh Kumar K.V start += fix; 507ffad0a44SAneesh Kumar K.V 508e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 509e7dfb246SAneesh Kumar K.V if (ret > max) 510e7dfb246SAneesh Kumar K.V return max; 511e7dfb246SAneesh Kumar K.V return ret; 512ffad0a44SAneesh Kumar K.V } 513ffad0a44SAneesh Kumar K.V 514ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 515ffad0a44SAneesh Kumar K.V { 516e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 517ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 518e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 519ffad0a44SAneesh Kumar K.V start += fix; 520ffad0a44SAneesh Kumar K.V 521e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 522e7dfb246SAneesh Kumar K.V if (ret > max) 523e7dfb246SAneesh Kumar K.V return max; 524e7dfb246SAneesh Kumar K.V return ret; 525ffad0a44SAneesh Kumar K.V } 526ffad0a44SAneesh Kumar K.V 527c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 528c9de560dSAlex Tomas { 529c9de560dSAlex Tomas char *bb; 530c9de560dSAlex Tomas 531c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 532c9de560dSAlex Tomas BUG_ON(max == NULL); 533c9de560dSAlex Tomas 534c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 535c9de560dSAlex Tomas *max = 0; 536c9de560dSAlex Tomas return NULL; 537c9de560dSAlex Tomas } 538c9de560dSAlex Tomas 539c9de560dSAlex Tomas /* at order 0 we see each particular block */ 54084b775a3SColy Li if (order == 0) { 541c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 542c5e8f3f3STheodore Ts'o return e4b->bd_bitmap; 54384b775a3SColy Li } 544c9de560dSAlex Tomas 545c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 546c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 547c9de560dSAlex Tomas 548c9de560dSAlex Tomas return bb; 549c9de560dSAlex Tomas } 550c9de560dSAlex Tomas 551c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 552c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 553c9de560dSAlex Tomas int first, int count) 554c9de560dSAlex Tomas { 555c9de560dSAlex Tomas int i; 556c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 557c9de560dSAlex Tomas 558c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 559c9de560dSAlex Tomas return; 560bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 561c9de560dSAlex Tomas for (i = 0; i < count; i++) { 562c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 563c9de560dSAlex Tomas ext4_fsblk_t blocknr; 5645661bd68SAkinobu Mita 5655661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 56653accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), first + i); 5675d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 568e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 569e29136f8STheodore Ts'o blocknr, 570e29136f8STheodore Ts'o "freeing block already freed " 571e29136f8STheodore Ts'o "(bit %u)", 572e29136f8STheodore Ts'o first + i); 573736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 574736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 575c9de560dSAlex Tomas } 576c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 577c9de560dSAlex Tomas } 578c9de560dSAlex Tomas } 579c9de560dSAlex Tomas 580c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 581c9de560dSAlex Tomas { 582c9de560dSAlex Tomas int i; 583c9de560dSAlex Tomas 584c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 585c9de560dSAlex Tomas return; 586bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 587c9de560dSAlex Tomas for (i = 0; i < count; i++) { 588c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 589c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 590c9de560dSAlex Tomas } 591c9de560dSAlex Tomas } 592c9de560dSAlex Tomas 593c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 594c9de560dSAlex Tomas { 595eb2b8ebbSRitesh Harjani if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 596eb2b8ebbSRitesh Harjani return; 597c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 598c9de560dSAlex Tomas unsigned char *b1, *b2; 599c9de560dSAlex Tomas int i; 600c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 601c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 602c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 603c9de560dSAlex Tomas if (b1[i] != b2[i]) { 6049d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_ERR, 6059d8b9ec4STheodore Ts'o "corruption in group %u " 6064776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 6079d8b9ec4STheodore Ts'o "on disk/prealloc", 608c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 609c9de560dSAlex Tomas BUG(); 610c9de560dSAlex Tomas } 611c9de560dSAlex Tomas } 612c9de560dSAlex Tomas } 613c9de560dSAlex Tomas } 614c9de560dSAlex Tomas 615a3450215SRitesh Harjani static void mb_group_bb_bitmap_alloc(struct super_block *sb, 616a3450215SRitesh Harjani struct ext4_group_info *grp, ext4_group_t group) 617a3450215SRitesh Harjani { 618a3450215SRitesh Harjani struct buffer_head *bh; 619a3450215SRitesh Harjani 620a3450215SRitesh Harjani grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); 621eb2b8ebbSRitesh Harjani if (!grp->bb_bitmap) 622eb2b8ebbSRitesh Harjani return; 623a3450215SRitesh Harjani 624a3450215SRitesh Harjani bh = ext4_read_block_bitmap(sb, group); 625eb2b8ebbSRitesh Harjani if (IS_ERR_OR_NULL(bh)) { 626eb2b8ebbSRitesh Harjani kfree(grp->bb_bitmap); 627eb2b8ebbSRitesh Harjani grp->bb_bitmap = NULL; 628eb2b8ebbSRitesh Harjani return; 629eb2b8ebbSRitesh Harjani } 630a3450215SRitesh Harjani 631a3450215SRitesh Harjani memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize); 632a3450215SRitesh Harjani put_bh(bh); 633a3450215SRitesh Harjani } 634a3450215SRitesh Harjani 635a3450215SRitesh Harjani static void mb_group_bb_bitmap_free(struct ext4_group_info *grp) 636a3450215SRitesh Harjani { 637a3450215SRitesh Harjani kfree(grp->bb_bitmap); 638a3450215SRitesh Harjani } 639a3450215SRitesh Harjani 640c9de560dSAlex Tomas #else 641c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 642c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 643c9de560dSAlex Tomas { 644c9de560dSAlex Tomas return; 645c9de560dSAlex Tomas } 646c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 647c9de560dSAlex Tomas int first, int count) 648c9de560dSAlex Tomas { 649c9de560dSAlex Tomas return; 650c9de560dSAlex Tomas } 651c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 652c9de560dSAlex Tomas { 653c9de560dSAlex Tomas return; 654c9de560dSAlex Tomas } 655a3450215SRitesh Harjani 656a3450215SRitesh Harjani static inline void mb_group_bb_bitmap_alloc(struct super_block *sb, 657a3450215SRitesh Harjani struct ext4_group_info *grp, ext4_group_t group) 658a3450215SRitesh Harjani { 659a3450215SRitesh Harjani return; 660a3450215SRitesh Harjani } 661a3450215SRitesh Harjani 662a3450215SRitesh Harjani static inline void mb_group_bb_bitmap_free(struct ext4_group_info *grp) 663a3450215SRitesh Harjani { 664a3450215SRitesh Harjani return; 665a3450215SRitesh Harjani } 666c9de560dSAlex Tomas #endif 667c9de560dSAlex Tomas 668c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 669c9de560dSAlex Tomas 670c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 671c9de560dSAlex Tomas do { \ 672c9de560dSAlex Tomas if (!(assert)) { \ 673c9de560dSAlex Tomas printk(KERN_EMERG \ 674c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 675c9de560dSAlex Tomas function, file, line, # assert); \ 676c9de560dSAlex Tomas BUG(); \ 677c9de560dSAlex Tomas } \ 678c9de560dSAlex Tomas } while (0) 679c9de560dSAlex Tomas 680c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 681c9de560dSAlex Tomas const char *function, int line) 682c9de560dSAlex Tomas { 683c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 684c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 685c9de560dSAlex Tomas int max; 686c9de560dSAlex Tomas int max2; 687c9de560dSAlex Tomas int i; 688c9de560dSAlex Tomas int j; 689c9de560dSAlex Tomas int k; 690c9de560dSAlex Tomas int count; 691c9de560dSAlex Tomas struct ext4_group_info *grp; 692c9de560dSAlex Tomas int fragments = 0; 693c9de560dSAlex Tomas int fstart; 694c9de560dSAlex Tomas struct list_head *cur; 695c9de560dSAlex Tomas void *buddy; 696c9de560dSAlex Tomas void *buddy2; 697c9de560dSAlex Tomas 698addd752cSChunguang Xu if (e4b->bd_info->bb_check_counter++ % 10) 699c9de560dSAlex Tomas return 0; 700c9de560dSAlex Tomas 701c9de560dSAlex Tomas while (order > 1) { 702c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 703c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 704c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 705c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 706c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 707c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 708c9de560dSAlex Tomas 709c9de560dSAlex Tomas count = 0; 710c9de560dSAlex Tomas for (i = 0; i < max; i++) { 711c9de560dSAlex Tomas 712c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 713af2b3275SJinke Han /* only single bit in buddy2 may be 0 */ 714c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 715c9de560dSAlex Tomas MB_CHECK_ASSERT( 716c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 717c9de560dSAlex Tomas } 718c9de560dSAlex Tomas continue; 719c9de560dSAlex Tomas } 720c9de560dSAlex Tomas 7210a10da73SRobin Dong /* both bits in buddy2 must be 1 */ 722c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 723c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 724c9de560dSAlex Tomas 725c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 726c9de560dSAlex Tomas k = (i * (1 << order)) + j; 727c9de560dSAlex Tomas MB_CHECK_ASSERT( 728c5e8f3f3STheodore Ts'o !mb_test_bit(k, e4b->bd_bitmap)); 729c9de560dSAlex Tomas } 730c9de560dSAlex Tomas count++; 731c9de560dSAlex Tomas } 732c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 733c9de560dSAlex Tomas order--; 734c9de560dSAlex Tomas } 735c9de560dSAlex Tomas 736c9de560dSAlex Tomas fstart = -1; 737c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 738c9de560dSAlex Tomas for (i = 0; i < max; i++) { 739c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 740c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 741c9de560dSAlex Tomas if (fstart == -1) { 742c9de560dSAlex Tomas fragments++; 743c9de560dSAlex Tomas fstart = i; 744c9de560dSAlex Tomas } 745c9de560dSAlex Tomas continue; 746c9de560dSAlex Tomas } 747c9de560dSAlex Tomas fstart = -1; 748c9de560dSAlex Tomas /* check used bits only */ 749c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 750c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 751c9de560dSAlex Tomas k = i >> j; 752c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 753c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 754c9de560dSAlex Tomas } 755c9de560dSAlex Tomas } 756c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 757c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 758c9de560dSAlex Tomas 759c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 7605354b2afSTheodore Ts'o if (!grp) 7615354b2afSTheodore Ts'o return NULL; 762c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 763c9de560dSAlex Tomas ext4_group_t groupnr; 764c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 76560bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 76660bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 767c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 76860bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 769c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 770c9de560dSAlex Tomas } 771c9de560dSAlex Tomas return 0; 772c9de560dSAlex Tomas } 773c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 774c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 77546e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 776c9de560dSAlex Tomas #else 777c9de560dSAlex Tomas #define mb_check_buddy(e4b) 778c9de560dSAlex Tomas #endif 779c9de560dSAlex Tomas 7807c786059SColy Li /* 7817c786059SColy Li * Divide blocks started from @first with length @len into 7827c786059SColy Li * smaller chunks with power of 2 blocks. 7837c786059SColy Li * Clear the bits in bitmap which the blocks of the chunk(s) covered, 7847c786059SColy Li * then increase bb_counters[] for corresponded chunk size. 7857c786059SColy Li */ 786c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 787a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 788c9de560dSAlex Tomas struct ext4_group_info *grp) 789c9de560dSAlex Tomas { 790c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 791a36b4498SEric Sandeen ext4_grpblk_t min; 792a36b4498SEric Sandeen ext4_grpblk_t max; 793a36b4498SEric Sandeen ext4_grpblk_t chunk; 79469e43e8cSChandan Rajendra unsigned int border; 795c9de560dSAlex Tomas 7967137d7a4STheodore Ts'o BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); 797c9de560dSAlex Tomas 798c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 799c9de560dSAlex Tomas 800c9de560dSAlex Tomas while (len > 0) { 801c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 802c9de560dSAlex Tomas max = ffs(first | border) - 1; 803c9de560dSAlex Tomas 804c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 805c9de560dSAlex Tomas min = fls(len) - 1; 806c9de560dSAlex Tomas 807c9de560dSAlex Tomas if (max < min) 808c9de560dSAlex Tomas min = max; 809c9de560dSAlex Tomas chunk = 1 << min; 810c9de560dSAlex Tomas 811c9de560dSAlex Tomas /* mark multiblock chunks only */ 812c9de560dSAlex Tomas grp->bb_counters[min]++; 813c9de560dSAlex Tomas if (min > 0) 814c9de560dSAlex Tomas mb_clear_bit(first >> min, 815c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 816c9de560dSAlex Tomas 817c9de560dSAlex Tomas len -= chunk; 818c9de560dSAlex Tomas first += chunk; 819c9de560dSAlex Tomas } 820c9de560dSAlex Tomas } 821c9de560dSAlex Tomas 82283e80a6eSJan Kara static int mb_avg_fragment_size_order(struct super_block *sb, ext4_grpblk_t len) 823196e402aSHarshad Shirwadkar { 82483e80a6eSJan Kara int order; 825196e402aSHarshad Shirwadkar 826196e402aSHarshad Shirwadkar /* 82783e80a6eSJan Kara * We don't bother with a special lists groups with only 1 block free 82883e80a6eSJan Kara * extents and for completely empty groups. 829196e402aSHarshad Shirwadkar */ 83083e80a6eSJan Kara order = fls(len) - 2; 83183e80a6eSJan Kara if (order < 0) 83283e80a6eSJan Kara return 0; 83383e80a6eSJan Kara if (order == MB_NUM_ORDERS(sb)) 83483e80a6eSJan Kara order--; 83583e80a6eSJan Kara return order; 83683e80a6eSJan Kara } 83783e80a6eSJan Kara 83883e80a6eSJan Kara /* Move group to appropriate avg_fragment_size list */ 839196e402aSHarshad Shirwadkar static void 840196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) 841196e402aSHarshad Shirwadkar { 842196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 84383e80a6eSJan Kara int new_order; 844196e402aSHarshad Shirwadkar 845196e402aSHarshad Shirwadkar if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_free == 0) 846196e402aSHarshad Shirwadkar return; 847196e402aSHarshad Shirwadkar 84883e80a6eSJan Kara new_order = mb_avg_fragment_size_order(sb, 84983e80a6eSJan Kara grp->bb_free / grp->bb_fragments); 85083e80a6eSJan Kara if (new_order == grp->bb_avg_fragment_size_order) 85183e80a6eSJan Kara return; 852196e402aSHarshad Shirwadkar 85383e80a6eSJan Kara if (grp->bb_avg_fragment_size_order != -1) { 85483e80a6eSJan Kara write_lock(&sbi->s_mb_avg_fragment_size_locks[ 85583e80a6eSJan Kara grp->bb_avg_fragment_size_order]); 85683e80a6eSJan Kara list_del(&grp->bb_avg_fragment_size_node); 85783e80a6eSJan Kara write_unlock(&sbi->s_mb_avg_fragment_size_locks[ 85883e80a6eSJan Kara grp->bb_avg_fragment_size_order]); 85983e80a6eSJan Kara } 86083e80a6eSJan Kara grp->bb_avg_fragment_size_order = new_order; 86183e80a6eSJan Kara write_lock(&sbi->s_mb_avg_fragment_size_locks[ 86283e80a6eSJan Kara grp->bb_avg_fragment_size_order]); 86383e80a6eSJan Kara list_add_tail(&grp->bb_avg_fragment_size_node, 86483e80a6eSJan Kara &sbi->s_mb_avg_fragment_size[grp->bb_avg_fragment_size_order]); 86583e80a6eSJan Kara write_unlock(&sbi->s_mb_avg_fragment_size_locks[ 86683e80a6eSJan Kara grp->bb_avg_fragment_size_order]); 867196e402aSHarshad Shirwadkar } 868196e402aSHarshad Shirwadkar 869196e402aSHarshad Shirwadkar /* 870196e402aSHarshad Shirwadkar * Choose next group by traversing largest_free_order lists. Updates *new_cr if 871196e402aSHarshad Shirwadkar * cr level needs an update. 872196e402aSHarshad Shirwadkar */ 873f52f3d2bSOjaswin Mujoo static void ext4_mb_choose_next_group_p2_aligned(struct ext4_allocation_context *ac, 8744eb7a4a1SOjaswin Mujoo enum criteria *new_cr, ext4_group_t *group, ext4_group_t ngroups) 875196e402aSHarshad Shirwadkar { 876196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 877*919eb90cSKemeng Shi struct ext4_group_info *iter; 878196e402aSHarshad Shirwadkar int i; 879196e402aSHarshad Shirwadkar 880196e402aSHarshad Shirwadkar if (ac->ac_status == AC_STATUS_FOUND) 881196e402aSHarshad Shirwadkar return; 882196e402aSHarshad Shirwadkar 883f52f3d2bSOjaswin Mujoo if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED)) 884f52f3d2bSOjaswin Mujoo atomic_inc(&sbi->s_bal_p2_aligned_bad_suggestions); 885196e402aSHarshad Shirwadkar 886196e402aSHarshad Shirwadkar for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { 887196e402aSHarshad Shirwadkar if (list_empty(&sbi->s_mb_largest_free_orders[i])) 888196e402aSHarshad Shirwadkar continue; 889196e402aSHarshad Shirwadkar read_lock(&sbi->s_mb_largest_free_orders_locks[i]); 890196e402aSHarshad Shirwadkar if (list_empty(&sbi->s_mb_largest_free_orders[i])) { 891196e402aSHarshad Shirwadkar read_unlock(&sbi->s_mb_largest_free_orders_locks[i]); 892196e402aSHarshad Shirwadkar continue; 893196e402aSHarshad Shirwadkar } 894196e402aSHarshad Shirwadkar list_for_each_entry(iter, &sbi->s_mb_largest_free_orders[i], 895196e402aSHarshad Shirwadkar bb_largest_free_order_node) { 896196e402aSHarshad Shirwadkar if (sbi->s_mb_stats) 897f52f3d2bSOjaswin Mujoo atomic64_inc(&sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED]); 898f52f3d2bSOjaswin Mujoo if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) { 899*919eb90cSKemeng Shi *group = iter->bb_group; 900*919eb90cSKemeng Shi ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED; 901*919eb90cSKemeng Shi read_unlock(&sbi->s_mb_largest_free_orders_locks[i]); 902*919eb90cSKemeng Shi return; 903196e402aSHarshad Shirwadkar } 904196e402aSHarshad Shirwadkar } 905196e402aSHarshad Shirwadkar read_unlock(&sbi->s_mb_largest_free_orders_locks[i]); 906196e402aSHarshad Shirwadkar } 907196e402aSHarshad Shirwadkar 908*919eb90cSKemeng Shi /* Increment cr and search again if no group is found */ 909f52f3d2bSOjaswin Mujoo *new_cr = CR_GOAL_LEN_FAST; 910196e402aSHarshad Shirwadkar } 911196e402aSHarshad Shirwadkar 912196e402aSHarshad Shirwadkar /* 913856d865cSOjaswin Mujoo * Find a suitable group of given order from the average fragments list. 914856d865cSOjaswin Mujoo */ 915856d865cSOjaswin Mujoo static struct ext4_group_info * 916856d865cSOjaswin Mujoo ext4_mb_find_good_group_avg_frag_lists(struct ext4_allocation_context *ac, int order) 917856d865cSOjaswin Mujoo { 918856d865cSOjaswin Mujoo struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 919856d865cSOjaswin Mujoo struct list_head *frag_list = &sbi->s_mb_avg_fragment_size[order]; 920856d865cSOjaswin Mujoo rwlock_t *frag_list_lock = &sbi->s_mb_avg_fragment_size_locks[order]; 921856d865cSOjaswin Mujoo struct ext4_group_info *grp = NULL, *iter; 922856d865cSOjaswin Mujoo enum criteria cr = ac->ac_criteria; 923856d865cSOjaswin Mujoo 924856d865cSOjaswin Mujoo if (list_empty(frag_list)) 925856d865cSOjaswin Mujoo return NULL; 926856d865cSOjaswin Mujoo read_lock(frag_list_lock); 927856d865cSOjaswin Mujoo if (list_empty(frag_list)) { 928856d865cSOjaswin Mujoo read_unlock(frag_list_lock); 929856d865cSOjaswin Mujoo return NULL; 930856d865cSOjaswin Mujoo } 931856d865cSOjaswin Mujoo list_for_each_entry(iter, frag_list, bb_avg_fragment_size_node) { 932856d865cSOjaswin Mujoo if (sbi->s_mb_stats) 933856d865cSOjaswin Mujoo atomic64_inc(&sbi->s_bal_cX_groups_considered[cr]); 934856d865cSOjaswin Mujoo if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) { 935856d865cSOjaswin Mujoo grp = iter; 936856d865cSOjaswin Mujoo break; 937856d865cSOjaswin Mujoo } 938856d865cSOjaswin Mujoo } 939856d865cSOjaswin Mujoo read_unlock(frag_list_lock); 940856d865cSOjaswin Mujoo return grp; 941856d865cSOjaswin Mujoo } 942856d865cSOjaswin Mujoo 943856d865cSOjaswin Mujoo /* 94483e80a6eSJan Kara * Choose next group by traversing average fragment size list of suitable 94583e80a6eSJan Kara * order. Updates *new_cr if cr level needs an update. 946196e402aSHarshad Shirwadkar */ 947f52f3d2bSOjaswin Mujoo static void ext4_mb_choose_next_group_goal_fast(struct ext4_allocation_context *ac, 9484eb7a4a1SOjaswin Mujoo enum criteria *new_cr, ext4_group_t *group, ext4_group_t ngroups) 949196e402aSHarshad Shirwadkar { 950196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 951856d865cSOjaswin Mujoo struct ext4_group_info *grp = NULL; 95283e80a6eSJan Kara int i; 953196e402aSHarshad Shirwadkar 954f52f3d2bSOjaswin Mujoo if (unlikely(ac->ac_flags & EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED)) { 955196e402aSHarshad Shirwadkar if (sbi->s_mb_stats) 956f52f3d2bSOjaswin Mujoo atomic_inc(&sbi->s_bal_goal_fast_bad_suggestions); 95783e80a6eSJan Kara } 95883e80a6eSJan Kara 95983e80a6eSJan Kara for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); 96083e80a6eSJan Kara i < MB_NUM_ORDERS(ac->ac_sb); i++) { 961856d865cSOjaswin Mujoo grp = ext4_mb_find_good_group_avg_frag_lists(ac, i); 96283e80a6eSJan Kara if (grp) 96383e80a6eSJan Kara break; 964196e402aSHarshad Shirwadkar } 965196e402aSHarshad Shirwadkar 96683e80a6eSJan Kara if (grp) { 967196e402aSHarshad Shirwadkar *group = grp->bb_group; 968f52f3d2bSOjaswin Mujoo ac->ac_flags |= EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED; 969196e402aSHarshad Shirwadkar } else { 970f52f3d2bSOjaswin Mujoo *new_cr = CR_BEST_AVAIL_LEN; 9717e170922SOjaswin Mujoo } 9727e170922SOjaswin Mujoo } 9737e170922SOjaswin Mujoo 9747e170922SOjaswin Mujoo /* 975f52f3d2bSOjaswin Mujoo * We couldn't find a group in CR_GOAL_LEN_FAST so try to find the highest free fragment 9767e170922SOjaswin Mujoo * order we have and proactively trim the goal request length to that order to 9777e170922SOjaswin Mujoo * find a suitable group faster. 9787e170922SOjaswin Mujoo * 9797e170922SOjaswin Mujoo * This optimizes allocation speed at the cost of slightly reduced 9807e170922SOjaswin Mujoo * preallocations. However, we make sure that we don't trim the request too 981f52f3d2bSOjaswin Mujoo * much and fall to CR_GOAL_LEN_SLOW in that case. 9827e170922SOjaswin Mujoo */ 983f52f3d2bSOjaswin Mujoo static void ext4_mb_choose_next_group_best_avail(struct ext4_allocation_context *ac, 9847e170922SOjaswin Mujoo enum criteria *new_cr, ext4_group_t *group, ext4_group_t ngroups) 9857e170922SOjaswin Mujoo { 9867e170922SOjaswin Mujoo struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 9877e170922SOjaswin Mujoo struct ext4_group_info *grp = NULL; 9887e170922SOjaswin Mujoo int i, order, min_order; 9897e170922SOjaswin Mujoo unsigned long num_stripe_clusters = 0; 9907e170922SOjaswin Mujoo 991f52f3d2bSOjaswin Mujoo if (unlikely(ac->ac_flags & EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED)) { 9927e170922SOjaswin Mujoo if (sbi->s_mb_stats) 993f52f3d2bSOjaswin Mujoo atomic_inc(&sbi->s_bal_best_avail_bad_suggestions); 9947e170922SOjaswin Mujoo } 9957e170922SOjaswin Mujoo 9967e170922SOjaswin Mujoo /* 9977e170922SOjaswin Mujoo * mb_avg_fragment_size_order() returns order in a way that makes 9987e170922SOjaswin Mujoo * retrieving back the length using (1 << order) inaccurate. Hence, use 9997e170922SOjaswin Mujoo * fls() instead since we need to know the actual length while modifying 10007e170922SOjaswin Mujoo * goal length. 10017e170922SOjaswin Mujoo */ 10025d5460faSOjaswin Mujoo order = fls(ac->ac_g_ex.fe_len) - 1; 1003f52f3d2bSOjaswin Mujoo min_order = order - sbi->s_mb_best_avail_max_trim_order; 10047e170922SOjaswin Mujoo if (min_order < 0) 10057e170922SOjaswin Mujoo min_order = 0; 10067e170922SOjaswin Mujoo 10077e170922SOjaswin Mujoo if (sbi->s_stripe > 0) { 10087e170922SOjaswin Mujoo /* 10097e170922SOjaswin Mujoo * We are assuming that stripe size is always a multiple of 10107e170922SOjaswin Mujoo * cluster ratio otherwise __ext4_fill_super exists early. 10117e170922SOjaswin Mujoo */ 10127e170922SOjaswin Mujoo num_stripe_clusters = EXT4_NUM_B2C(sbi, sbi->s_stripe); 10137e170922SOjaswin Mujoo if (1 << min_order < num_stripe_clusters) 10145d5460faSOjaswin Mujoo /* 10155d5460faSOjaswin Mujoo * We consider 1 order less because later we round 10165d5460faSOjaswin Mujoo * up the goal len to num_stripe_clusters 10175d5460faSOjaswin Mujoo */ 10185d5460faSOjaswin Mujoo min_order = fls(num_stripe_clusters) - 1; 10197e170922SOjaswin Mujoo } 10207e170922SOjaswin Mujoo 10215d5460faSOjaswin Mujoo if (1 << min_order < ac->ac_o_ex.fe_len) 10225d5460faSOjaswin Mujoo min_order = fls(ac->ac_o_ex.fe_len); 10235d5460faSOjaswin Mujoo 10247e170922SOjaswin Mujoo for (i = order; i >= min_order; i--) { 10257e170922SOjaswin Mujoo int frag_order; 10267e170922SOjaswin Mujoo /* 10277e170922SOjaswin Mujoo * Scale down goal len to make sure we find something 10287e170922SOjaswin Mujoo * in the free fragments list. Basically, reduce 10297e170922SOjaswin Mujoo * preallocations. 10307e170922SOjaswin Mujoo */ 10317e170922SOjaswin Mujoo ac->ac_g_ex.fe_len = 1 << i; 10327e170922SOjaswin Mujoo 10337e170922SOjaswin Mujoo if (num_stripe_clusters > 0) { 10347e170922SOjaswin Mujoo /* 10354c0cfebdSTheodore Ts'o * Try to round up the adjusted goal length to 10364c0cfebdSTheodore Ts'o * stripe size (in cluster units) multiple for 10374c0cfebdSTheodore Ts'o * efficiency. 10387e170922SOjaswin Mujoo */ 10397e170922SOjaswin Mujoo ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len, 10407e170922SOjaswin Mujoo num_stripe_clusters); 10417e170922SOjaswin Mujoo } 10427e170922SOjaswin Mujoo 10437e170922SOjaswin Mujoo frag_order = mb_avg_fragment_size_order(ac->ac_sb, 10447e170922SOjaswin Mujoo ac->ac_g_ex.fe_len); 10457e170922SOjaswin Mujoo 10467e170922SOjaswin Mujoo grp = ext4_mb_find_good_group_avg_frag_lists(ac, frag_order); 10477e170922SOjaswin Mujoo if (grp) 10487e170922SOjaswin Mujoo break; 10497e170922SOjaswin Mujoo } 10507e170922SOjaswin Mujoo 10517e170922SOjaswin Mujoo if (grp) { 10527e170922SOjaswin Mujoo *group = grp->bb_group; 1053f52f3d2bSOjaswin Mujoo ac->ac_flags |= EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED; 10547e170922SOjaswin Mujoo } else { 1055f52f3d2bSOjaswin Mujoo /* Reset goal length to original goal length before falling into CR_GOAL_LEN_SLOW */ 10567e170922SOjaswin Mujoo ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; 1057f52f3d2bSOjaswin Mujoo *new_cr = CR_GOAL_LEN_SLOW; 1058196e402aSHarshad Shirwadkar } 1059196e402aSHarshad Shirwadkar } 1060196e402aSHarshad Shirwadkar 1061196e402aSHarshad Shirwadkar static inline int should_optimize_scan(struct ext4_allocation_context *ac) 1062196e402aSHarshad Shirwadkar { 1063196e402aSHarshad Shirwadkar if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN))) 1064196e402aSHarshad Shirwadkar return 0; 1065f52f3d2bSOjaswin Mujoo if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) 1066196e402aSHarshad Shirwadkar return 0; 1067077d0c2cSOjaswin Mujoo if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) 1068196e402aSHarshad Shirwadkar return 0; 1069196e402aSHarshad Shirwadkar return 1; 1070196e402aSHarshad Shirwadkar } 1071196e402aSHarshad Shirwadkar 1072196e402aSHarshad Shirwadkar /* 1073196e402aSHarshad Shirwadkar * Return next linear group for allocation. If linear traversal should not be 1074196e402aSHarshad Shirwadkar * performed, this function just returns the same group 1075196e402aSHarshad Shirwadkar */ 107660c672b7SKemeng Shi static ext4_group_t 107760c672b7SKemeng Shi next_linear_group(struct ext4_allocation_context *ac, ext4_group_t group, 107860c672b7SKemeng Shi ext4_group_t ngroups) 1079196e402aSHarshad Shirwadkar { 1080196e402aSHarshad Shirwadkar if (!should_optimize_scan(ac)) 1081196e402aSHarshad Shirwadkar goto inc_and_return; 1082196e402aSHarshad Shirwadkar 1083196e402aSHarshad Shirwadkar if (ac->ac_groups_linear_remaining) { 1084196e402aSHarshad Shirwadkar ac->ac_groups_linear_remaining--; 1085196e402aSHarshad Shirwadkar goto inc_and_return; 1086196e402aSHarshad Shirwadkar } 1087196e402aSHarshad Shirwadkar 1088196e402aSHarshad Shirwadkar return group; 1089196e402aSHarshad Shirwadkar inc_and_return: 1090196e402aSHarshad Shirwadkar /* 1091196e402aSHarshad Shirwadkar * Artificially restricted ngroups for non-extent 1092196e402aSHarshad Shirwadkar * files makes group > ngroups possible on first loop. 1093196e402aSHarshad Shirwadkar */ 1094196e402aSHarshad Shirwadkar return group + 1 >= ngroups ? 0 : group + 1; 1095196e402aSHarshad Shirwadkar } 1096196e402aSHarshad Shirwadkar 1097196e402aSHarshad Shirwadkar /* 1098196e402aSHarshad Shirwadkar * ext4_mb_choose_next_group: choose next group for allocation. 1099196e402aSHarshad Shirwadkar * 1100196e402aSHarshad Shirwadkar * @ac Allocation Context 1101196e402aSHarshad Shirwadkar * @new_cr This is an output parameter. If the there is no good group 1102196e402aSHarshad Shirwadkar * available at current CR level, this field is updated to indicate 1103196e402aSHarshad Shirwadkar * the new cr level that should be used. 1104196e402aSHarshad Shirwadkar * @group This is an input / output parameter. As an input it indicates the 1105196e402aSHarshad Shirwadkar * next group that the allocator intends to use for allocation. As 1106196e402aSHarshad Shirwadkar * output, this field indicates the next group that should be used as 1107196e402aSHarshad Shirwadkar * determined by the optimization functions. 1108196e402aSHarshad Shirwadkar * @ngroups Total number of groups 1109196e402aSHarshad Shirwadkar */ 1110196e402aSHarshad Shirwadkar static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, 11114eb7a4a1SOjaswin Mujoo enum criteria *new_cr, ext4_group_t *group, ext4_group_t ngroups) 1112196e402aSHarshad Shirwadkar { 1113196e402aSHarshad Shirwadkar *new_cr = ac->ac_criteria; 1114196e402aSHarshad Shirwadkar 11154fca50d4SJan Kara if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) { 11164fca50d4SJan Kara *group = next_linear_group(ac, *group, ngroups); 1117196e402aSHarshad Shirwadkar return; 11184fca50d4SJan Kara } 1119196e402aSHarshad Shirwadkar 1120f52f3d2bSOjaswin Mujoo if (*new_cr == CR_POWER2_ALIGNED) { 1121f52f3d2bSOjaswin Mujoo ext4_mb_choose_next_group_p2_aligned(ac, new_cr, group, ngroups); 1122f52f3d2bSOjaswin Mujoo } else if (*new_cr == CR_GOAL_LEN_FAST) { 1123f52f3d2bSOjaswin Mujoo ext4_mb_choose_next_group_goal_fast(ac, new_cr, group, ngroups); 1124f52f3d2bSOjaswin Mujoo } else if (*new_cr == CR_BEST_AVAIL_LEN) { 1125f52f3d2bSOjaswin Mujoo ext4_mb_choose_next_group_best_avail(ac, new_cr, group, ngroups); 1126196e402aSHarshad Shirwadkar } else { 1127196e402aSHarshad Shirwadkar /* 1128196e402aSHarshad Shirwadkar * TODO: For CR=2, we can arrange groups in an rb tree sorted by 1129196e402aSHarshad Shirwadkar * bb_free. But until that happens, we should never come here. 1130196e402aSHarshad Shirwadkar */ 1131196e402aSHarshad Shirwadkar WARN_ON(1); 1132196e402aSHarshad Shirwadkar } 1133196e402aSHarshad Shirwadkar } 1134196e402aSHarshad Shirwadkar 11358a57d9d6SCurt Wohlgemuth /* 11368a57d9d6SCurt Wohlgemuth * Cache the order of the largest free extent we have available in this block 11378a57d9d6SCurt Wohlgemuth * group. 11388a57d9d6SCurt Wohlgemuth */ 11398a57d9d6SCurt Wohlgemuth static void 11408a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) 11418a57d9d6SCurt Wohlgemuth { 1142196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 11438a57d9d6SCurt Wohlgemuth int i; 11448a57d9d6SCurt Wohlgemuth 11451940265eSJan Kara for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) 11461940265eSJan Kara if (grp->bb_counters[i] > 0) 11471940265eSJan Kara break; 11481940265eSJan Kara /* No need to move between order lists? */ 11491940265eSJan Kara if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || 11501940265eSJan Kara i == grp->bb_largest_free_order) { 11511940265eSJan Kara grp->bb_largest_free_order = i; 11521940265eSJan Kara return; 11531940265eSJan Kara } 11541940265eSJan Kara 11551940265eSJan Kara if (grp->bb_largest_free_order >= 0) { 1156196e402aSHarshad Shirwadkar write_lock(&sbi->s_mb_largest_free_orders_locks[ 1157196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1158196e402aSHarshad Shirwadkar list_del_init(&grp->bb_largest_free_order_node); 1159196e402aSHarshad Shirwadkar write_unlock(&sbi->s_mb_largest_free_orders_locks[ 1160196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1161196e402aSHarshad Shirwadkar } 11628a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = i; 11631940265eSJan Kara if (grp->bb_largest_free_order >= 0 && grp->bb_free) { 1164196e402aSHarshad Shirwadkar write_lock(&sbi->s_mb_largest_free_orders_locks[ 1165196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1166196e402aSHarshad Shirwadkar list_add_tail(&grp->bb_largest_free_order_node, 1167196e402aSHarshad Shirwadkar &sbi->s_mb_largest_free_orders[grp->bb_largest_free_order]); 1168196e402aSHarshad Shirwadkar write_unlock(&sbi->s_mb_largest_free_orders_locks[ 1169196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1170196e402aSHarshad Shirwadkar } 11718a57d9d6SCurt Wohlgemuth } 11728a57d9d6SCurt Wohlgemuth 1173089ceeccSEric Sandeen static noinline_for_stack 1174089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 11755354b2afSTheodore Ts'o void *buddy, void *bitmap, ext4_group_t group, 11765354b2afSTheodore Ts'o struct ext4_group_info *grp) 1177c9de560dSAlex Tomas { 1178e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 11797137d7a4STheodore Ts'o ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 1180a36b4498SEric Sandeen ext4_grpblk_t i = 0; 1181a36b4498SEric Sandeen ext4_grpblk_t first; 1182a36b4498SEric Sandeen ext4_grpblk_t len; 1183c9de560dSAlex Tomas unsigned free = 0; 1184c9de560dSAlex Tomas unsigned fragments = 0; 1185c9de560dSAlex Tomas unsigned long long period = get_cycles(); 1186c9de560dSAlex Tomas 1187c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 1188c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 1189ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 1190c9de560dSAlex Tomas grp->bb_first_free = i; 1191c9de560dSAlex Tomas while (i < max) { 1192c9de560dSAlex Tomas fragments++; 1193c9de560dSAlex Tomas first = i; 1194ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 1195c9de560dSAlex Tomas len = i - first; 1196c9de560dSAlex Tomas free += len; 1197c9de560dSAlex Tomas if (len > 1) 1198c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 1199c9de560dSAlex Tomas else 1200c9de560dSAlex Tomas grp->bb_counters[0]++; 1201c9de560dSAlex Tomas if (i < max) 1202ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 1203c9de560dSAlex Tomas } 1204c9de560dSAlex Tomas grp->bb_fragments = fragments; 1205c9de560dSAlex Tomas 1206c9de560dSAlex Tomas if (free != grp->bb_free) { 1207e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, 120894d4c066STheodore Ts'o "block bitmap and bg descriptor " 120994d4c066STheodore Ts'o "inconsistent: %u vs %u free clusters", 1210e29136f8STheodore Ts'o free, grp->bb_free); 1211e56eb659SAneesh Kumar K.V /* 1212163a203dSDarrick J. Wong * If we intend to continue, we consider group descriptor 1213e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 1214e56eb659SAneesh Kumar K.V */ 1215c9de560dSAlex Tomas grp->bb_free = free; 1216db79e6d1SWang Shilong ext4_mark_group_bitmap_corrupted(sb, group, 1217db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1218c9de560dSAlex Tomas } 12198a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, grp); 122083e80a6eSJan Kara mb_update_avg_fragment_size(sb, grp); 1221c9de560dSAlex Tomas 1222c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 1223c9de560dSAlex Tomas 1224c9de560dSAlex Tomas period = get_cycles() - period; 122567d25186SHarshad Shirwadkar atomic_inc(&sbi->s_mb_buddies_generated); 122667d25186SHarshad Shirwadkar atomic64_add(period, &sbi->s_mb_generation_time); 1227c9de560dSAlex Tomas } 1228c9de560dSAlex Tomas 1229c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 1230c9de560dSAlex Tomas * for convenience. The information regarding each group 1231c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 1232c9de560dSAlex Tomas * block bitmap and buddy information. The information are 1233c9de560dSAlex Tomas * stored in the inode as 1234c9de560dSAlex Tomas * 1235c9de560dSAlex Tomas * { page } 1236c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 1237c9de560dSAlex Tomas * 1238c9de560dSAlex Tomas * 1239c9de560dSAlex Tomas * one block each for bitmap and buddy information. 1240c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 1241ea1754a0SKirill A. Shutemov * contain blocks_per_page (PAGE_SIZE / blocksize) blocks. 1242c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 1243c9de560dSAlex Tomas * is blocks_per_page/2 12448a57d9d6SCurt Wohlgemuth * 12458a57d9d6SCurt Wohlgemuth * Locking note: This routine takes the block group lock of all groups 12468a57d9d6SCurt Wohlgemuth * for this page; do not hold this lock when calling this routine! 1247c9de560dSAlex Tomas */ 1248c9de560dSAlex Tomas 1249adb7ef60SKonstantin Khlebnikov static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) 1250c9de560dSAlex Tomas { 12518df9675fSTheodore Ts'o ext4_group_t ngroups; 1252c9de560dSAlex Tomas int blocksize; 1253c9de560dSAlex Tomas int blocks_per_page; 1254c9de560dSAlex Tomas int groups_per_page; 1255c9de560dSAlex Tomas int err = 0; 1256c9de560dSAlex Tomas int i; 1257813e5727STheodore Ts'o ext4_group_t first_group, group; 1258c9de560dSAlex Tomas int first_block; 1259c9de560dSAlex Tomas struct super_block *sb; 1260c9de560dSAlex Tomas struct buffer_head *bhs; 1261fa77dcfaSDarrick J. Wong struct buffer_head **bh = NULL; 1262c9de560dSAlex Tomas struct inode *inode; 1263c9de560dSAlex Tomas char *data; 1264c9de560dSAlex Tomas char *bitmap; 12659b8b7d35SAmir Goldstein struct ext4_group_info *grinfo; 1266c9de560dSAlex Tomas 1267c9de560dSAlex Tomas inode = page->mapping->host; 1268c9de560dSAlex Tomas sb = inode->i_sb; 12698df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 127093407472SFabian Frederick blocksize = i_blocksize(inode); 127109cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / blocksize; 1272c9de560dSAlex Tomas 1273d3df1453SRitesh Harjani mb_debug(sb, "init page %lu\n", page->index); 1274d3df1453SRitesh Harjani 1275c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 1276c9de560dSAlex Tomas if (groups_per_page == 0) 1277c9de560dSAlex Tomas groups_per_page = 1; 1278c9de560dSAlex Tomas 1279c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 1280c9de560dSAlex Tomas if (groups_per_page > 1) { 1281c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 1282adb7ef60SKonstantin Khlebnikov bh = kzalloc(i, gfp); 1283139f46d3SKemeng Shi if (bh == NULL) 1284139f46d3SKemeng Shi return -ENOMEM; 1285c9de560dSAlex Tomas } else 1286c9de560dSAlex Tomas bh = &bhs; 1287c9de560dSAlex Tomas 1288c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 1289c9de560dSAlex Tomas 1290c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 1291813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 1292813e5727STheodore Ts'o if (group >= ngroups) 1293c9de560dSAlex Tomas break; 1294c9de560dSAlex Tomas 1295813e5727STheodore Ts'o grinfo = ext4_get_group_info(sb, group); 12965354b2afSTheodore Ts'o if (!grinfo) 12975354b2afSTheodore Ts'o continue; 12989b8b7d35SAmir Goldstein /* 12999b8b7d35SAmir Goldstein * If page is uptodate then we came here after online resize 13009b8b7d35SAmir Goldstein * which added some new uninitialized group info structs, so 13019b8b7d35SAmir Goldstein * we must skip all initialized uptodate buddies on the page, 13029b8b7d35SAmir Goldstein * which may be currently in use by an allocating task. 13039b8b7d35SAmir Goldstein */ 13049b8b7d35SAmir Goldstein if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { 13059b8b7d35SAmir Goldstein bh[i] = NULL; 13069b8b7d35SAmir Goldstein continue; 13079b8b7d35SAmir Goldstein } 1308cfd73237SAlex Zhuravlev bh[i] = ext4_read_block_bitmap_nowait(sb, group, false); 13099008a58eSDarrick J. Wong if (IS_ERR(bh[i])) { 13109008a58eSDarrick J. Wong err = PTR_ERR(bh[i]); 13119008a58eSDarrick J. Wong bh[i] = NULL; 1312c9de560dSAlex Tomas goto out; 13132ccb5fb9SAneesh Kumar K.V } 1314d3df1453SRitesh Harjani mb_debug(sb, "read bitmap for group %u\n", group); 1315c9de560dSAlex Tomas } 1316c9de560dSAlex Tomas 1317c9de560dSAlex Tomas /* wait for I/O completion */ 1318813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 13199008a58eSDarrick J. Wong int err2; 13209008a58eSDarrick J. Wong 13219008a58eSDarrick J. Wong if (!bh[i]) 13229008a58eSDarrick J. Wong continue; 13239008a58eSDarrick J. Wong err2 = ext4_wait_block_bitmap(sb, group, bh[i]); 13249008a58eSDarrick J. Wong if (!err) 13259008a58eSDarrick J. Wong err = err2; 1326813e5727STheodore Ts'o } 1327c9de560dSAlex Tomas 1328c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 1329c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 1330c9de560dSAlex Tomas group = (first_block + i) >> 1; 13318df9675fSTheodore Ts'o if (group >= ngroups) 1332c9de560dSAlex Tomas break; 1333c9de560dSAlex Tomas 13349b8b7d35SAmir Goldstein if (!bh[group - first_group]) 13359b8b7d35SAmir Goldstein /* skip initialized uptodate buddy */ 13369b8b7d35SAmir Goldstein continue; 13379b8b7d35SAmir Goldstein 1338bbdc322fSLukas Czerner if (!buffer_verified(bh[group - first_group])) 1339bbdc322fSLukas Czerner /* Skip faulty bitmaps */ 1340bbdc322fSLukas Czerner continue; 1341bbdc322fSLukas Czerner err = 0; 1342bbdc322fSLukas Czerner 1343c9de560dSAlex Tomas /* 1344c9de560dSAlex Tomas * data carry information regarding this 1345c9de560dSAlex Tomas * particular group in the format specified 1346c9de560dSAlex Tomas * above 1347c9de560dSAlex Tomas * 1348c9de560dSAlex Tomas */ 1349c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 1350c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 1351c9de560dSAlex Tomas 1352c9de560dSAlex Tomas /* 1353c9de560dSAlex Tomas * We place the buddy block and bitmap block 1354c9de560dSAlex Tomas * close together 1355c9de560dSAlex Tomas */ 1356c9de560dSAlex Tomas if ((first_block + i) & 1) { 1357c9de560dSAlex Tomas /* this is block of buddy */ 1358c9de560dSAlex Tomas BUG_ON(incore == NULL); 1359d3df1453SRitesh Harjani mb_debug(sb, "put buddy for group %u in page %lu/%x\n", 1360c9de560dSAlex Tomas group, page->index, i * blocksize); 1361f307333eSTheodore Ts'o trace_ext4_mb_buddy_bitmap_load(sb, group); 1362c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 13635354b2afSTheodore Ts'o if (!grinfo) { 13645354b2afSTheodore Ts'o err = -EFSCORRUPTED; 13655354b2afSTheodore Ts'o goto out; 13665354b2afSTheodore Ts'o } 1367c9de560dSAlex Tomas grinfo->bb_fragments = 0; 1368c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 13691927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 13704b68f6dfSHarshad Shirwadkar (MB_NUM_ORDERS(sb))); 1371c9de560dSAlex Tomas /* 1372c9de560dSAlex Tomas * incore got set to the group block bitmap below 1373c9de560dSAlex Tomas */ 13747a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 13759b8b7d35SAmir Goldstein /* init the buddy */ 13769b8b7d35SAmir Goldstein memset(data, 0xff, blocksize); 13775354b2afSTheodore Ts'o ext4_mb_generate_buddy(sb, data, incore, group, grinfo); 13787a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 1379c9de560dSAlex Tomas incore = NULL; 1380c9de560dSAlex Tomas } else { 1381c9de560dSAlex Tomas /* this is block of bitmap */ 1382c9de560dSAlex Tomas BUG_ON(incore != NULL); 1383d3df1453SRitesh Harjani mb_debug(sb, "put bitmap for group %u in page %lu/%x\n", 1384c9de560dSAlex Tomas group, page->index, i * blocksize); 1385f307333eSTheodore Ts'o trace_ext4_mb_bitmap_load(sb, group); 1386c9de560dSAlex Tomas 1387c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 1388c9de560dSAlex Tomas ext4_lock_group(sb, group); 1389c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 1390c9de560dSAlex Tomas 1391c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 1392c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 13937a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 1394c9de560dSAlex Tomas ext4_unlock_group(sb, group); 1395c9de560dSAlex Tomas 1396c9de560dSAlex Tomas /* set incore so that the buddy information can be 1397c9de560dSAlex Tomas * generated using this 1398c9de560dSAlex Tomas */ 1399c9de560dSAlex Tomas incore = data; 1400c9de560dSAlex Tomas } 1401c9de560dSAlex Tomas } 1402c9de560dSAlex Tomas SetPageUptodate(page); 1403c9de560dSAlex Tomas 1404c9de560dSAlex Tomas out: 1405c9de560dSAlex Tomas if (bh) { 14069b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 1407c9de560dSAlex Tomas brelse(bh[i]); 1408c9de560dSAlex Tomas if (bh != &bhs) 1409c9de560dSAlex Tomas kfree(bh); 1410c9de560dSAlex Tomas } 1411c9de560dSAlex Tomas return err; 1412c9de560dSAlex Tomas } 1413c9de560dSAlex Tomas 14148a57d9d6SCurt Wohlgemuth /* 14152de8807bSAmir Goldstein * Lock the buddy and bitmap pages. This make sure other parallel init_group 14162de8807bSAmir Goldstein * on the same buddy page doesn't happen whild holding the buddy page lock. 14172de8807bSAmir Goldstein * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap 14182de8807bSAmir Goldstein * are on the same page e4b->bd_buddy_page is NULL and return value is 0. 1419eee4adc7SEric Sandeen */ 14202de8807bSAmir Goldstein static int ext4_mb_get_buddy_page_lock(struct super_block *sb, 1421adb7ef60SKonstantin Khlebnikov ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp) 1422eee4adc7SEric Sandeen { 14232de8807bSAmir Goldstein struct inode *inode = EXT4_SB(sb)->s_buddy_cache; 14242de8807bSAmir Goldstein int block, pnum, poff; 1425eee4adc7SEric Sandeen int blocks_per_page; 14262de8807bSAmir Goldstein struct page *page; 14272de8807bSAmir Goldstein 14282de8807bSAmir Goldstein e4b->bd_buddy_page = NULL; 14292de8807bSAmir Goldstein e4b->bd_bitmap_page = NULL; 1430eee4adc7SEric Sandeen 143109cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1432eee4adc7SEric Sandeen /* 1433eee4adc7SEric Sandeen * the buddy cache inode stores the block bitmap 1434eee4adc7SEric Sandeen * and buddy information in consecutive blocks. 1435eee4adc7SEric Sandeen * So for each group we need two blocks. 1436eee4adc7SEric Sandeen */ 1437eee4adc7SEric Sandeen block = group * 2; 1438eee4adc7SEric Sandeen pnum = block / blocks_per_page; 14392de8807bSAmir Goldstein poff = block % blocks_per_page; 1440adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 14412de8807bSAmir Goldstein if (!page) 1442c57ab39bSYounger Liu return -ENOMEM; 14432de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 14442de8807bSAmir Goldstein e4b->bd_bitmap_page = page; 14452de8807bSAmir Goldstein e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1446eee4adc7SEric Sandeen 14472de8807bSAmir Goldstein if (blocks_per_page >= 2) { 14482de8807bSAmir Goldstein /* buddy and bitmap are on the same page */ 14492de8807bSAmir Goldstein return 0; 1450eee4adc7SEric Sandeen } 1451eee4adc7SEric Sandeen 14522de8807bSAmir Goldstein block++; 1453eee4adc7SEric Sandeen pnum = block / blocks_per_page; 1454adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 14552de8807bSAmir Goldstein if (!page) 1456c57ab39bSYounger Liu return -ENOMEM; 14572de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 14582de8807bSAmir Goldstein e4b->bd_buddy_page = page; 14592de8807bSAmir Goldstein return 0; 1460eee4adc7SEric Sandeen } 1461eee4adc7SEric Sandeen 14622de8807bSAmir Goldstein static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) 14632de8807bSAmir Goldstein { 14642de8807bSAmir Goldstein if (e4b->bd_bitmap_page) { 14652de8807bSAmir Goldstein unlock_page(e4b->bd_bitmap_page); 146609cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 14672de8807bSAmir Goldstein } 14682de8807bSAmir Goldstein if (e4b->bd_buddy_page) { 14692de8807bSAmir Goldstein unlock_page(e4b->bd_buddy_page); 147009cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 14712de8807bSAmir Goldstein } 1472eee4adc7SEric Sandeen } 1473eee4adc7SEric Sandeen 1474eee4adc7SEric Sandeen /* 14758a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 14768a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 14778a57d9d6SCurt Wohlgemuth * calling this routine! 14788a57d9d6SCurt Wohlgemuth */ 1479b6a758ecSAneesh Kumar K.V static noinline_for_stack 1480adb7ef60SKonstantin Khlebnikov int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) 1481b6a758ecSAneesh Kumar K.V { 1482b6a758ecSAneesh Kumar K.V 1483b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 14842de8807bSAmir Goldstein struct ext4_buddy e4b; 14852de8807bSAmir Goldstein struct page *page; 14862de8807bSAmir Goldstein int ret = 0; 1487b6a758ecSAneesh Kumar K.V 1488b10a44c3STheodore Ts'o might_sleep(); 1489d3df1453SRitesh Harjani mb_debug(sb, "init group %u\n", group); 1490b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 14915354b2afSTheodore Ts'o if (!this_grp) 14925354b2afSTheodore Ts'o return -EFSCORRUPTED; 14935354b2afSTheodore Ts'o 1494b6a758ecSAneesh Kumar K.V /* 149508c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 149608c3a813SAneesh Kumar K.V * page which map to the group from which we are already 149708c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 149808c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 14992de8807bSAmir Goldstein * would have pinned buddy page to page cache. 15002457aec6SMel Gorman * The call to ext4_mb_get_buddy_page_lock will mark the 15012457aec6SMel Gorman * page accessed. 1502b6a758ecSAneesh Kumar K.V */ 1503adb7ef60SKonstantin Khlebnikov ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp); 15042de8807bSAmir Goldstein if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { 1505b6a758ecSAneesh Kumar K.V /* 1506b6a758ecSAneesh Kumar K.V * somebody initialized the group 1507b6a758ecSAneesh Kumar K.V * return without doing anything 1508b6a758ecSAneesh Kumar K.V */ 1509b6a758ecSAneesh Kumar K.V goto err; 1510b6a758ecSAneesh Kumar K.V } 15112de8807bSAmir Goldstein 15122de8807bSAmir Goldstein page = e4b.bd_bitmap_page; 1513adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 15142de8807bSAmir Goldstein if (ret) 1515b6a758ecSAneesh Kumar K.V goto err; 15162de8807bSAmir Goldstein if (!PageUptodate(page)) { 1517b6a758ecSAneesh Kumar K.V ret = -EIO; 1518b6a758ecSAneesh Kumar K.V goto err; 1519b6a758ecSAneesh Kumar K.V } 1520b6a758ecSAneesh Kumar K.V 15212de8807bSAmir Goldstein if (e4b.bd_buddy_page == NULL) { 1522b6a758ecSAneesh Kumar K.V /* 1523b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 1524b6a758ecSAneesh Kumar K.V * the same page we don't need to force 1525b6a758ecSAneesh Kumar K.V * init the buddy 1526b6a758ecSAneesh Kumar K.V */ 15272de8807bSAmir Goldstein ret = 0; 1528b6a758ecSAneesh Kumar K.V goto err; 1529b6a758ecSAneesh Kumar K.V } 15302de8807bSAmir Goldstein /* init buddy cache */ 15312de8807bSAmir Goldstein page = e4b.bd_buddy_page; 1532adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp); 15332de8807bSAmir Goldstein if (ret) 15342de8807bSAmir Goldstein goto err; 15352de8807bSAmir Goldstein if (!PageUptodate(page)) { 1536b6a758ecSAneesh Kumar K.V ret = -EIO; 1537b6a758ecSAneesh Kumar K.V goto err; 1538b6a758ecSAneesh Kumar K.V } 1539b6a758ecSAneesh Kumar K.V err: 15402de8807bSAmir Goldstein ext4_mb_put_buddy_page_lock(&e4b); 1541b6a758ecSAneesh Kumar K.V return ret; 1542b6a758ecSAneesh Kumar K.V } 1543b6a758ecSAneesh Kumar K.V 15448a57d9d6SCurt Wohlgemuth /* 15458a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 15468a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 15478a57d9d6SCurt Wohlgemuth * calling this routine! 15488a57d9d6SCurt Wohlgemuth */ 15494ddfef7bSEric Sandeen static noinline_for_stack int 1550adb7ef60SKonstantin Khlebnikov ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, 1551adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b, gfp_t gfp) 1552c9de560dSAlex Tomas { 1553c9de560dSAlex Tomas int blocks_per_page; 1554c9de560dSAlex Tomas int block; 1555c9de560dSAlex Tomas int pnum; 1556c9de560dSAlex Tomas int poff; 1557c9de560dSAlex Tomas struct page *page; 1558fdf6c7a7SShen Feng int ret; 1559920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1560920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1561920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1562c9de560dSAlex Tomas 1563b10a44c3STheodore Ts'o might_sleep(); 1564d3df1453SRitesh Harjani mb_debug(sb, "load group %u\n", group); 1565c9de560dSAlex Tomas 156609cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1567920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 15685354b2afSTheodore Ts'o if (!grp) 15695354b2afSTheodore Ts'o return -EFSCORRUPTED; 1570c9de560dSAlex Tomas 1571c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1572529da704STao Ma e4b->bd_info = grp; 1573c9de560dSAlex Tomas e4b->bd_sb = sb; 1574c9de560dSAlex Tomas e4b->bd_group = group; 1575c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1576c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1577c9de560dSAlex Tomas 1578f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1579f41c0750SAneesh Kumar K.V /* 1580f41c0750SAneesh Kumar K.V * we need full data about the group 1581f41c0750SAneesh Kumar K.V * to make a good selection 1582f41c0750SAneesh Kumar K.V */ 1583adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, gfp); 1584f41c0750SAneesh Kumar K.V if (ret) 1585f41c0750SAneesh Kumar K.V return ret; 1586f41c0750SAneesh Kumar K.V } 1587f41c0750SAneesh Kumar K.V 1588c9de560dSAlex Tomas /* 1589c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1590c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1591c9de560dSAlex Tomas * So for each group we need two blocks. 1592c9de560dSAlex Tomas */ 1593c9de560dSAlex Tomas block = group * 2; 1594c9de560dSAlex Tomas pnum = block / blocks_per_page; 1595c9de560dSAlex Tomas poff = block % blocks_per_page; 1596c9de560dSAlex Tomas 1597c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1598c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 15992457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1600c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1601c9de560dSAlex Tomas if (page) 1602920313a7SAneesh Kumar K.V /* 1603920313a7SAneesh Kumar K.V * drop the page reference and try 1604920313a7SAneesh Kumar K.V * to get the page with lock. If we 1605920313a7SAneesh Kumar K.V * are not uptodate that implies 1606920313a7SAneesh Kumar K.V * somebody just created the page but 1607920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1608920313a7SAneesh Kumar K.V * wait for it to initialize. 1609920313a7SAneesh Kumar K.V */ 161009cbfeafSKirill A. Shutemov put_page(page); 1611adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1612c9de560dSAlex Tomas if (page) { 161319b8b035STheodore Ts'o if (WARN_RATELIMIT(page->mapping != inode->i_mapping, 161419b8b035STheodore Ts'o "ext4: bitmap's paging->mapping != inode->i_mapping\n")) { 161519b8b035STheodore Ts'o /* should never happen */ 161619b8b035STheodore Ts'o unlock_page(page); 161719b8b035STheodore Ts'o ret = -EINVAL; 161819b8b035STheodore Ts'o goto err; 161919b8b035STheodore Ts'o } 1620c9de560dSAlex Tomas if (!PageUptodate(page)) { 1621adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 1622fdf6c7a7SShen Feng if (ret) { 1623fdf6c7a7SShen Feng unlock_page(page); 1624fdf6c7a7SShen Feng goto err; 1625fdf6c7a7SShen Feng } 1626c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1627c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1628c9de560dSAlex Tomas } 1629c9de560dSAlex Tomas unlock_page(page); 1630c9de560dSAlex Tomas } 1631c9de560dSAlex Tomas } 1632c57ab39bSYounger Liu if (page == NULL) { 1633c57ab39bSYounger Liu ret = -ENOMEM; 1634c57ab39bSYounger Liu goto err; 1635c57ab39bSYounger Liu } 1636c57ab39bSYounger Liu if (!PageUptodate(page)) { 1637fdf6c7a7SShen Feng ret = -EIO; 1638c9de560dSAlex Tomas goto err; 1639fdf6c7a7SShen Feng } 16402457aec6SMel Gorman 16412457aec6SMel Gorman /* Pages marked accessed already */ 1642c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1643c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1644c9de560dSAlex Tomas 1645c9de560dSAlex Tomas block++; 1646c9de560dSAlex Tomas pnum = block / blocks_per_page; 1647c9de560dSAlex Tomas poff = block % blocks_per_page; 1648c9de560dSAlex Tomas 16492457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1650c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1651c9de560dSAlex Tomas if (page) 165209cbfeafSKirill A. Shutemov put_page(page); 1653adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1654c9de560dSAlex Tomas if (page) { 165519b8b035STheodore Ts'o if (WARN_RATELIMIT(page->mapping != inode->i_mapping, 165619b8b035STheodore Ts'o "ext4: buddy bitmap's page->mapping != inode->i_mapping\n")) { 165719b8b035STheodore Ts'o /* should never happen */ 165819b8b035STheodore Ts'o unlock_page(page); 165919b8b035STheodore Ts'o ret = -EINVAL; 166019b8b035STheodore Ts'o goto err; 166119b8b035STheodore Ts'o } 1662fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1663adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b->bd_bitmap, 1664adb7ef60SKonstantin Khlebnikov gfp); 1665fdf6c7a7SShen Feng if (ret) { 1666fdf6c7a7SShen Feng unlock_page(page); 1667fdf6c7a7SShen Feng goto err; 1668fdf6c7a7SShen Feng } 1669fdf6c7a7SShen Feng } 1670c9de560dSAlex Tomas unlock_page(page); 1671c9de560dSAlex Tomas } 1672c9de560dSAlex Tomas } 1673c57ab39bSYounger Liu if (page == NULL) { 1674c57ab39bSYounger Liu ret = -ENOMEM; 1675c57ab39bSYounger Liu goto err; 1676c57ab39bSYounger Liu } 1677c57ab39bSYounger Liu if (!PageUptodate(page)) { 1678fdf6c7a7SShen Feng ret = -EIO; 1679c9de560dSAlex Tomas goto err; 1680fdf6c7a7SShen Feng } 16812457aec6SMel Gorman 16822457aec6SMel Gorman /* Pages marked accessed already */ 1683c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1684c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1685c9de560dSAlex Tomas 1686c9de560dSAlex Tomas return 0; 1687c9de560dSAlex Tomas 1688c9de560dSAlex Tomas err: 168926626f11SYang Ruirui if (page) 169009cbfeafSKirill A. Shutemov put_page(page); 1691c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 169209cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1693285164b8SKemeng Shi 1694c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1695c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1696fdf6c7a7SShen Feng return ret; 1697c9de560dSAlex Tomas } 1698c9de560dSAlex Tomas 1699adb7ef60SKonstantin Khlebnikov static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1700adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b) 1701adb7ef60SKonstantin Khlebnikov { 1702adb7ef60SKonstantin Khlebnikov return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS); 1703adb7ef60SKonstantin Khlebnikov } 1704adb7ef60SKonstantin Khlebnikov 1705e39e07fdSJing Zhang static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) 1706c9de560dSAlex Tomas { 1707c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 170809cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1709c9de560dSAlex Tomas if (e4b->bd_buddy_page) 171009cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1711c9de560dSAlex Tomas } 1712c9de560dSAlex Tomas 1713c9de560dSAlex Tomas 1714c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1715c9de560dSAlex Tomas { 1716ce3cca33SChunguang Xu int order = 1, max; 1717c9de560dSAlex Tomas void *bb; 1718c9de560dSAlex Tomas 1719c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 1720c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1721c9de560dSAlex Tomas 1722c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1723ce3cca33SChunguang Xu bb = mb_find_buddy(e4b, order, &max); 1724ce3cca33SChunguang Xu if (!mb_test_bit(block >> order, bb)) { 1725c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1726c9de560dSAlex Tomas return order; 1727c9de560dSAlex Tomas } 1728c9de560dSAlex Tomas order++; 1729c9de560dSAlex Tomas } 1730c9de560dSAlex Tomas return 0; 1731c9de560dSAlex Tomas } 1732c9de560dSAlex Tomas 1733955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1734c9de560dSAlex Tomas { 1735c9de560dSAlex Tomas __u32 *addr; 1736c9de560dSAlex Tomas 1737c9de560dSAlex Tomas len = cur + len; 1738c9de560dSAlex Tomas while (cur < len) { 1739c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1740c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1741c9de560dSAlex Tomas addr = bm + (cur >> 3); 1742c9de560dSAlex Tomas *addr = 0; 1743c9de560dSAlex Tomas cur += 32; 1744c9de560dSAlex Tomas continue; 1745c9de560dSAlex Tomas } 1746e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1747c9de560dSAlex Tomas cur++; 1748c9de560dSAlex Tomas } 1749c9de560dSAlex Tomas } 1750c9de560dSAlex Tomas 1751eabe0444SAndrey Sidorov /* clear bits in given range 1752eabe0444SAndrey Sidorov * will return first found zero bit if any, -1 otherwise 1753eabe0444SAndrey Sidorov */ 1754eabe0444SAndrey Sidorov static int mb_test_and_clear_bits(void *bm, int cur, int len) 1755eabe0444SAndrey Sidorov { 1756eabe0444SAndrey Sidorov __u32 *addr; 1757eabe0444SAndrey Sidorov int zero_bit = -1; 1758eabe0444SAndrey Sidorov 1759eabe0444SAndrey Sidorov len = cur + len; 1760eabe0444SAndrey Sidorov while (cur < len) { 1761eabe0444SAndrey Sidorov if ((cur & 31) == 0 && (len - cur) >= 32) { 1762eabe0444SAndrey Sidorov /* fast path: clear whole word at once */ 1763eabe0444SAndrey Sidorov addr = bm + (cur >> 3); 1764eabe0444SAndrey Sidorov if (*addr != (__u32)(-1) && zero_bit == -1) 1765eabe0444SAndrey Sidorov zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0); 1766eabe0444SAndrey Sidorov *addr = 0; 1767eabe0444SAndrey Sidorov cur += 32; 1768eabe0444SAndrey Sidorov continue; 1769eabe0444SAndrey Sidorov } 1770eabe0444SAndrey Sidorov if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) 1771eabe0444SAndrey Sidorov zero_bit = cur; 1772eabe0444SAndrey Sidorov cur++; 1773eabe0444SAndrey Sidorov } 1774eabe0444SAndrey Sidorov 1775eabe0444SAndrey Sidorov return zero_bit; 1776eabe0444SAndrey Sidorov } 1777eabe0444SAndrey Sidorov 1778123e3016SRitesh Harjani void mb_set_bits(void *bm, int cur, int len) 1779c9de560dSAlex Tomas { 1780c9de560dSAlex Tomas __u32 *addr; 1781c9de560dSAlex Tomas 1782c9de560dSAlex Tomas len = cur + len; 1783c9de560dSAlex Tomas while (cur < len) { 1784c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1785c9de560dSAlex Tomas /* fast path: set whole word at once */ 1786c9de560dSAlex Tomas addr = bm + (cur >> 3); 1787c9de560dSAlex Tomas *addr = 0xffffffff; 1788c9de560dSAlex Tomas cur += 32; 1789c9de560dSAlex Tomas continue; 1790c9de560dSAlex Tomas } 1791e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1792c9de560dSAlex Tomas cur++; 1793c9de560dSAlex Tomas } 1794c9de560dSAlex Tomas } 1795c9de560dSAlex Tomas 1796eabe0444SAndrey Sidorov static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side) 1797eabe0444SAndrey Sidorov { 1798eabe0444SAndrey Sidorov if (mb_test_bit(*bit + side, bitmap)) { 1799eabe0444SAndrey Sidorov mb_clear_bit(*bit, bitmap); 1800eabe0444SAndrey Sidorov (*bit) -= side; 1801eabe0444SAndrey Sidorov return 1; 1802eabe0444SAndrey Sidorov } 1803eabe0444SAndrey Sidorov else { 1804eabe0444SAndrey Sidorov (*bit) += side; 1805eabe0444SAndrey Sidorov mb_set_bit(*bit, bitmap); 1806eabe0444SAndrey Sidorov return -1; 1807eabe0444SAndrey Sidorov } 1808eabe0444SAndrey Sidorov } 1809eabe0444SAndrey Sidorov 1810eabe0444SAndrey Sidorov static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last) 1811eabe0444SAndrey Sidorov { 1812eabe0444SAndrey Sidorov int max; 1813eabe0444SAndrey Sidorov int order = 1; 1814eabe0444SAndrey Sidorov void *buddy = mb_find_buddy(e4b, order, &max); 1815eabe0444SAndrey Sidorov 1816eabe0444SAndrey Sidorov while (buddy) { 1817eabe0444SAndrey Sidorov void *buddy2; 1818eabe0444SAndrey Sidorov 1819eabe0444SAndrey Sidorov /* Bits in range [first; last] are known to be set since 1820eabe0444SAndrey Sidorov * corresponding blocks were allocated. Bits in range 1821eabe0444SAndrey Sidorov * (first; last) will stay set because they form buddies on 1822eabe0444SAndrey Sidorov * upper layer. We just deal with borders if they don't 1823eabe0444SAndrey Sidorov * align with upper layer and then go up. 1824eabe0444SAndrey Sidorov * Releasing entire group is all about clearing 1825eabe0444SAndrey Sidorov * single bit of highest order buddy. 1826eabe0444SAndrey Sidorov */ 1827eabe0444SAndrey Sidorov 1828eabe0444SAndrey Sidorov /* Example: 1829eabe0444SAndrey Sidorov * --------------------------------- 1830eabe0444SAndrey Sidorov * | 1 | 1 | 1 | 1 | 1831eabe0444SAndrey Sidorov * --------------------------------- 1832eabe0444SAndrey Sidorov * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1833eabe0444SAndrey Sidorov * --------------------------------- 1834eabe0444SAndrey Sidorov * 0 1 2 3 4 5 6 7 1835eabe0444SAndrey Sidorov * \_____________________/ 1836eabe0444SAndrey Sidorov * 1837eabe0444SAndrey Sidorov * Neither [1] nor [6] is aligned to above layer. 1838eabe0444SAndrey Sidorov * Left neighbour [0] is free, so mark it busy, 1839eabe0444SAndrey Sidorov * decrease bb_counters and extend range to 1840eabe0444SAndrey Sidorov * [0; 6] 1841eabe0444SAndrey Sidorov * Right neighbour [7] is busy. It can't be coaleasced with [6], so 1842eabe0444SAndrey Sidorov * mark [6] free, increase bb_counters and shrink range to 1843eabe0444SAndrey Sidorov * [0; 5]. 1844eabe0444SAndrey Sidorov * Then shift range to [0; 2], go up and do the same. 1845eabe0444SAndrey Sidorov */ 1846eabe0444SAndrey Sidorov 1847eabe0444SAndrey Sidorov 1848eabe0444SAndrey Sidorov if (first & 1) 1849eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); 1850eabe0444SAndrey Sidorov if (!(last & 1)) 1851eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); 1852eabe0444SAndrey Sidorov if (first > last) 1853eabe0444SAndrey Sidorov break; 1854eabe0444SAndrey Sidorov order++; 1855eabe0444SAndrey Sidorov 1856976620bdSKemeng Shi buddy2 = mb_find_buddy(e4b, order, &max); 1857976620bdSKemeng Shi if (!buddy2) { 1858eabe0444SAndrey Sidorov mb_clear_bits(buddy, first, last - first + 1); 1859eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order - 1] += last - first + 1; 1860eabe0444SAndrey Sidorov break; 1861eabe0444SAndrey Sidorov } 1862eabe0444SAndrey Sidorov first >>= 1; 1863eabe0444SAndrey Sidorov last >>= 1; 1864eabe0444SAndrey Sidorov buddy = buddy2; 1865eabe0444SAndrey Sidorov } 1866eabe0444SAndrey Sidorov } 1867eabe0444SAndrey Sidorov 18687e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1869c9de560dSAlex Tomas int first, int count) 1870c9de560dSAlex Tomas { 1871eabe0444SAndrey Sidorov int left_is_free = 0; 1872eabe0444SAndrey Sidorov int right_is_free = 0; 1873eabe0444SAndrey Sidorov int block; 1874eabe0444SAndrey Sidorov int last = first + count - 1; 1875c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1876c9de560dSAlex Tomas 1877c99d1e6eSTheodore Ts'o if (WARN_ON(count == 0)) 1878c99d1e6eSTheodore Ts'o return; 1879eabe0444SAndrey Sidorov BUG_ON(last >= (sb->s_blocksize << 3)); 1880bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1881163a203dSDarrick J. Wong /* Don't bother if the block group is corrupt. */ 1882163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) 1883163a203dSDarrick J. Wong return; 1884163a203dSDarrick J. Wong 1885c9de560dSAlex Tomas mb_check_buddy(e4b); 1886c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1887c9de560dSAlex Tomas 188807b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 1889c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1890c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1891c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1892c9de560dSAlex Tomas 1893eabe0444SAndrey Sidorov /* access memory sequentially: check left neighbour, 1894eabe0444SAndrey Sidorov * clear range and then check right neighbour 1895eabe0444SAndrey Sidorov */ 1896c9de560dSAlex Tomas if (first != 0) 1897eabe0444SAndrey Sidorov left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); 1898eabe0444SAndrey Sidorov block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); 1899eabe0444SAndrey Sidorov if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) 1900eabe0444SAndrey Sidorov right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); 1901c9de560dSAlex Tomas 1902eabe0444SAndrey Sidorov if (unlikely(block != -1)) { 1903e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 1904c9de560dSAlex Tomas ext4_fsblk_t blocknr; 19055661bd68SAkinobu Mita 19065661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 190749598e04SJun Piao blocknr += EXT4_C2B(sbi, block); 19088016e29fSHarshad Shirwadkar if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) { 19095d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 1910e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 1911e29136f8STheodore Ts'o blocknr, 19128016e29fSHarshad Shirwadkar "freeing already freed block (bit %u); block bitmap corrupt.", 1913163a203dSDarrick J. Wong block); 19148016e29fSHarshad Shirwadkar ext4_mark_group_bitmap_corrupted( 19158016e29fSHarshad Shirwadkar sb, e4b->bd_group, 1916db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 19178016e29fSHarshad Shirwadkar } 1918eabe0444SAndrey Sidorov goto done; 1919c9de560dSAlex Tomas } 1920c9de560dSAlex Tomas 1921eabe0444SAndrey Sidorov /* let's maintain fragments counter */ 1922eabe0444SAndrey Sidorov if (left_is_free && right_is_free) 1923eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments--; 1924eabe0444SAndrey Sidorov else if (!left_is_free && !right_is_free) 1925eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments++; 1926c9de560dSAlex Tomas 1927eabe0444SAndrey Sidorov /* buddy[0] == bd_bitmap is a special case, so handle 1928eabe0444SAndrey Sidorov * it right away and let mb_buddy_mark_free stay free of 1929eabe0444SAndrey Sidorov * zero order checks. 1930eabe0444SAndrey Sidorov * Check if neighbours are to be coaleasced, 1931eabe0444SAndrey Sidorov * adjust bitmap bb_counters and borders appropriately. 1932eabe0444SAndrey Sidorov */ 1933eabe0444SAndrey Sidorov if (first & 1) { 1934eabe0444SAndrey Sidorov first += !left_is_free; 1935eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; 1936c9de560dSAlex Tomas } 1937eabe0444SAndrey Sidorov if (!(last & 1)) { 1938eabe0444SAndrey Sidorov last -= !right_is_free; 1939eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; 1940c9de560dSAlex Tomas } 1941eabe0444SAndrey Sidorov 1942eabe0444SAndrey Sidorov if (first <= last) 1943eabe0444SAndrey Sidorov mb_buddy_mark_free(e4b, first >> 1, last >> 1); 1944eabe0444SAndrey Sidorov 1945eabe0444SAndrey Sidorov done: 19468a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, e4b->bd_info); 1947196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(sb, e4b->bd_info); 1948c9de560dSAlex Tomas mb_check_buddy(e4b); 1949c9de560dSAlex Tomas } 1950c9de560dSAlex Tomas 195115c006a2SRobin Dong static int mb_find_extent(struct ext4_buddy *e4b, int block, 1952c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1953c9de560dSAlex Tomas { 1954c9de560dSAlex Tomas int next = block; 195515c006a2SRobin Dong int max, order; 1956c9de560dSAlex Tomas void *buddy; 1957c9de560dSAlex Tomas 1958bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1959c9de560dSAlex Tomas BUG_ON(ex == NULL); 1960c9de560dSAlex Tomas 196115c006a2SRobin Dong buddy = mb_find_buddy(e4b, 0, &max); 1962c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1963c9de560dSAlex Tomas BUG_ON(block >= max); 1964c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1965c9de560dSAlex Tomas ex->fe_len = 0; 1966c9de560dSAlex Tomas ex->fe_start = 0; 1967c9de560dSAlex Tomas ex->fe_group = 0; 1968c9de560dSAlex Tomas return 0; 1969c9de560dSAlex Tomas } 1970c9de560dSAlex Tomas 1971c9de560dSAlex Tomas /* find actual order */ 1972c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1973c9de560dSAlex Tomas block = block >> order; 1974c9de560dSAlex Tomas 1975c9de560dSAlex Tomas ex->fe_len = 1 << order; 1976c9de560dSAlex Tomas ex->fe_start = block << order; 1977c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1978c9de560dSAlex Tomas 1979c9de560dSAlex Tomas /* calc difference from given start */ 1980c9de560dSAlex Tomas next = next - ex->fe_start; 1981c9de560dSAlex Tomas ex->fe_len -= next; 1982c9de560dSAlex Tomas ex->fe_start += next; 1983c9de560dSAlex Tomas 1984c9de560dSAlex Tomas while (needed > ex->fe_len && 1985d8ec0c39SAlan Cox mb_find_buddy(e4b, order, &max)) { 1986c9de560dSAlex Tomas 1987c9de560dSAlex Tomas if (block + 1 >= max) 1988c9de560dSAlex Tomas break; 1989c9de560dSAlex Tomas 1990c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1991c5e8f3f3STheodore Ts'o if (mb_test_bit(next, e4b->bd_bitmap)) 1992c9de560dSAlex Tomas break; 1993c9de560dSAlex Tomas 1994b051d8dcSRobin Dong order = mb_find_order_for_block(e4b, next); 1995c9de560dSAlex Tomas 1996c9de560dSAlex Tomas block = next >> order; 1997c9de560dSAlex Tomas ex->fe_len += 1 << order; 1998c9de560dSAlex Tomas } 1999c9de560dSAlex Tomas 200031562b95SJan Kara if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) { 200143c73221STheodore Ts'o /* Should never happen! (but apparently sometimes does?!?) */ 200243c73221STheodore Ts'o WARN_ON(1); 2003cd84bbbaSStephen Brennan ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0, 2004cd84bbbaSStephen Brennan "corruption or bug in mb_find_extent " 200543c73221STheodore Ts'o "block=%d, order=%d needed=%d ex=%u/%d/%d@%u", 200643c73221STheodore Ts'o block, order, needed, ex->fe_group, ex->fe_start, 200743c73221STheodore Ts'o ex->fe_len, ex->fe_logical); 200843c73221STheodore Ts'o ex->fe_len = 0; 200943c73221STheodore Ts'o ex->fe_start = 0; 201043c73221STheodore Ts'o ex->fe_group = 0; 201143c73221STheodore Ts'o } 2012c9de560dSAlex Tomas return ex->fe_len; 2013c9de560dSAlex Tomas } 2014c9de560dSAlex Tomas 2015c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 2016c9de560dSAlex Tomas { 2017c9de560dSAlex Tomas int ord; 2018c9de560dSAlex Tomas int mlen = 0; 2019c9de560dSAlex Tomas int max = 0; 2020c9de560dSAlex Tomas int cur; 2021c9de560dSAlex Tomas int start = ex->fe_start; 2022c9de560dSAlex Tomas int len = ex->fe_len; 2023c9de560dSAlex Tomas unsigned ret = 0; 2024c9de560dSAlex Tomas int len0 = len; 2025c9de560dSAlex Tomas void *buddy; 2026218a6944Shanjinke bool split = false; 2027c9de560dSAlex Tomas 2028c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 2029c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 2030bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 2031c9de560dSAlex Tomas mb_check_buddy(e4b); 2032c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 2033c9de560dSAlex Tomas 203407b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 2035c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 2036c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 2037c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 2038c9de560dSAlex Tomas 2039c9de560dSAlex Tomas /* let's maintain fragments counter */ 2040c9de560dSAlex Tomas if (start != 0) 2041c5e8f3f3STheodore Ts'o mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); 2042c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 2043c5e8f3f3STheodore Ts'o max = !mb_test_bit(start + len, e4b->bd_bitmap); 2044c9de560dSAlex Tomas if (mlen && max) 2045c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 2046c9de560dSAlex Tomas else if (!mlen && !max) 2047c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 2048c9de560dSAlex Tomas 2049c9de560dSAlex Tomas /* let's maintain buddy itself */ 2050c9de560dSAlex Tomas while (len) { 2051218a6944Shanjinke if (!split) 2052c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 2053c9de560dSAlex Tomas 2054c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 2055c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 2056c9de560dSAlex Tomas mlen = 1 << ord; 2057218a6944Shanjinke if (!split) 2058c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 2059218a6944Shanjinke else 2060218a6944Shanjinke split = false; 2061c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 2062c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 2063c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 2064c9de560dSAlex Tomas start += mlen; 2065c9de560dSAlex Tomas len -= mlen; 2066c9de560dSAlex Tomas BUG_ON(len < 0); 2067c9de560dSAlex Tomas continue; 2068c9de560dSAlex Tomas } 2069c9de560dSAlex Tomas 2070c9de560dSAlex Tomas /* store for history */ 2071c9de560dSAlex Tomas if (ret == 0) 2072c9de560dSAlex Tomas ret = len | (ord << 16); 2073c9de560dSAlex Tomas 2074c9de560dSAlex Tomas /* we have to split large buddy */ 2075c9de560dSAlex Tomas BUG_ON(ord <= 0); 2076c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 2077c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 2078c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 2079c9de560dSAlex Tomas 2080c9de560dSAlex Tomas ord--; 2081c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 2082c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 2083c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 2084c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 2085c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 2086c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 2087218a6944Shanjinke split = true; 2088c9de560dSAlex Tomas } 20898a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); 2090c9de560dSAlex Tomas 2091196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(e4b->bd_sb, e4b->bd_info); 2092123e3016SRitesh Harjani mb_set_bits(e4b->bd_bitmap, ex->fe_start, len0); 2093c9de560dSAlex Tomas mb_check_buddy(e4b); 2094c9de560dSAlex Tomas 2095c9de560dSAlex Tomas return ret; 2096c9de560dSAlex Tomas } 2097c9de560dSAlex Tomas 2098c9de560dSAlex Tomas /* 2099c9de560dSAlex Tomas * Must be called under group lock! 2100c9de560dSAlex Tomas */ 2101c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 2102c9de560dSAlex Tomas struct ext4_buddy *e4b) 2103c9de560dSAlex Tomas { 2104c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2105c9de560dSAlex Tomas int ret; 2106c9de560dSAlex Tomas 2107c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 2108c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2109c9de560dSAlex Tomas 2110c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 2111c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 2112c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 2113c9de560dSAlex Tomas 2114c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 2115c9de560dSAlex Tomas * allocated blocks for history */ 2116c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 2117c9de560dSAlex Tomas 2118c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 2119c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 2120c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 2121c9de560dSAlex Tomas 2122c3a326a6SAneesh Kumar K.V /* 2123c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 2124c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 2125c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 2126c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 2127c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 2128c3a326a6SAneesh Kumar K.V */ 2129c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 2130c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 2131c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 2132c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 2133c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 21344ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2135c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2136c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 2137c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 2138c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2139c9de560dSAlex Tomas } 214053f86b17SRitesh Harjani /* 214153f86b17SRitesh Harjani * As we've just preallocated more space than 214253f86b17SRitesh Harjani * user requested originally, we store allocated 214353f86b17SRitesh Harjani * space in a special descriptor. 214453f86b17SRitesh Harjani */ 214553f86b17SRitesh Harjani if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 214653f86b17SRitesh Harjani ext4_mb_new_preallocation(ac); 214753f86b17SRitesh Harjani 2148c9de560dSAlex Tomas } 2149c9de560dSAlex Tomas 2150c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 2151c9de560dSAlex Tomas struct ext4_buddy *e4b, 2152c9de560dSAlex Tomas int finish_group) 2153c9de560dSAlex Tomas { 2154c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2155c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 2156c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 2157c9de560dSAlex Tomas 2158032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 2159032115fcSAneesh Kumar K.V return; 2160c9de560dSAlex Tomas /* 2161c9de560dSAlex Tomas * We don't want to scan for a whole year 2162c9de560dSAlex Tomas */ 2163c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 2164c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2165c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 2166c9de560dSAlex Tomas return; 2167c9de560dSAlex Tomas } 2168c9de560dSAlex Tomas 2169c9de560dSAlex Tomas /* 2170c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 2171c9de560dSAlex Tomas */ 2172c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 2173c9de560dSAlex Tomas return; 2174c9de560dSAlex Tomas 21753582e745SOjaswin Mujoo if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 2176c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2177c9de560dSAlex Tomas } 2178c9de560dSAlex Tomas 2179c9de560dSAlex Tomas /* 2180c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 2181c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 2182c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 2183c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 2184c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 2185c9de560dSAlex Tomas * mballoc can't find good enough extent. 2186c9de560dSAlex Tomas * 21873582e745SOjaswin Mujoo * The algorithm used is roughly as follows: 21883582e745SOjaswin Mujoo * 21893582e745SOjaswin Mujoo * * If free extent found is exactly as big as goal, then 21903582e745SOjaswin Mujoo * stop the scan and use it immediately 21913582e745SOjaswin Mujoo * 21923582e745SOjaswin Mujoo * * If free extent found is smaller than goal, then keep retrying 21933582e745SOjaswin Mujoo * upto a max of sbi->s_mb_max_to_scan times (default 200). After 21943582e745SOjaswin Mujoo * that stop scanning and use whatever we have. 21953582e745SOjaswin Mujoo * 21963582e745SOjaswin Mujoo * * If free extent found is bigger than goal, then keep retrying 21973582e745SOjaswin Mujoo * upto a max of sbi->s_mb_min_to_scan times (default 10) before 21983582e745SOjaswin Mujoo * stopping the scan and using the extent. 21993582e745SOjaswin Mujoo * 22003582e745SOjaswin Mujoo * 2201c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 2202c9de560dSAlex Tomas */ 2203c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 2204c9de560dSAlex Tomas struct ext4_free_extent *ex, 2205c9de560dSAlex Tomas struct ext4_buddy *e4b) 2206c9de560dSAlex Tomas { 2207c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 2208c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 2209c9de560dSAlex Tomas 2210c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 22117137d7a4STheodore Ts'o BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 22127137d7a4STheodore Ts'o BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 2213c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 2214c9de560dSAlex Tomas 2215c9de560dSAlex Tomas ac->ac_found++; 2216fdd9a009SOjaswin Mujoo ac->ac_cX_found[ac->ac_criteria]++; 2217c9de560dSAlex Tomas 2218c9de560dSAlex Tomas /* 2219c9de560dSAlex Tomas * The special case - take what you catch first 2220c9de560dSAlex Tomas */ 2221c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2222c9de560dSAlex Tomas *bex = *ex; 2223c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2224c9de560dSAlex Tomas return; 2225c9de560dSAlex Tomas } 2226c9de560dSAlex Tomas 2227c9de560dSAlex Tomas /* 2228c9de560dSAlex Tomas * Let's check whether the chuck is good enough 2229c9de560dSAlex Tomas */ 2230c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 2231c9de560dSAlex Tomas *bex = *ex; 2232c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2233c9de560dSAlex Tomas return; 2234c9de560dSAlex Tomas } 2235c9de560dSAlex Tomas 2236c9de560dSAlex Tomas /* 2237c9de560dSAlex Tomas * If this is first found extent, just store it in the context 2238c9de560dSAlex Tomas */ 2239c9de560dSAlex Tomas if (bex->fe_len == 0) { 2240c9de560dSAlex Tomas *bex = *ex; 2241c9de560dSAlex Tomas return; 2242c9de560dSAlex Tomas } 2243c9de560dSAlex Tomas 2244c9de560dSAlex Tomas /* 2245c9de560dSAlex Tomas * If new found extent is better, store it in the context 2246c9de560dSAlex Tomas */ 2247c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 2248c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 2249c9de560dSAlex Tomas * larger than previous best one is better */ 2250c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 2251c9de560dSAlex Tomas *bex = *ex; 2252c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 2253c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 2254c9de560dSAlex Tomas * an extent that still satisfy the request, but is 2255c9de560dSAlex Tomas * smaller than previous one */ 2256c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 2257c9de560dSAlex Tomas *bex = *ex; 2258c9de560dSAlex Tomas } 2259c9de560dSAlex Tomas 2260c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 2261c9de560dSAlex Tomas } 2262c9de560dSAlex Tomas 2263089ceeccSEric Sandeen static noinline_for_stack 226485b67ffbSKemeng Shi void ext4_mb_try_best_found(struct ext4_allocation_context *ac, 2265c9de560dSAlex Tomas struct ext4_buddy *e4b) 2266c9de560dSAlex Tomas { 2267c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 2268c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 2269c9de560dSAlex Tomas int max; 2270c9de560dSAlex Tomas int err; 2271c9de560dSAlex Tomas 2272c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 2273c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 2274c9de560dSAlex Tomas if (err) 227585b67ffbSKemeng Shi return; 2276c9de560dSAlex Tomas 2277c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 227815c006a2SRobin Dong max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex); 2279c9de560dSAlex Tomas 2280c9de560dSAlex Tomas if (max > 0) { 2281c9de560dSAlex Tomas ac->ac_b_ex = ex; 2282c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2283c9de560dSAlex Tomas } 2284c9de560dSAlex Tomas 2285c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 2286e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 2287c9de560dSAlex Tomas } 2288c9de560dSAlex Tomas 2289089ceeccSEric Sandeen static noinline_for_stack 2290089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 2291c9de560dSAlex Tomas struct ext4_buddy *e4b) 2292c9de560dSAlex Tomas { 2293c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 2294c9de560dSAlex Tomas int max; 2295c9de560dSAlex Tomas int err; 2296c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2297838cd0cfSYongqiang Yang struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 2298c9de560dSAlex Tomas struct ext4_free_extent ex; 2299c9de560dSAlex Tomas 23005354b2afSTheodore Ts'o if (!grp) 23015354b2afSTheodore Ts'o return -EFSCORRUPTED; 230201e4ca29SKemeng Shi if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY))) 2303c9de560dSAlex Tomas return 0; 2304838cd0cfSYongqiang Yang if (grp->bb_free == 0) 2305838cd0cfSYongqiang Yang return 0; 2306c9de560dSAlex Tomas 2307c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 2308c9de560dSAlex Tomas if (err) 2309c9de560dSAlex Tomas return err; 2310c9de560dSAlex Tomas 2311163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) { 2312163a203dSDarrick J. Wong ext4_mb_unload_buddy(e4b); 2313163a203dSDarrick J. Wong return 0; 2314163a203dSDarrick J. Wong } 2315163a203dSDarrick J. Wong 2316c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 231715c006a2SRobin Dong max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, 2318c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 2319ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADFA11; /* debug value */ 2320c9de560dSAlex Tomas 2321c3defd99SKemeng Shi if (max >= ac->ac_g_ex.fe_len && 2322c3defd99SKemeng Shi ac->ac_g_ex.fe_len == EXT4_B2C(sbi, sbi->s_stripe)) { 2323c9de560dSAlex Tomas ext4_fsblk_t start; 2324c9de560dSAlex Tomas 232599c515e3SKemeng Shi start = ext4_grp_offs_to_block(ac->ac_sb, &ex); 2326c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 2327c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 2328c9de560dSAlex Tomas ac->ac_found++; 2329c9de560dSAlex Tomas ac->ac_b_ex = ex; 2330c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2331c9de560dSAlex Tomas } 2332c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 2333c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 2334c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 2335c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 2336c9de560dSAlex Tomas ac->ac_found++; 2337c9de560dSAlex Tomas ac->ac_b_ex = ex; 2338c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2339c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 2340c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 2341c9de560dSAlex Tomas * number of blocks to an existing extent */ 2342c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 2343c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 2344c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 2345c9de560dSAlex Tomas ac->ac_found++; 2346c9de560dSAlex Tomas ac->ac_b_ex = ex; 2347c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2348c9de560dSAlex Tomas } 2349c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 2350e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 2351c9de560dSAlex Tomas 2352c9de560dSAlex Tomas return 0; 2353c9de560dSAlex Tomas } 2354c9de560dSAlex Tomas 2355c9de560dSAlex Tomas /* 2356c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 2357c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 2358c9de560dSAlex Tomas */ 2359089ceeccSEric Sandeen static noinline_for_stack 2360089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 2361c9de560dSAlex Tomas struct ext4_buddy *e4b) 2362c9de560dSAlex Tomas { 2363c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2364c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 2365c9de560dSAlex Tomas void *buddy; 2366c9de560dSAlex Tomas int i; 2367c9de560dSAlex Tomas int k; 2368c9de560dSAlex Tomas int max; 2369c9de560dSAlex Tomas 2370c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 23714b68f6dfSHarshad Shirwadkar for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) { 2372c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 2373c9de560dSAlex Tomas continue; 2374c9de560dSAlex Tomas 2375c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 237619b8b035STheodore Ts'o if (WARN_RATELIMIT(buddy == NULL, 237719b8b035STheodore Ts'o "ext4: mb_simple_scan_group: mb_find_buddy failed, (%d)\n", i)) 237819b8b035STheodore Ts'o continue; 2379c9de560dSAlex Tomas 2380ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 2381eb576086SDmitry Monakhov if (k >= max) { 2382eb576086SDmitry Monakhov ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, 2383eb576086SDmitry Monakhov "%d free clusters of order %d. But found 0", 2384eb576086SDmitry Monakhov grp->bb_counters[i], i); 2385eb576086SDmitry Monakhov ext4_mark_group_bitmap_corrupted(ac->ac_sb, 2386eb576086SDmitry Monakhov e4b->bd_group, 2387eb576086SDmitry Monakhov EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2388eb576086SDmitry Monakhov break; 2389eb576086SDmitry Monakhov } 2390c9de560dSAlex Tomas ac->ac_found++; 2391fdd9a009SOjaswin Mujoo ac->ac_cX_found[ac->ac_criteria]++; 2392c9de560dSAlex Tomas 2393c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 2394c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 2395c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 2396c9de560dSAlex Tomas 2397c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2398c9de560dSAlex Tomas 239953f86b17SRitesh Harjani BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); 2400c9de560dSAlex Tomas 2401c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 2402c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 2403c9de560dSAlex Tomas 2404c9de560dSAlex Tomas break; 2405c9de560dSAlex Tomas } 2406c9de560dSAlex Tomas } 2407c9de560dSAlex Tomas 2408c9de560dSAlex Tomas /* 2409c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 2410c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 2411c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 2412c9de560dSAlex Tomas */ 2413089ceeccSEric Sandeen static noinline_for_stack 2414089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 2415c9de560dSAlex Tomas struct ext4_buddy *e4b) 2416c9de560dSAlex Tomas { 2417c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2418c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2419c9de560dSAlex Tomas struct ext4_free_extent ex; 24201b420011SOjaswin Mujoo int i, j, freelen; 2421c9de560dSAlex Tomas int free; 2422c9de560dSAlex Tomas 2423c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 2424907ea529STheodore Ts'o if (WARN_ON(free <= 0)) 2425907ea529STheodore Ts'o return; 2426c9de560dSAlex Tomas 2427c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 2428c9de560dSAlex Tomas 2429c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 2430ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 24317137d7a4STheodore Ts'o EXT4_CLUSTERS_PER_GROUP(sb), i); 24327137d7a4STheodore Ts'o if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { 243326346ff6SAneesh Kumar K.V /* 2434e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 243526346ff6SAneesh Kumar K.V * free blocks even though group info says we 2436b483bb77SRandy Dunlap * have free blocks 243726346ff6SAneesh Kumar K.V */ 2438e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 243953accfa9STheodore Ts'o "%d free clusters as per " 2440fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 244126346ff6SAneesh Kumar K.V free); 2442736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 2443736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2444c9de560dSAlex Tomas break; 2445c9de560dSAlex Tomas } 2446c9de560dSAlex Tomas 2447304749c0SOjaswin Mujoo if (!ext4_mb_cr_expensive(ac->ac_criteria)) { 24481b420011SOjaswin Mujoo /* 2449f52f3d2bSOjaswin Mujoo * In CR_GOAL_LEN_FAST and CR_BEST_AVAIL_LEN, we are 2450f52f3d2bSOjaswin Mujoo * sure that this group will have a large enough 2451f52f3d2bSOjaswin Mujoo * continuous free extent, so skip over the smaller free 2452f52f3d2bSOjaswin Mujoo * extents 24531b420011SOjaswin Mujoo */ 24541b420011SOjaswin Mujoo j = mb_find_next_bit(bitmap, 24551b420011SOjaswin Mujoo EXT4_CLUSTERS_PER_GROUP(sb), i); 24561b420011SOjaswin Mujoo freelen = j - i; 24571b420011SOjaswin Mujoo 24581b420011SOjaswin Mujoo if (freelen < ac->ac_g_ex.fe_len) { 24591b420011SOjaswin Mujoo i = j; 24601b420011SOjaswin Mujoo free -= freelen; 24611b420011SOjaswin Mujoo continue; 24621b420011SOjaswin Mujoo } 24631b420011SOjaswin Mujoo } 24641b420011SOjaswin Mujoo 246515c006a2SRobin Dong mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); 2466907ea529STheodore Ts'o if (WARN_ON(ex.fe_len <= 0)) 2467907ea529STheodore Ts'o break; 246826346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 2469e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 247053accfa9STheodore Ts'o "%d free clusters as per " 2471fde4d95aSTheodore Ts'o "group info. But got %d blocks", 247226346ff6SAneesh Kumar K.V free, ex.fe_len); 2473736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 2474736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2475e56eb659SAneesh Kumar K.V /* 2476e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 2477e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 2478e56eb659SAneesh Kumar K.V * without claiming the space. 2479e56eb659SAneesh Kumar K.V */ 2480e56eb659SAneesh Kumar K.V break; 248126346ff6SAneesh Kumar K.V } 2482ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADC0DE; /* debug value */ 2483c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 2484c9de560dSAlex Tomas 2485c9de560dSAlex Tomas i += ex.fe_len; 2486c9de560dSAlex Tomas free -= ex.fe_len; 2487c9de560dSAlex Tomas } 2488c9de560dSAlex Tomas 2489c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 2490c9de560dSAlex Tomas } 2491c9de560dSAlex Tomas 2492c9de560dSAlex Tomas /* 2493c9de560dSAlex Tomas * This is a special case for storages like raid5 2494506bf2d8SEric Sandeen * we try to find stripe-aligned chunks for stripe-size-multiple requests 2495c9de560dSAlex Tomas */ 2496089ceeccSEric Sandeen static noinline_for_stack 2497089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 2498c9de560dSAlex Tomas struct ext4_buddy *e4b) 2499c9de560dSAlex Tomas { 2500c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2501c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2502c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2503c9de560dSAlex Tomas struct ext4_free_extent ex; 2504c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 2505c9de560dSAlex Tomas ext4_fsblk_t a; 2506c3defd99SKemeng Shi ext4_grpblk_t i, stripe; 2507c9de560dSAlex Tomas int max; 2508c9de560dSAlex Tomas 2509c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 2510c9de560dSAlex Tomas 2511c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 25125661bd68SAkinobu Mita first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); 25135661bd68SAkinobu Mita 2514c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 2515c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 2516c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 2517c9de560dSAlex Tomas 2518c3defd99SKemeng Shi stripe = EXT4_B2C(sbi, sbi->s_stripe); 2519c3defd99SKemeng Shi i = EXT4_B2C(sbi, i); 25207137d7a4STheodore Ts'o while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { 2521c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 2522c3defd99SKemeng Shi max = mb_find_extent(e4b, i, stripe, &ex); 2523c3defd99SKemeng Shi if (max >= stripe) { 2524c9de560dSAlex Tomas ac->ac_found++; 2525fdd9a009SOjaswin Mujoo ac->ac_cX_found[ac->ac_criteria]++; 2526ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADF00D; /* debug value */ 2527c9de560dSAlex Tomas ac->ac_b_ex = ex; 2528c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2529c9de560dSAlex Tomas break; 2530c9de560dSAlex Tomas } 2531c9de560dSAlex Tomas } 2532c3defd99SKemeng Shi i += stripe; 2533c9de560dSAlex Tomas } 2534c9de560dSAlex Tomas } 2535c9de560dSAlex Tomas 253642ac1848SLukas Czerner /* 25378ef123feSRitesh Harjani * This is also called BEFORE we load the buddy bitmap. 253842ac1848SLukas Czerner * Returns either 1 or 0 indicating that the group is either suitable 25398ef123feSRitesh Harjani * for the allocation or not. 254042ac1848SLukas Czerner */ 25418ef123feSRitesh Harjani static bool ext4_mb_good_group(struct ext4_allocation_context *ac, 25424eb7a4a1SOjaswin Mujoo ext4_group_t group, enum criteria cr) 2543c9de560dSAlex Tomas { 25448ef123feSRitesh Harjani ext4_grpblk_t free, fragments; 2545a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 2546c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 2547c9de560dSAlex Tomas 2548f52f3d2bSOjaswin Mujoo BUG_ON(cr < CR_POWER2_ALIGNED || cr >= EXT4_MB_NUM_CRS); 25498a57d9d6SCurt Wohlgemuth 2550a9ce5993SKemeng Shi if (unlikely(!grp || EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 25518ef123feSRitesh Harjani return false; 255201fc48e8STheodore Ts'o 2553dddcd2f9Sbrookxu free = grp->bb_free; 2554dddcd2f9Sbrookxu if (free == 0) 25558ef123feSRitesh Harjani return false; 2556c9de560dSAlex Tomas 2557c9de560dSAlex Tomas fragments = grp->bb_fragments; 2558c9de560dSAlex Tomas if (fragments == 0) 25598ef123feSRitesh Harjani return false; 2560c9de560dSAlex Tomas 2561c9de560dSAlex Tomas switch (cr) { 2562f52f3d2bSOjaswin Mujoo case CR_POWER2_ALIGNED: 2563c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 2564c9de560dSAlex Tomas 2565a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 2566a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 2567a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 2568a4912123STheodore Ts'o ((group % flex_size) == 0)) 25698ef123feSRitesh Harjani return false; 2570a4912123STheodore Ts'o 2571dddcd2f9Sbrookxu if (free < ac->ac_g_ex.fe_len) 2572dddcd2f9Sbrookxu return false; 2573dddcd2f9Sbrookxu 25744b68f6dfSHarshad Shirwadkar if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb)) 25758ef123feSRitesh Harjani return true; 257640ae3487STheodore Ts'o 257740ae3487STheodore Ts'o if (grp->bb_largest_free_order < ac->ac_2order) 25788ef123feSRitesh Harjani return false; 257940ae3487STheodore Ts'o 25808ef123feSRitesh Harjani return true; 2581f52f3d2bSOjaswin Mujoo case CR_GOAL_LEN_FAST: 2582f52f3d2bSOjaswin Mujoo case CR_BEST_AVAIL_LEN: 2583c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 25848ef123feSRitesh Harjani return true; 2585c9de560dSAlex Tomas break; 2586f52f3d2bSOjaswin Mujoo case CR_GOAL_LEN_SLOW: 2587c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 25888ef123feSRitesh Harjani return true; 2589c9de560dSAlex Tomas break; 2590f52f3d2bSOjaswin Mujoo case CR_ANY_FREE: 25918ef123feSRitesh Harjani return true; 2592c9de560dSAlex Tomas default: 2593c9de560dSAlex Tomas BUG(); 2594c9de560dSAlex Tomas } 2595c9de560dSAlex Tomas 25968ef123feSRitesh Harjani return false; 25978ef123feSRitesh Harjani } 25988ef123feSRitesh Harjani 25998ef123feSRitesh Harjani /* 26008ef123feSRitesh Harjani * This could return negative error code if something goes wrong 26018ef123feSRitesh Harjani * during ext4_mb_init_group(). This should not be called with 26028ef123feSRitesh Harjani * ext4_lock_group() held. 2603a5fda113STheodore Ts'o * 2604a5fda113STheodore Ts'o * Note: because we are conditionally operating with the group lock in 2605a5fda113STheodore Ts'o * the EXT4_MB_STRICT_CHECK case, we need to fake out sparse in this 2606a5fda113STheodore Ts'o * function using __acquire and __release. This means we need to be 2607a5fda113STheodore Ts'o * super careful before messing with the error path handling via "goto 2608a5fda113STheodore Ts'o * out"! 26098ef123feSRitesh Harjani */ 26108ef123feSRitesh Harjani static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, 26114eb7a4a1SOjaswin Mujoo ext4_group_t group, enum criteria cr) 26128ef123feSRitesh Harjani { 26138ef123feSRitesh Harjani struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 261499377830SRitesh Harjani struct super_block *sb = ac->ac_sb; 2615c1d2c7d4SAlex Zhuravlev struct ext4_sb_info *sbi = EXT4_SB(sb); 261699377830SRitesh Harjani bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; 26178ef123feSRitesh Harjani ext4_grpblk_t free; 26188ef123feSRitesh Harjani int ret = 0; 26198ef123feSRitesh Harjani 26205354b2afSTheodore Ts'o if (!grp) 26215354b2afSTheodore Ts'o return -EFSCORRUPTED; 2622a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats) 2623a6c75eafSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); 2624a5fda113STheodore Ts'o if (should_lock) { 262599377830SRitesh Harjani ext4_lock_group(sb, group); 2626a5fda113STheodore Ts'o __release(ext4_group_lock_ptr(sb, group)); 2627a5fda113STheodore Ts'o } 26288ef123feSRitesh Harjani free = grp->bb_free; 26298ef123feSRitesh Harjani if (free == 0) 26308ef123feSRitesh Harjani goto out; 2631304749c0SOjaswin Mujoo /* 2632304749c0SOjaswin Mujoo * In all criterias except CR_ANY_FREE we try to avoid groups that 2633304749c0SOjaswin Mujoo * can't possibly satisfy the full goal request due to insufficient 2634304749c0SOjaswin Mujoo * free blocks. 2635304749c0SOjaswin Mujoo */ 2636304749c0SOjaswin Mujoo if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len) 26378ef123feSRitesh Harjani goto out; 26388ef123feSRitesh Harjani if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 26398ef123feSRitesh Harjani goto out; 2640a5fda113STheodore Ts'o if (should_lock) { 2641a5fda113STheodore Ts'o __acquire(ext4_group_lock_ptr(sb, group)); 264299377830SRitesh Harjani ext4_unlock_group(sb, group); 2643a5fda113STheodore Ts'o } 26448ef123feSRitesh Harjani 26458ef123feSRitesh Harjani /* We only do this if the grp has never been initialized */ 26468ef123feSRitesh Harjani if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 2647c1d2c7d4SAlex Zhuravlev struct ext4_group_desc *gdp = 2648c1d2c7d4SAlex Zhuravlev ext4_get_group_desc(sb, group, NULL); 2649c1d2c7d4SAlex Zhuravlev int ret; 2650c1d2c7d4SAlex Zhuravlev 2651f52f3d2bSOjaswin Mujoo /* 2652f52f3d2bSOjaswin Mujoo * cr=CR_POWER2_ALIGNED/CR_GOAL_LEN_FAST is a very optimistic 2653f52f3d2bSOjaswin Mujoo * search to find large good chunks almost for free. If buddy 2654f52f3d2bSOjaswin Mujoo * data is not ready, then this optimization makes no sense. But 2655f52f3d2bSOjaswin Mujoo * we never skip the first block group in a flex_bg, since this 2656f52f3d2bSOjaswin Mujoo * gets used for metadata block allocation, and we want to make 2657f52f3d2bSOjaswin Mujoo * sure we locate metadata blocks in the first block group in 2658f52f3d2bSOjaswin Mujoo * the flex_bg if possible. 2659c1d2c7d4SAlex Zhuravlev */ 2660304749c0SOjaswin Mujoo if (!ext4_mb_cr_expensive(cr) && 2661c1d2c7d4SAlex Zhuravlev (!sbi->s_log_groups_per_flex || 2662c1d2c7d4SAlex Zhuravlev ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && 2663c1d2c7d4SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2664c1d2c7d4SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) 2665c1d2c7d4SAlex Zhuravlev return 0; 2666c1d2c7d4SAlex Zhuravlev ret = ext4_mb_init_group(sb, group, GFP_NOFS); 26678ef123feSRitesh Harjani if (ret) 26688ef123feSRitesh Harjani return ret; 26698ef123feSRitesh Harjani } 26708ef123feSRitesh Harjani 2671a5fda113STheodore Ts'o if (should_lock) { 267299377830SRitesh Harjani ext4_lock_group(sb, group); 2673a5fda113STheodore Ts'o __release(ext4_group_lock_ptr(sb, group)); 2674a5fda113STheodore Ts'o } 26758ef123feSRitesh Harjani ret = ext4_mb_good_group(ac, group, cr); 26768ef123feSRitesh Harjani out: 2677a5fda113STheodore Ts'o if (should_lock) { 2678a5fda113STheodore Ts'o __acquire(ext4_group_lock_ptr(sb, group)); 267999377830SRitesh Harjani ext4_unlock_group(sb, group); 2680a5fda113STheodore Ts'o } 26818ef123feSRitesh Harjani return ret; 2682c9de560dSAlex Tomas } 2683c9de560dSAlex Tomas 2684cfd73237SAlex Zhuravlev /* 2685cfd73237SAlex Zhuravlev * Start prefetching @nr block bitmaps starting at @group. 2686cfd73237SAlex Zhuravlev * Return the next group which needs to be prefetched. 2687cfd73237SAlex Zhuravlev */ 26883d392b26STheodore Ts'o ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, 2689cfd73237SAlex Zhuravlev unsigned int nr, int *cnt) 2690cfd73237SAlex Zhuravlev { 2691cfd73237SAlex Zhuravlev ext4_group_t ngroups = ext4_get_groups_count(sb); 2692cfd73237SAlex Zhuravlev struct buffer_head *bh; 2693cfd73237SAlex Zhuravlev struct blk_plug plug; 2694cfd73237SAlex Zhuravlev 2695cfd73237SAlex Zhuravlev blk_start_plug(&plug); 2696cfd73237SAlex Zhuravlev while (nr-- > 0) { 2697cfd73237SAlex Zhuravlev struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, 2698cfd73237SAlex Zhuravlev NULL); 2699cfd73237SAlex Zhuravlev struct ext4_group_info *grp = ext4_get_group_info(sb, group); 2700cfd73237SAlex Zhuravlev 2701cfd73237SAlex Zhuravlev /* 2702cfd73237SAlex Zhuravlev * Prefetch block groups with free blocks; but don't 2703cfd73237SAlex Zhuravlev * bother if it is marked uninitialized on disk, since 2704cfd73237SAlex Zhuravlev * it won't require I/O to read. Also only try to 2705cfd73237SAlex Zhuravlev * prefetch once, so we avoid getblk() call, which can 2706cfd73237SAlex Zhuravlev * be expensive. 2707cfd73237SAlex Zhuravlev */ 27085354b2afSTheodore Ts'o if (gdp && grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && 2709cfd73237SAlex Zhuravlev EXT4_MB_GRP_NEED_INIT(grp) && 27103c629604SOjaswin Mujoo ext4_free_group_clusters(sb, gdp) > 0 ) { 2711cfd73237SAlex Zhuravlev bh = ext4_read_block_bitmap_nowait(sb, group, true); 2712cfd73237SAlex Zhuravlev if (bh && !IS_ERR(bh)) { 2713cfd73237SAlex Zhuravlev if (!buffer_uptodate(bh) && cnt) 2714cfd73237SAlex Zhuravlev (*cnt)++; 2715cfd73237SAlex Zhuravlev brelse(bh); 2716cfd73237SAlex Zhuravlev } 2717cfd73237SAlex Zhuravlev } 2718cfd73237SAlex Zhuravlev if (++group >= ngroups) 2719cfd73237SAlex Zhuravlev group = 0; 2720cfd73237SAlex Zhuravlev } 2721cfd73237SAlex Zhuravlev blk_finish_plug(&plug); 2722cfd73237SAlex Zhuravlev return group; 2723cfd73237SAlex Zhuravlev } 2724cfd73237SAlex Zhuravlev 2725cfd73237SAlex Zhuravlev /* 2726cfd73237SAlex Zhuravlev * Prefetching reads the block bitmap into the buffer cache; but we 2727cfd73237SAlex Zhuravlev * need to make sure that the buddy bitmap in the page cache has been 2728cfd73237SAlex Zhuravlev * initialized. Note that ext4_mb_init_group() will block if the I/O 2729cfd73237SAlex Zhuravlev * is not yet completed, or indeed if it was not initiated by 2730cfd73237SAlex Zhuravlev * ext4_mb_prefetch did not start the I/O. 2731cfd73237SAlex Zhuravlev * 2732cfd73237SAlex Zhuravlev * TODO: We should actually kick off the buddy bitmap setup in a work 2733cfd73237SAlex Zhuravlev * queue when the buffer I/O is completed, so that we don't block 2734cfd73237SAlex Zhuravlev * waiting for the block allocation bitmap read to finish when 2735cfd73237SAlex Zhuravlev * ext4_mb_prefetch_fini is called from ext4_mb_regular_allocator(). 2736cfd73237SAlex Zhuravlev */ 27373d392b26STheodore Ts'o void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group, 2738cfd73237SAlex Zhuravlev unsigned int nr) 2739cfd73237SAlex Zhuravlev { 274022fab984SKemeng Shi struct ext4_group_desc *gdp; 274122fab984SKemeng Shi struct ext4_group_info *grp; 2742cfd73237SAlex Zhuravlev 274322fab984SKemeng Shi while (nr-- > 0) { 2744cfd73237SAlex Zhuravlev if (!group) 2745cfd73237SAlex Zhuravlev group = ext4_get_groups_count(sb); 2746cfd73237SAlex Zhuravlev group--; 274722fab984SKemeng Shi gdp = ext4_get_group_desc(sb, group, NULL); 2748cfd73237SAlex Zhuravlev grp = ext4_get_group_info(sb, group); 2749cfd73237SAlex Zhuravlev 27505354b2afSTheodore Ts'o if (grp && gdp && EXT4_MB_GRP_NEED_INIT(grp) && 27513c629604SOjaswin Mujoo ext4_free_group_clusters(sb, gdp) > 0) { 2752cfd73237SAlex Zhuravlev if (ext4_mb_init_group(sb, group, GFP_NOFS)) 2753cfd73237SAlex Zhuravlev break; 2754cfd73237SAlex Zhuravlev } 2755cfd73237SAlex Zhuravlev } 2756cfd73237SAlex Zhuravlev } 2757cfd73237SAlex Zhuravlev 27584ddfef7bSEric Sandeen static noinline_for_stack int 27594ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 2760c9de560dSAlex Tomas { 2761cfd73237SAlex Zhuravlev ext4_group_t prefetch_grp = 0, ngroups, group, i; 27624c0cfebdSTheodore Ts'o enum criteria new_cr, cr = CR_GOAL_LEN_FAST; 276342ac1848SLukas Czerner int err = 0, first_err = 0; 2764cfd73237SAlex Zhuravlev unsigned int nr = 0, prefetch_ios = 0; 2765c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2766c9de560dSAlex Tomas struct super_block *sb; 2767c9de560dSAlex Tomas struct ext4_buddy e4b; 276866d5e027Sbrookxu int lost; 2769c9de560dSAlex Tomas 2770c9de560dSAlex Tomas sb = ac->ac_sb; 2771c9de560dSAlex Tomas sbi = EXT4_SB(sb); 27728df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 2773fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 277412e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) 2775fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 2776fb0a387dSEric Sandeen 2777c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2778c9de560dSAlex Tomas 2779c9de560dSAlex Tomas /* first, try the goal */ 2780c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 2781c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 2782c9de560dSAlex Tomas goto out; 2783c9de560dSAlex Tomas 2784c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 2785c9de560dSAlex Tomas goto out; 2786c9de560dSAlex Tomas 2787c9de560dSAlex Tomas /* 2788e9a3cd48Sbrookxu * ac->ac_2order is set only if the fe_len is a power of 2 2789e9a3cd48Sbrookxu * if ac->ac_2order is set we also set criteria to 0 so that we 2790c9de560dSAlex Tomas * try exact allocation using buddy. 2791c9de560dSAlex Tomas */ 2792c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 2793c9de560dSAlex Tomas ac->ac_2order = 0; 2794c9de560dSAlex Tomas /* 2795c9de560dSAlex Tomas * We search using buddy data only if the order of the request 2796c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 2797b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 2798d9b22cf9SJan Kara * We also support searching for power-of-two requests only for 2799d9b22cf9SJan Kara * requests upto maximum buddy size we have constructed. 2800c9de560dSAlex Tomas */ 28014b68f6dfSHarshad Shirwadkar if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) { 2802c9de560dSAlex Tomas /* 2803c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 2804c9de560dSAlex Tomas */ 2805c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 28061a5d5e5dSJeremy Cline ac->ac_2order = array_index_nospec(i - 1, 28074b68f6dfSHarshad Shirwadkar MB_NUM_ORDERS(sb)); 2808c9de560dSAlex Tomas } 2809c9de560dSAlex Tomas 28104ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 28114ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2812c9de560dSAlex Tomas /* TBD: may be hot point */ 2813c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2814c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2815c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2816c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2817c9de560dSAlex Tomas } 28184ba74d00STheodore Ts'o 2819c9de560dSAlex Tomas /* 28204c0cfebdSTheodore Ts'o * Let's just scan groups to find more-less suitable blocks We 28214c0cfebdSTheodore Ts'o * start with CR_GOAL_LEN_FAST, unless it is power of 2 28224c0cfebdSTheodore Ts'o * aligned, in which case let's do that faster approach first. 2823c9de560dSAlex Tomas */ 28244c0cfebdSTheodore Ts'o if (ac->ac_2order) 28254c0cfebdSTheodore Ts'o cr = CR_POWER2_ALIGNED; 2826c9de560dSAlex Tomas repeat: 28274eb7a4a1SOjaswin Mujoo for (; cr < EXT4_MB_NUM_CRS && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2828c9de560dSAlex Tomas ac->ac_criteria = cr; 2829ed8f9c75SAneesh Kumar K.V /* 2830ed8f9c75SAneesh Kumar K.V * searching for the right group start 2831ed8f9c75SAneesh Kumar K.V * from the goal value specified 2832ed8f9c75SAneesh Kumar K.V */ 2833ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2834196e402aSHarshad Shirwadkar ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; 2835cfd73237SAlex Zhuravlev prefetch_grp = group; 2836ed8f9c75SAneesh Kumar K.V 28374fca50d4SJan Kara for (i = 0, new_cr = cr; i < ngroups; i++, 28384fca50d4SJan Kara ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) { 28394fca50d4SJan Kara int ret = 0; 2840196e402aSHarshad Shirwadkar 28412ed5724dSTheodore Ts'o cond_resched(); 2842196e402aSHarshad Shirwadkar if (new_cr != cr) { 2843196e402aSHarshad Shirwadkar cr = new_cr; 2844196e402aSHarshad Shirwadkar goto repeat; 2845196e402aSHarshad Shirwadkar } 2846c9de560dSAlex Tomas 2847cfd73237SAlex Zhuravlev /* 2848cfd73237SAlex Zhuravlev * Batch reads of the block allocation bitmaps 2849cfd73237SAlex Zhuravlev * to get multiple READs in flight; limit 2850cfd73237SAlex Zhuravlev * prefetching at cr=0/1, otherwise mballoc can 2851cfd73237SAlex Zhuravlev * spend a lot of time loading imperfect groups 2852cfd73237SAlex Zhuravlev */ 2853cfd73237SAlex Zhuravlev if ((prefetch_grp == group) && 2854304749c0SOjaswin Mujoo (ext4_mb_cr_expensive(cr) || 2855cfd73237SAlex Zhuravlev prefetch_ios < sbi->s_mb_prefetch_limit)) { 2856cfd73237SAlex Zhuravlev nr = sbi->s_mb_prefetch; 2857cfd73237SAlex Zhuravlev if (ext4_has_feature_flex_bg(sb)) { 285882ef1370SChunguang Xu nr = 1 << sbi->s_log_groups_per_flex; 285982ef1370SChunguang Xu nr -= group & (nr - 1); 286082ef1370SChunguang Xu nr = min(nr, sbi->s_mb_prefetch); 2861cfd73237SAlex Zhuravlev } 2862cfd73237SAlex Zhuravlev prefetch_grp = ext4_mb_prefetch(sb, group, 2863cfd73237SAlex Zhuravlev nr, &prefetch_ios); 2864cfd73237SAlex Zhuravlev } 2865cfd73237SAlex Zhuravlev 28668a57d9d6SCurt Wohlgemuth /* This now checks without needing the buddy page */ 28678ef123feSRitesh Harjani ret = ext4_mb_good_group_nolock(ac, group, cr); 286842ac1848SLukas Czerner if (ret <= 0) { 286942ac1848SLukas Czerner if (!first_err) 287042ac1848SLukas Czerner first_err = ret; 2871c9de560dSAlex Tomas continue; 287242ac1848SLukas Czerner } 2873c9de560dSAlex Tomas 2874c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2875c9de560dSAlex Tomas if (err) 2876c9de560dSAlex Tomas goto out; 2877c9de560dSAlex Tomas 2878c9de560dSAlex Tomas ext4_lock_group(sb, group); 28798a57d9d6SCurt Wohlgemuth 28808a57d9d6SCurt Wohlgemuth /* 28818a57d9d6SCurt Wohlgemuth * We need to check again after locking the 28828a57d9d6SCurt Wohlgemuth * block group 28838a57d9d6SCurt Wohlgemuth */ 288442ac1848SLukas Czerner ret = ext4_mb_good_group(ac, group, cr); 28858ef123feSRitesh Harjani if (ret == 0) { 2886c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2887e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2888c9de560dSAlex Tomas continue; 2889c9de560dSAlex Tomas } 2890c9de560dSAlex Tomas 2891c9de560dSAlex Tomas ac->ac_groups_scanned++; 2892f52f3d2bSOjaswin Mujoo if (cr == CR_POWER2_ALIGNED) 2893c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2894f52f3d2bSOjaswin Mujoo else if ((cr == CR_GOAL_LEN_FAST || 2895f52f3d2bSOjaswin Mujoo cr == CR_BEST_AVAIL_LEN) && 2896f52f3d2bSOjaswin Mujoo sbi->s_stripe && 2897c3defd99SKemeng Shi !(ac->ac_g_ex.fe_len % 2898c3defd99SKemeng Shi EXT4_B2C(sbi, sbi->s_stripe))) 2899c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2900c9de560dSAlex Tomas else 2901c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2902c9de560dSAlex Tomas 2903c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2904e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2905c9de560dSAlex Tomas 2906c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2907c9de560dSAlex Tomas break; 2908c9de560dSAlex Tomas } 2909a6c75eafSHarshad Shirwadkar /* Processed all groups and haven't found blocks */ 2910a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats && i == ngroups) 2911a6c75eafSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_failed[cr]); 29127e170922SOjaswin Mujoo 2913f52f3d2bSOjaswin Mujoo if (i == ngroups && ac->ac_criteria == CR_BEST_AVAIL_LEN) 29147e170922SOjaswin Mujoo /* Reset goal length to original goal length before 2915f52f3d2bSOjaswin Mujoo * falling into CR_GOAL_LEN_SLOW */ 29167e170922SOjaswin Mujoo ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; 2917c9de560dSAlex Tomas } 2918c9de560dSAlex Tomas 2919c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2920c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2921c9de560dSAlex Tomas /* 2922c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2923c9de560dSAlex Tomas * the best chunk we've found so far 2924c9de560dSAlex Tomas */ 2925c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2926c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2927c9de560dSAlex Tomas /* 2928c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2929c9de560dSAlex Tomas * The only thing we can do is just take first 2930c9de560dSAlex Tomas * found block(s) 2931c9de560dSAlex Tomas */ 293266d5e027Sbrookxu lost = atomic_inc_return(&sbi->s_mb_lost_chunks); 293366d5e027Sbrookxu mb_debug(sb, "lost chunk, group: %u, start: %d, len: %d, lost: %d\n", 2934c55ee7d2Sbrookxu ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, 2935c55ee7d2Sbrookxu ac->ac_b_ex.fe_len, lost); 2936c55ee7d2Sbrookxu 2937c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2938c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2939c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2940c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2941c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2942f52f3d2bSOjaswin Mujoo cr = CR_ANY_FREE; 2943c9de560dSAlex Tomas goto repeat; 2944c9de560dSAlex Tomas } 2945c9de560dSAlex Tomas } 2946a6c75eafSHarshad Shirwadkar 2947a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) 2948a6c75eafSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); 2949c9de560dSAlex Tomas out: 295042ac1848SLukas Czerner if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) 295142ac1848SLukas Czerner err = first_err; 2952bbc4ec77SRitesh Harjani 2953d3df1453SRitesh Harjani mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n", 2954bbc4ec77SRitesh Harjani ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, 2955bbc4ec77SRitesh Harjani ac->ac_flags, cr, err); 2956cfd73237SAlex Zhuravlev 2957cfd73237SAlex Zhuravlev if (nr) 2958cfd73237SAlex Zhuravlev ext4_mb_prefetch_fini(sb, prefetch_grp, nr); 2959cfd73237SAlex Zhuravlev 2960c9de560dSAlex Tomas return err; 2961c9de560dSAlex Tomas } 2962c9de560dSAlex Tomas 2963c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2964c9de560dSAlex Tomas { 2965359745d7SMuchun Song struct super_block *sb = pde_data(file_inode(seq->file)); 2966c9de560dSAlex Tomas ext4_group_t group; 2967c9de560dSAlex Tomas 29688df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2969c9de560dSAlex Tomas return NULL; 2970c9de560dSAlex Tomas group = *pos + 1; 2971a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2972c9de560dSAlex Tomas } 2973c9de560dSAlex Tomas 2974c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2975c9de560dSAlex Tomas { 2976359745d7SMuchun Song struct super_block *sb = pde_data(file_inode(seq->file)); 2977c9de560dSAlex Tomas ext4_group_t group; 2978c9de560dSAlex Tomas 2979c9de560dSAlex Tomas ++*pos; 29808df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2981c9de560dSAlex Tomas return NULL; 2982c9de560dSAlex Tomas group = *pos + 1; 2983a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2984c9de560dSAlex Tomas } 2985c9de560dSAlex Tomas 2986c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2987c9de560dSAlex Tomas { 2988359745d7SMuchun Song struct super_block *sb = pde_data(file_inode(seq->file)); 2989a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2990c9de560dSAlex Tomas int i; 29911c8457caSAditya Kali int err, buddy_loaded = 0; 2992c9de560dSAlex Tomas struct ext4_buddy e4b; 29931c8457caSAditya Kali struct ext4_group_info *grinfo; 29942df2c340SArnd Bergmann unsigned char blocksize_bits = min_t(unsigned char, 29952df2c340SArnd Bergmann sb->s_blocksize_bits, 29962df2c340SArnd Bergmann EXT4_MAX_BLOCK_LOG_SIZE); 2997c9de560dSAlex Tomas struct sg { 2998c9de560dSAlex Tomas struct ext4_group_info info; 2999b80b32b6STheodore Ts'o ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2]; 3000c9de560dSAlex Tomas } sg; 3001c9de560dSAlex Tomas 3002c9de560dSAlex Tomas group--; 3003c9de560dSAlex Tomas if (group == 0) 300497b4af2fSRasmus Villemoes seq_puts(seq, "#group: free frags first [" 300597b4af2fSRasmus Villemoes " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " 3006802cf1f9SHuaitong Han " 2^7 2^8 2^9 2^10 2^11 2^12 2^13 ]\n"); 3007c9de560dSAlex Tomas 3008b80b32b6STheodore Ts'o i = (blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 3009b80b32b6STheodore Ts'o sizeof(struct ext4_group_info); 3010b80b32b6STheodore Ts'o 30111c8457caSAditya Kali grinfo = ext4_get_group_info(sb, group); 30125354b2afSTheodore Ts'o if (!grinfo) 30135354b2afSTheodore Ts'o return 0; 30141c8457caSAditya Kali /* Load the group info in memory only if not already loaded. */ 30151c8457caSAditya Kali if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) { 3016c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3017c9de560dSAlex Tomas if (err) { 3018a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 3019c9de560dSAlex Tomas return 0; 3020c9de560dSAlex Tomas } 30211c8457caSAditya Kali buddy_loaded = 1; 30221c8457caSAditya Kali } 30231c8457caSAditya Kali 30245354b2afSTheodore Ts'o memcpy(&sg, grinfo, i); 30251c8457caSAditya Kali 30261c8457caSAditya Kali if (buddy_loaded) 3027e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3028c9de560dSAlex Tomas 3029a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 3030c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 3031c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 30322df2c340SArnd Bergmann seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? 3033c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 3034e0d438c7SXu Wang seq_puts(seq, " ]\n"); 3035c9de560dSAlex Tomas 3036c9de560dSAlex Tomas return 0; 3037c9de560dSAlex Tomas } 3038c9de560dSAlex Tomas 3039c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 3040c9de560dSAlex Tomas { 3041c9de560dSAlex Tomas } 3042c9de560dSAlex Tomas 3043247dbed8SChristoph Hellwig const struct seq_operations ext4_mb_seq_groups_ops = { 3044c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 3045c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 3046c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 3047c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 3048c9de560dSAlex Tomas }; 3049c9de560dSAlex Tomas 3050a6c75eafSHarshad Shirwadkar int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) 3051a6c75eafSHarshad Shirwadkar { 3052c30365b9SYu Zhe struct super_block *sb = seq->private; 3053a6c75eafSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 3054a6c75eafSHarshad Shirwadkar 3055a6c75eafSHarshad Shirwadkar seq_puts(seq, "mballoc:\n"); 3056a6c75eafSHarshad Shirwadkar if (!sbi->s_mb_stats) { 3057a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tmb stats collection turned off.\n"); 3058f52f3d2bSOjaswin Mujoo seq_puts( 3059f52f3d2bSOjaswin Mujoo seq, 3060f52f3d2bSOjaswin Mujoo "\tTo enable, please write \"1\" to sysfs file mb_stats.\n"); 3061a6c75eafSHarshad Shirwadkar return 0; 3062a6c75eafSHarshad Shirwadkar } 3063a6c75eafSHarshad Shirwadkar seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs)); 3064a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success)); 3065a6c75eafSHarshad Shirwadkar 3066f52f3d2bSOjaswin Mujoo seq_printf(seq, "\tgroups_scanned: %u\n", 3067f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_groups_scanned)); 3068a6c75eafSHarshad Shirwadkar 3069f52f3d2bSOjaswin Mujoo /* CR_POWER2_ALIGNED stats */ 3070f52f3d2bSOjaswin Mujoo seq_puts(seq, "\tcr_p2_aligned_stats:\n"); 3071f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\thits: %llu\n", 3072f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_hits[CR_POWER2_ALIGNED])); 3073f52f3d2bSOjaswin Mujoo seq_printf( 3074f52f3d2bSOjaswin Mujoo seq, "\t\tgroups_considered: %llu\n", 3075f52f3d2bSOjaswin Mujoo atomic64_read( 3076f52f3d2bSOjaswin Mujoo &sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED])); 3077f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\textents_scanned: %u\n", 3078f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_cX_ex_scanned[CR_POWER2_ALIGNED])); 3079a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 3080f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_failed[CR_POWER2_ALIGNED])); 3081196e402aSHarshad Shirwadkar seq_printf(seq, "\t\tbad_suggestions: %u\n", 3082f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_p2_aligned_bad_suggestions)); 3083a6c75eafSHarshad Shirwadkar 3084f52f3d2bSOjaswin Mujoo /* CR_GOAL_LEN_FAST stats */ 3085f52f3d2bSOjaswin Mujoo seq_puts(seq, "\tcr_goal_fast_stats:\n"); 3086f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\thits: %llu\n", 3087f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_FAST])); 3088a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgroups_considered: %llu\n", 3089f52f3d2bSOjaswin Mujoo atomic64_read( 3090f52f3d2bSOjaswin Mujoo &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_FAST])); 3091f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\textents_scanned: %u\n", 3092f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_FAST])); 3093a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 3094f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_FAST])); 3095196e402aSHarshad Shirwadkar seq_printf(seq, "\t\tbad_suggestions: %u\n", 3096f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_goal_fast_bad_suggestions)); 3097a6c75eafSHarshad Shirwadkar 3098f52f3d2bSOjaswin Mujoo /* CR_BEST_AVAIL_LEN stats */ 3099f52f3d2bSOjaswin Mujoo seq_puts(seq, "\tcr_best_avail_stats:\n"); 3100f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\thits: %llu\n", 3101f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_hits[CR_BEST_AVAIL_LEN])); 3102f52f3d2bSOjaswin Mujoo seq_printf( 3103f52f3d2bSOjaswin Mujoo seq, "\t\tgroups_considered: %llu\n", 3104f52f3d2bSOjaswin Mujoo atomic64_read( 3105f52f3d2bSOjaswin Mujoo &sbi->s_bal_cX_groups_considered[CR_BEST_AVAIL_LEN])); 3106f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\textents_scanned: %u\n", 3107f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_cX_ex_scanned[CR_BEST_AVAIL_LEN])); 31087e170922SOjaswin Mujoo seq_printf(seq, "\t\tuseless_loops: %llu\n", 3109f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_failed[CR_BEST_AVAIL_LEN])); 31107e170922SOjaswin Mujoo seq_printf(seq, "\t\tbad_suggestions: %u\n", 3111f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_best_avail_bad_suggestions)); 31127e170922SOjaswin Mujoo 3113f52f3d2bSOjaswin Mujoo /* CR_GOAL_LEN_SLOW stats */ 3114f52f3d2bSOjaswin Mujoo seq_puts(seq, "\tcr_goal_slow_stats:\n"); 3115f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\thits: %llu\n", 3116f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_SLOW])); 3117a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgroups_considered: %llu\n", 3118f52f3d2bSOjaswin Mujoo atomic64_read( 3119f52f3d2bSOjaswin Mujoo &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_SLOW])); 3120f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\textents_scanned: %u\n", 3121f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_SLOW])); 3122a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 3123f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_SLOW])); 3124a6c75eafSHarshad Shirwadkar 3125f52f3d2bSOjaswin Mujoo /* CR_ANY_FREE stats */ 3126f52f3d2bSOjaswin Mujoo seq_puts(seq, "\tcr_any_free_stats:\n"); 3127f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\thits: %llu\n", 3128f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_hits[CR_ANY_FREE])); 3129f52f3d2bSOjaswin Mujoo seq_printf( 3130f52f3d2bSOjaswin Mujoo seq, "\t\tgroups_considered: %llu\n", 3131f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_groups_considered[CR_ANY_FREE])); 3132f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\textents_scanned: %u\n", 3133f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_cX_ex_scanned[CR_ANY_FREE])); 3134a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 3135f52f3d2bSOjaswin Mujoo atomic64_read(&sbi->s_bal_cX_failed[CR_ANY_FREE])); 3136f52f3d2bSOjaswin Mujoo 3137f52f3d2bSOjaswin Mujoo /* Aggregates */ 3138f52f3d2bSOjaswin Mujoo seq_printf(seq, "\textents_scanned: %u\n", 3139f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_ex_scanned)); 3140a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals)); 3141f52f3d2bSOjaswin Mujoo seq_printf(seq, "\t\tlen_goal_hits: %u\n", 3142f52f3d2bSOjaswin Mujoo atomic_read(&sbi->s_bal_len_goals)); 3143a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders)); 3144a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks)); 3145a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks)); 3146a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tbuddies_generated: %u/%u\n", 3147a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_mb_buddies_generated), 3148a6c75eafSHarshad Shirwadkar ext4_get_groups_count(sb)); 3149a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tbuddies_time_used: %llu\n", 3150a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_mb_generation_time)); 3151a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tpreallocated: %u\n", 3152a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_mb_preallocated)); 3153f52f3d2bSOjaswin Mujoo seq_printf(seq, "\tdiscarded: %u\n", atomic_read(&sbi->s_mb_discarded)); 3154a6c75eafSHarshad Shirwadkar return 0; 3155a6c75eafSHarshad Shirwadkar } 3156a6c75eafSHarshad Shirwadkar 3157f68f4063SHarshad Shirwadkar static void *ext4_mb_seq_structs_summary_start(struct seq_file *seq, loff_t *pos) 3158a5fda113STheodore Ts'o __acquires(&EXT4_SB(sb)->s_mb_rb_lock) 3159f68f4063SHarshad Shirwadkar { 3160359745d7SMuchun Song struct super_block *sb = pde_data(file_inode(seq->file)); 3161f68f4063SHarshad Shirwadkar unsigned long position; 3162f68f4063SHarshad Shirwadkar 316383e80a6eSJan Kara if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb)) 3164f68f4063SHarshad Shirwadkar return NULL; 3165f68f4063SHarshad Shirwadkar position = *pos + 1; 3166f68f4063SHarshad Shirwadkar return (void *) ((unsigned long) position); 3167f68f4063SHarshad Shirwadkar } 3168f68f4063SHarshad Shirwadkar 3169f68f4063SHarshad Shirwadkar static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, loff_t *pos) 3170f68f4063SHarshad Shirwadkar { 3171359745d7SMuchun Song struct super_block *sb = pde_data(file_inode(seq->file)); 3172f68f4063SHarshad Shirwadkar unsigned long position; 3173f68f4063SHarshad Shirwadkar 3174f68f4063SHarshad Shirwadkar ++*pos; 317583e80a6eSJan Kara if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb)) 3176f68f4063SHarshad Shirwadkar return NULL; 3177f68f4063SHarshad Shirwadkar position = *pos + 1; 3178f68f4063SHarshad Shirwadkar return (void *) ((unsigned long) position); 3179f68f4063SHarshad Shirwadkar } 3180f68f4063SHarshad Shirwadkar 3181f68f4063SHarshad Shirwadkar static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) 3182f68f4063SHarshad Shirwadkar { 3183359745d7SMuchun Song struct super_block *sb = pde_data(file_inode(seq->file)); 3184f68f4063SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 3185f68f4063SHarshad Shirwadkar unsigned long position = ((unsigned long) v); 3186f68f4063SHarshad Shirwadkar struct ext4_group_info *grp; 318783e80a6eSJan Kara unsigned int count; 3188f68f4063SHarshad Shirwadkar 3189f68f4063SHarshad Shirwadkar position--; 3190f68f4063SHarshad Shirwadkar if (position >= MB_NUM_ORDERS(sb)) { 319183e80a6eSJan Kara position -= MB_NUM_ORDERS(sb); 319283e80a6eSJan Kara if (position == 0) 319383e80a6eSJan Kara seq_puts(seq, "avg_fragment_size_lists:\n"); 3194f68f4063SHarshad Shirwadkar 319583e80a6eSJan Kara count = 0; 319683e80a6eSJan Kara read_lock(&sbi->s_mb_avg_fragment_size_locks[position]); 319783e80a6eSJan Kara list_for_each_entry(grp, &sbi->s_mb_avg_fragment_size[position], 319883e80a6eSJan Kara bb_avg_fragment_size_node) 319983e80a6eSJan Kara count++; 320083e80a6eSJan Kara read_unlock(&sbi->s_mb_avg_fragment_size_locks[position]); 320183e80a6eSJan Kara seq_printf(seq, "\tlist_order_%u_groups: %u\n", 320283e80a6eSJan Kara (unsigned int)position, count); 3203f68f4063SHarshad Shirwadkar return 0; 3204f68f4063SHarshad Shirwadkar } 3205f68f4063SHarshad Shirwadkar 3206f68f4063SHarshad Shirwadkar if (position == 0) { 3207f68f4063SHarshad Shirwadkar seq_printf(seq, "optimize_scan: %d\n", 3208f68f4063SHarshad Shirwadkar test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0); 3209f68f4063SHarshad Shirwadkar seq_puts(seq, "max_free_order_lists:\n"); 3210f68f4063SHarshad Shirwadkar } 3211f68f4063SHarshad Shirwadkar count = 0; 321283e80a6eSJan Kara read_lock(&sbi->s_mb_largest_free_orders_locks[position]); 3213f68f4063SHarshad Shirwadkar list_for_each_entry(grp, &sbi->s_mb_largest_free_orders[position], 3214f68f4063SHarshad Shirwadkar bb_largest_free_order_node) 3215f68f4063SHarshad Shirwadkar count++; 321683e80a6eSJan Kara read_unlock(&sbi->s_mb_largest_free_orders_locks[position]); 3217f68f4063SHarshad Shirwadkar seq_printf(seq, "\tlist_order_%u_groups: %u\n", 3218f68f4063SHarshad Shirwadkar (unsigned int)position, count); 3219f68f4063SHarshad Shirwadkar 3220f68f4063SHarshad Shirwadkar return 0; 3221f68f4063SHarshad Shirwadkar } 3222f68f4063SHarshad Shirwadkar 3223f68f4063SHarshad Shirwadkar static void ext4_mb_seq_structs_summary_stop(struct seq_file *seq, void *v) 3224f68f4063SHarshad Shirwadkar { 3225f68f4063SHarshad Shirwadkar } 3226f68f4063SHarshad Shirwadkar 3227f68f4063SHarshad Shirwadkar const struct seq_operations ext4_mb_seq_structs_summary_ops = { 3228f68f4063SHarshad Shirwadkar .start = ext4_mb_seq_structs_summary_start, 3229f68f4063SHarshad Shirwadkar .next = ext4_mb_seq_structs_summary_next, 3230f68f4063SHarshad Shirwadkar .stop = ext4_mb_seq_structs_summary_stop, 3231f68f4063SHarshad Shirwadkar .show = ext4_mb_seq_structs_summary_show, 3232f68f4063SHarshad Shirwadkar }; 3233f68f4063SHarshad Shirwadkar 3234fb1813f4SCurt Wohlgemuth static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) 3235fb1813f4SCurt Wohlgemuth { 3236fb1813f4SCurt Wohlgemuth int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 3237fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; 3238fb1813f4SCurt Wohlgemuth 3239fb1813f4SCurt Wohlgemuth BUG_ON(!cachep); 3240fb1813f4SCurt Wohlgemuth return cachep; 3241fb1813f4SCurt Wohlgemuth } 32425f21b0e6SFrederic Bohe 324328623c2fSTheodore Ts'o /* 324428623c2fSTheodore Ts'o * Allocate the top-level s_group_info array for the specified number 324528623c2fSTheodore Ts'o * of groups 324628623c2fSTheodore Ts'o */ 324728623c2fSTheodore Ts'o int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups) 324828623c2fSTheodore Ts'o { 324928623c2fSTheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 325028623c2fSTheodore Ts'o unsigned size; 3251df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***old_groupinfo, ***new_groupinfo; 325228623c2fSTheodore Ts'o 325328623c2fSTheodore Ts'o size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> 325428623c2fSTheodore Ts'o EXT4_DESC_PER_BLOCK_BITS(sb); 325528623c2fSTheodore Ts'o if (size <= sbi->s_group_info_size) 325628623c2fSTheodore Ts'o return 0; 325728623c2fSTheodore Ts'o 325828623c2fSTheodore Ts'o size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); 3259a7c3e901SMichal Hocko new_groupinfo = kvzalloc(size, GFP_KERNEL); 326028623c2fSTheodore Ts'o if (!new_groupinfo) { 326128623c2fSTheodore Ts'o ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); 326228623c2fSTheodore Ts'o return -ENOMEM; 326328623c2fSTheodore Ts'o } 3264df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3265df3da4eaSSuraj Jitindar Singh old_groupinfo = rcu_dereference(sbi->s_group_info); 3266df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 3267df3da4eaSSuraj Jitindar Singh memcpy(new_groupinfo, old_groupinfo, 326828623c2fSTheodore Ts'o sbi->s_group_info_size * sizeof(*sbi->s_group_info)); 3269df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3270df3da4eaSSuraj Jitindar Singh rcu_assign_pointer(sbi->s_group_info, new_groupinfo); 327128623c2fSTheodore Ts'o sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); 3272df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 3273df3da4eaSSuraj Jitindar Singh ext4_kvfree_array_rcu(old_groupinfo); 327428623c2fSTheodore Ts'o ext4_debug("allocated s_groupinfo array for %d meta_bg's\n", 327528623c2fSTheodore Ts'o sbi->s_group_info_size); 327628623c2fSTheodore Ts'o return 0; 327728623c2fSTheodore Ts'o } 327828623c2fSTheodore Ts'o 32795f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 3280920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 32815f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 32825f21b0e6SFrederic Bohe { 3283fb1813f4SCurt Wohlgemuth int i; 32845f21b0e6SFrederic Bohe int metalen = 0; 3285df3da4eaSSuraj Jitindar Singh int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb); 32865f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 32875f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 3288fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 32895f21b0e6SFrederic Bohe 32905f21b0e6SFrederic Bohe /* 32915f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 32925f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 32935f21b0e6SFrederic Bohe * to ext4_group_info structures 32945f21b0e6SFrederic Bohe */ 32955f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 32965f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 32975f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 32984fdb5543SDmitry Monakhov meta_group_info = kmalloc(metalen, GFP_NOFS); 32995f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 33007f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate mem " 33019d8b9ec4STheodore Ts'o "for a buddy group"); 3302df119095SKemeng Shi return -ENOMEM; 33035f21b0e6SFrederic Bohe } 3304df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3305df3da4eaSSuraj Jitindar Singh rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; 3306df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 33075f21b0e6SFrederic Bohe } 33085f21b0e6SFrederic Bohe 3309df3da4eaSSuraj Jitindar Singh meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx); 33105f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 33115f21b0e6SFrederic Bohe 33124fdb5543SDmitry Monakhov meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_NOFS); 33135f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 33147f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate buddy mem"); 33155f21b0e6SFrederic Bohe goto exit_group_info; 33165f21b0e6SFrederic Bohe } 33175f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 33185f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 33195f21b0e6SFrederic Bohe 33205f21b0e6SFrederic Bohe /* 33215f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 33225f21b0e6SFrederic Bohe * empty groups without initialization 33235f21b0e6SFrederic Bohe */ 33248844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 33258844618dSTheodore Ts'o (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 33265f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 3327cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, group, desc); 33285f21b0e6SFrederic Bohe } else { 33295f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 3330021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, desc); 33315f21b0e6SFrederic Bohe } 33325f21b0e6SFrederic Bohe 33335f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 3334920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 333564e290ecSVenkatesh Pallipadi meta_group_info[i]->bb_free_root = RB_ROOT; 3336196e402aSHarshad Shirwadkar INIT_LIST_HEAD(&meta_group_info[i]->bb_largest_free_order_node); 333783e80a6eSJan Kara INIT_LIST_HEAD(&meta_group_info[i]->bb_avg_fragment_size_node); 33388a57d9d6SCurt Wohlgemuth meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ 333983e80a6eSJan Kara meta_group_info[i]->bb_avg_fragment_size_order = -1; /* uninit */ 3340196e402aSHarshad Shirwadkar meta_group_info[i]->bb_group = group; 33415f21b0e6SFrederic Bohe 3342a3450215SRitesh Harjani mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group); 33435f21b0e6SFrederic Bohe return 0; 33445f21b0e6SFrederic Bohe 33455f21b0e6SFrederic Bohe exit_group_info: 33465f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 3347caaf7a29STao Ma if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 3348df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 3349df3da4eaSSuraj Jitindar Singh 3350df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3351df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 3352df3da4eaSSuraj Jitindar Singh kfree(group_info[idx]); 3353df3da4eaSSuraj Jitindar Singh group_info[idx] = NULL; 3354df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3355caaf7a29STao Ma } 33565f21b0e6SFrederic Bohe return -ENOMEM; 33575f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 33585f21b0e6SFrederic Bohe 3359c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 3360c9de560dSAlex Tomas { 33618df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 3362c9de560dSAlex Tomas ext4_group_t i; 3363c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 336428623c2fSTheodore Ts'o int err; 33655f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 3366df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 3367fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep; 3368c9de560dSAlex Tomas 336928623c2fSTheodore Ts'o err = ext4_mb_alloc_groupinfo(sb, ngroups); 337028623c2fSTheodore Ts'o if (err) 337128623c2fSTheodore Ts'o return err; 33725f21b0e6SFrederic Bohe 3373c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 3374c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 33759d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't get new inode"); 3376c9de560dSAlex Tomas goto err_freesgi; 3377c9de560dSAlex Tomas } 337848e6061bSYu Jian /* To avoid potentially colliding with an valid on-disk inode number, 337948e6061bSYu Jian * use EXT4_BAD_INO for the buddy cache inode number. This inode is 338048e6061bSYu Jian * not in the inode hash, so it should never be found by iget(), but 338148e6061bSYu Jian * this will avoid confusion if it ever shows up during debugging. */ 338248e6061bSYu Jian sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; 3383c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 33848df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 33854b99faa2SKhazhismel Kumykov cond_resched(); 3386c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 3387c9de560dSAlex Tomas if (desc == NULL) { 33889d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); 3389c9de560dSAlex Tomas goto err_freebuddy; 3390c9de560dSAlex Tomas } 33915f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 33925f21b0e6SFrederic Bohe goto err_freebuddy; 3393c9de560dSAlex Tomas } 3394c9de560dSAlex Tomas 3395cfd73237SAlex Zhuravlev if (ext4_has_feature_flex_bg(sb)) { 3396f91436d5SSabyrzhan Tasbolatov /* a single flex group is supposed to be read by a single IO. 3397f91436d5SSabyrzhan Tasbolatov * 2 ^ s_log_groups_per_flex != UINT_MAX as s_mb_prefetch is 3398f91436d5SSabyrzhan Tasbolatov * unsigned integer, so the maximum shift is 32. 3399f91436d5SSabyrzhan Tasbolatov */ 3400f91436d5SSabyrzhan Tasbolatov if (sbi->s_es->s_log_groups_per_flex >= 32) { 3401f91436d5SSabyrzhan Tasbolatov ext4_msg(sb, KERN_ERR, "too many log groups per flexible block group"); 3402a8867f4eSPhillip Potter goto err_freebuddy; 3403f91436d5SSabyrzhan Tasbolatov } 3404f91436d5SSabyrzhan Tasbolatov sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, 340582ef1370SChunguang Xu BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9)); 3406cfd73237SAlex Zhuravlev sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ 3407cfd73237SAlex Zhuravlev } else { 3408cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = 32; 3409cfd73237SAlex Zhuravlev } 3410cfd73237SAlex Zhuravlev if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) 3411cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = ext4_get_groups_count(sb); 3412cfd73237SAlex Zhuravlev /* now many real IOs to prefetch within a single allocation at cr=0 3413cfd73237SAlex Zhuravlev * given cr=0 is an CPU-related optimization we shouldn't try to 3414cfd73237SAlex Zhuravlev * load too many groups, at some point we should start to use what 3415cfd73237SAlex Zhuravlev * we've got in memory. 3416cfd73237SAlex Zhuravlev * with an average random access time 5ms, it'd take a second to get 3417cfd73237SAlex Zhuravlev * 200 groups (* N with flex_bg), so let's make this limit 4 3418cfd73237SAlex Zhuravlev */ 3419cfd73237SAlex Zhuravlev sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4; 3420cfd73237SAlex Zhuravlev if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb)) 3421cfd73237SAlex Zhuravlev sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb); 3422cfd73237SAlex Zhuravlev 3423c9de560dSAlex Tomas return 0; 3424c9de560dSAlex Tomas 3425c9de560dSAlex Tomas err_freebuddy: 3426fb1813f4SCurt Wohlgemuth cachep = get_groupinfo_cache(sb->s_blocksize_bits); 34275354b2afSTheodore Ts'o while (i-- > 0) { 34285354b2afSTheodore Ts'o struct ext4_group_info *grp = ext4_get_group_info(sb, i); 34295354b2afSTheodore Ts'o 34305354b2afSTheodore Ts'o if (grp) 34315354b2afSTheodore Ts'o kmem_cache_free(cachep, grp); 34325354b2afSTheodore Ts'o } 343328623c2fSTheodore Ts'o i = sbi->s_group_info_size; 3434df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3435df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 3436f1fa3342SRoel Kluin while (i-- > 0) 3437df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 3438df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3439c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 3440c9de560dSAlex Tomas err_freesgi: 3441df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3442df3da4eaSSuraj Jitindar Singh kvfree(rcu_dereference(sbi->s_group_info)); 3443df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3444c9de560dSAlex Tomas return -ENOMEM; 3445c9de560dSAlex Tomas } 3446c9de560dSAlex Tomas 34472892c15dSEric Sandeen static void ext4_groupinfo_destroy_slabs(void) 34482892c15dSEric Sandeen { 34492892c15dSEric Sandeen int i; 34502892c15dSEric Sandeen 34512892c15dSEric Sandeen for (i = 0; i < NR_GRPINFO_CACHES; i++) { 34522892c15dSEric Sandeen kmem_cache_destroy(ext4_groupinfo_caches[i]); 34532892c15dSEric Sandeen ext4_groupinfo_caches[i] = NULL; 34542892c15dSEric Sandeen } 34552892c15dSEric Sandeen } 34562892c15dSEric Sandeen 34572892c15dSEric Sandeen static int ext4_groupinfo_create_slab(size_t size) 34582892c15dSEric Sandeen { 34592892c15dSEric Sandeen static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); 34602892c15dSEric Sandeen int slab_size; 34612892c15dSEric Sandeen int blocksize_bits = order_base_2(size); 34622892c15dSEric Sandeen int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 34632892c15dSEric Sandeen struct kmem_cache *cachep; 34642892c15dSEric Sandeen 34652892c15dSEric Sandeen if (cache_index >= NR_GRPINFO_CACHES) 34662892c15dSEric Sandeen return -EINVAL; 34672892c15dSEric Sandeen 34682892c15dSEric Sandeen if (unlikely(cache_index < 0)) 34692892c15dSEric Sandeen cache_index = 0; 34702892c15dSEric Sandeen 34712892c15dSEric Sandeen mutex_lock(&ext4_grpinfo_slab_create_mutex); 34722892c15dSEric Sandeen if (ext4_groupinfo_caches[cache_index]) { 34732892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 34742892c15dSEric Sandeen return 0; /* Already created */ 34752892c15dSEric Sandeen } 34762892c15dSEric Sandeen 34772892c15dSEric Sandeen slab_size = offsetof(struct ext4_group_info, 34782892c15dSEric Sandeen bb_counters[blocksize_bits + 2]); 34792892c15dSEric Sandeen 34802892c15dSEric Sandeen cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], 34812892c15dSEric Sandeen slab_size, 0, SLAB_RECLAIM_ACCOUNT, 34822892c15dSEric Sandeen NULL); 34832892c15dSEric Sandeen 3484823ba01fSTao Ma ext4_groupinfo_caches[cache_index] = cachep; 3485823ba01fSTao Ma 34862892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 34872892c15dSEric Sandeen if (!cachep) { 34889d8b9ec4STheodore Ts'o printk(KERN_EMERG 34899d8b9ec4STheodore Ts'o "EXT4-fs: no memory for groupinfo slab cache\n"); 34902892c15dSEric Sandeen return -ENOMEM; 34912892c15dSEric Sandeen } 34922892c15dSEric Sandeen 34932892c15dSEric Sandeen return 0; 34942892c15dSEric Sandeen } 34952892c15dSEric Sandeen 349655cdd0afSWang Jianchao static void ext4_discard_work(struct work_struct *work) 349755cdd0afSWang Jianchao { 349855cdd0afSWang Jianchao struct ext4_sb_info *sbi = container_of(work, 349955cdd0afSWang Jianchao struct ext4_sb_info, s_discard_work); 350055cdd0afSWang Jianchao struct super_block *sb = sbi->s_sb; 350155cdd0afSWang Jianchao struct ext4_free_data *fd, *nfd; 350255cdd0afSWang Jianchao struct ext4_buddy e4b; 350355cdd0afSWang Jianchao struct list_head discard_list; 350455cdd0afSWang Jianchao ext4_group_t grp, load_grp; 350555cdd0afSWang Jianchao int err = 0; 350655cdd0afSWang Jianchao 350755cdd0afSWang Jianchao INIT_LIST_HEAD(&discard_list); 350855cdd0afSWang Jianchao spin_lock(&sbi->s_md_lock); 350955cdd0afSWang Jianchao list_splice_init(&sbi->s_discard_list, &discard_list); 351055cdd0afSWang Jianchao spin_unlock(&sbi->s_md_lock); 351155cdd0afSWang Jianchao 351255cdd0afSWang Jianchao load_grp = UINT_MAX; 351355cdd0afSWang Jianchao list_for_each_entry_safe(fd, nfd, &discard_list, efd_list) { 351455cdd0afSWang Jianchao /* 35155036ab8dSWang Jianchao * If filesystem is umounting or no memory or suffering 35165036ab8dSWang Jianchao * from no space, give up the discard 351755cdd0afSWang Jianchao */ 35185036ab8dSWang Jianchao if ((sb->s_flags & SB_ACTIVE) && !err && 35195036ab8dSWang Jianchao !atomic_read(&sbi->s_retry_alloc_pending)) { 352055cdd0afSWang Jianchao grp = fd->efd_group; 352155cdd0afSWang Jianchao if (grp != load_grp) { 352255cdd0afSWang Jianchao if (load_grp != UINT_MAX) 352355cdd0afSWang Jianchao ext4_mb_unload_buddy(&e4b); 352455cdd0afSWang Jianchao 352555cdd0afSWang Jianchao err = ext4_mb_load_buddy(sb, grp, &e4b); 352655cdd0afSWang Jianchao if (err) { 352755cdd0afSWang Jianchao kmem_cache_free(ext4_free_data_cachep, fd); 352855cdd0afSWang Jianchao load_grp = UINT_MAX; 352955cdd0afSWang Jianchao continue; 353055cdd0afSWang Jianchao } else { 353155cdd0afSWang Jianchao load_grp = grp; 353255cdd0afSWang Jianchao } 353355cdd0afSWang Jianchao } 353455cdd0afSWang Jianchao 353555cdd0afSWang Jianchao ext4_lock_group(sb, grp); 353655cdd0afSWang Jianchao ext4_try_to_trim_range(sb, &e4b, fd->efd_start_cluster, 353755cdd0afSWang Jianchao fd->efd_start_cluster + fd->efd_count - 1, 1); 353855cdd0afSWang Jianchao ext4_unlock_group(sb, grp); 353955cdd0afSWang Jianchao } 354055cdd0afSWang Jianchao kmem_cache_free(ext4_free_data_cachep, fd); 354155cdd0afSWang Jianchao } 354255cdd0afSWang Jianchao 354355cdd0afSWang Jianchao if (load_grp != UINT_MAX) 354455cdd0afSWang Jianchao ext4_mb_unload_buddy(&e4b); 354555cdd0afSWang Jianchao } 354655cdd0afSWang Jianchao 35479d99012fSAkira Fujita int ext4_mb_init(struct super_block *sb) 3548c9de560dSAlex Tomas { 3549c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 35506be2ded1SAneesh Kumar K.V unsigned i, j; 3551935244cdSNicolai Stange unsigned offset, offset_incr; 3552c9de560dSAlex Tomas unsigned max; 355374767c5aSShen Feng int ret; 3554c9de560dSAlex Tomas 35554b68f6dfSHarshad Shirwadkar i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets); 3556c9de560dSAlex Tomas 3557c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 3558c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 3559fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 3560fb1813f4SCurt Wohlgemuth goto out; 3561c9de560dSAlex Tomas } 3562ff7ef329SYasunori Goto 35634b68f6dfSHarshad Shirwadkar i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs); 3564c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 3565c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 3566fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 3567fb1813f4SCurt Wohlgemuth goto out; 3568fb1813f4SCurt Wohlgemuth } 3569fb1813f4SCurt Wohlgemuth 35702892c15dSEric Sandeen ret = ext4_groupinfo_create_slab(sb->s_blocksize); 35712892c15dSEric Sandeen if (ret < 0) 3572fb1813f4SCurt Wohlgemuth goto out; 3573c9de560dSAlex Tomas 3574c9de560dSAlex Tomas /* order 0 is regular bitmap */ 3575c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 3576c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 3577c9de560dSAlex Tomas 3578c9de560dSAlex Tomas i = 1; 3579c9de560dSAlex Tomas offset = 0; 3580935244cdSNicolai Stange offset_incr = 1 << (sb->s_blocksize_bits - 1); 3581c9de560dSAlex Tomas max = sb->s_blocksize << 2; 3582c9de560dSAlex Tomas do { 3583c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 3584c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 3585935244cdSNicolai Stange offset += offset_incr; 3586935244cdSNicolai Stange offset_incr = offset_incr >> 1; 3587c9de560dSAlex Tomas max = max >> 1; 3588c9de560dSAlex Tomas i++; 35894b68f6dfSHarshad Shirwadkar } while (i < MB_NUM_ORDERS(sb)); 35904b68f6dfSHarshad Shirwadkar 359183e80a6eSJan Kara sbi->s_mb_avg_fragment_size = 359283e80a6eSJan Kara kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head), 359383e80a6eSJan Kara GFP_KERNEL); 359483e80a6eSJan Kara if (!sbi->s_mb_avg_fragment_size) { 359583e80a6eSJan Kara ret = -ENOMEM; 359683e80a6eSJan Kara goto out; 359783e80a6eSJan Kara } 359883e80a6eSJan Kara sbi->s_mb_avg_fragment_size_locks = 359983e80a6eSJan Kara kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t), 360083e80a6eSJan Kara GFP_KERNEL); 360183e80a6eSJan Kara if (!sbi->s_mb_avg_fragment_size_locks) { 360283e80a6eSJan Kara ret = -ENOMEM; 360383e80a6eSJan Kara goto out; 360483e80a6eSJan Kara } 360583e80a6eSJan Kara for (i = 0; i < MB_NUM_ORDERS(sb); i++) { 360683e80a6eSJan Kara INIT_LIST_HEAD(&sbi->s_mb_avg_fragment_size[i]); 360783e80a6eSJan Kara rwlock_init(&sbi->s_mb_avg_fragment_size_locks[i]); 360883e80a6eSJan Kara } 3609196e402aSHarshad Shirwadkar sbi->s_mb_largest_free_orders = 3610196e402aSHarshad Shirwadkar kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head), 3611196e402aSHarshad Shirwadkar GFP_KERNEL); 3612196e402aSHarshad Shirwadkar if (!sbi->s_mb_largest_free_orders) { 3613196e402aSHarshad Shirwadkar ret = -ENOMEM; 3614196e402aSHarshad Shirwadkar goto out; 3615196e402aSHarshad Shirwadkar } 3616196e402aSHarshad Shirwadkar sbi->s_mb_largest_free_orders_locks = 3617196e402aSHarshad Shirwadkar kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t), 3618196e402aSHarshad Shirwadkar GFP_KERNEL); 3619196e402aSHarshad Shirwadkar if (!sbi->s_mb_largest_free_orders_locks) { 3620196e402aSHarshad Shirwadkar ret = -ENOMEM; 3621196e402aSHarshad Shirwadkar goto out; 3622196e402aSHarshad Shirwadkar } 3623196e402aSHarshad Shirwadkar for (i = 0; i < MB_NUM_ORDERS(sb); i++) { 3624196e402aSHarshad Shirwadkar INIT_LIST_HEAD(&sbi->s_mb_largest_free_orders[i]); 3625196e402aSHarshad Shirwadkar rwlock_init(&sbi->s_mb_largest_free_orders_locks[i]); 3626196e402aSHarshad Shirwadkar } 3627c9de560dSAlex Tomas 3628c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 3629d08854f5STheodore Ts'o sbi->s_mb_free_pending = 0; 3630a0154344SDaeho Jeong INIT_LIST_HEAD(&sbi->s_freed_data_list); 363155cdd0afSWang Jianchao INIT_LIST_HEAD(&sbi->s_discard_list); 363255cdd0afSWang Jianchao INIT_WORK(&sbi->s_discard_work, ext4_discard_work); 36335036ab8dSWang Jianchao atomic_set(&sbi->s_retry_alloc_pending, 0); 3634c9de560dSAlex Tomas 3635c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 3636c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 3637c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 3638c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 3639c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 3640f52f3d2bSOjaswin Mujoo sbi->s_mb_best_avail_max_trim_order = MB_DEFAULT_BEST_AVAIL_TRIM_ORDER; 36417e170922SOjaswin Mujoo 364227baebb8STheodore Ts'o /* 364327baebb8STheodore Ts'o * The default group preallocation is 512, which for 4k block 364427baebb8STheodore Ts'o * sizes translates to 2 megabytes. However for bigalloc file 364527baebb8STheodore Ts'o * systems, this is probably too big (i.e, if the cluster size 364627baebb8STheodore Ts'o * is 1 megabyte, then group preallocation size becomes half a 364727baebb8STheodore Ts'o * gigabyte!). As a default, we will keep a two megabyte 364827baebb8STheodore Ts'o * group pralloc size for cluster sizes up to 64k, and after 364927baebb8STheodore Ts'o * that, we will force a minimum group preallocation size of 365027baebb8STheodore Ts'o * 32 clusters. This translates to 8 megs when the cluster 365127baebb8STheodore Ts'o * size is 256k, and 32 megs when the cluster size is 1 meg, 365227baebb8STheodore Ts'o * which seems reasonable as a default. 365327baebb8STheodore Ts'o */ 365427baebb8STheodore Ts'o sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> 365527baebb8STheodore Ts'o sbi->s_cluster_bits, 32); 3656d7a1fee1SDan Ehrenberg /* 3657d7a1fee1SDan Ehrenberg * If there is a s_stripe > 1, then we set the s_mb_group_prealloc 3658d7a1fee1SDan Ehrenberg * to the lowest multiple of s_stripe which is bigger than 3659d7a1fee1SDan Ehrenberg * the s_mb_group_prealloc as determined above. We want 3660d7a1fee1SDan Ehrenberg * the preallocation size to be an exact multiple of the 3661d7a1fee1SDan Ehrenberg * RAID stripe size so that preallocations don't fragment 3662d7a1fee1SDan Ehrenberg * the stripes. 3663d7a1fee1SDan Ehrenberg */ 3664d7a1fee1SDan Ehrenberg if (sbi->s_stripe > 1) { 3665d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc = roundup( 3666c3defd99SKemeng Shi sbi->s_mb_group_prealloc, EXT4_B2C(sbi, sbi->s_stripe)); 3667d7a1fee1SDan Ehrenberg } 3668c9de560dSAlex Tomas 3669730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 3670c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 3671fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 3672029b10c5SAndrey Tsyvarev goto out; 3673c9de560dSAlex Tomas } 3674730c213cSEric Sandeen for_each_possible_cpu(i) { 3675c9de560dSAlex Tomas struct ext4_locality_group *lg; 3676730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 3677c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 36786be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 36796be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 3680c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 3681c9de560dSAlex Tomas } 3682c9de560dSAlex Tomas 368310f0d2a5SChristoph Hellwig if (bdev_nonrot(sb->s_bdev)) 3684196e402aSHarshad Shirwadkar sbi->s_mb_max_linear_groups = 0; 3685196e402aSHarshad Shirwadkar else 3686196e402aSHarshad Shirwadkar sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT; 368779a77c5aSYu Jian /* init file for buddy data */ 368879a77c5aSYu Jian ret = ext4_mb_init_backend(sb); 36897aa0baeaSTao Ma if (ret != 0) 36907aa0baeaSTao Ma goto out_free_locality_groups; 369179a77c5aSYu Jian 36927aa0baeaSTao Ma return 0; 36937aa0baeaSTao Ma 36947aa0baeaSTao Ma out_free_locality_groups: 36957aa0baeaSTao Ma free_percpu(sbi->s_locality_groups); 36967aa0baeaSTao Ma sbi->s_locality_groups = NULL; 3697fb1813f4SCurt Wohlgemuth out: 369883e80a6eSJan Kara kfree(sbi->s_mb_avg_fragment_size); 369983e80a6eSJan Kara kfree(sbi->s_mb_avg_fragment_size_locks); 3700196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders); 3701196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders_locks); 3702fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_offsets); 37037aa0baeaSTao Ma sbi->s_mb_offsets = NULL; 3704fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_maxs); 37057aa0baeaSTao Ma sbi->s_mb_maxs = NULL; 3706fb1813f4SCurt Wohlgemuth return ret; 3707c9de560dSAlex Tomas } 3708c9de560dSAlex Tomas 3709955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 3710d3df1453SRitesh Harjani static int ext4_mb_cleanup_pa(struct ext4_group_info *grp) 3711c9de560dSAlex Tomas { 3712c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3713c9de560dSAlex Tomas struct list_head *cur, *tmp; 3714c9de560dSAlex Tomas int count = 0; 3715c9de560dSAlex Tomas 3716c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 3717c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3718c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3719c9de560dSAlex Tomas count++; 3720688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 3721c9de560dSAlex Tomas } 3722d3df1453SRitesh Harjani return count; 3723c9de560dSAlex Tomas } 3724c9de560dSAlex Tomas 3725c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 3726c9de560dSAlex Tomas { 37278df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 3728c9de560dSAlex Tomas ext4_group_t i; 3729c9de560dSAlex Tomas int num_meta_group_infos; 3730df3da4eaSSuraj Jitindar Singh struct ext4_group_info *grinfo, ***group_info; 3731c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3732fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 3733d3df1453SRitesh Harjani int count; 3734c9de560dSAlex Tomas 373555cdd0afSWang Jianchao if (test_opt(sb, DISCARD)) { 373655cdd0afSWang Jianchao /* 373755cdd0afSWang Jianchao * wait the discard work to drain all of ext4_free_data 373855cdd0afSWang Jianchao */ 373955cdd0afSWang Jianchao flush_work(&sbi->s_discard_work); 374055cdd0afSWang Jianchao WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); 374155cdd0afSWang Jianchao } 374255cdd0afSWang Jianchao 3743c9de560dSAlex Tomas if (sbi->s_group_info) { 37448df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 37454b99faa2SKhazhismel Kumykov cond_resched(); 3746c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 37475354b2afSTheodore Ts'o if (!grinfo) 37485354b2afSTheodore Ts'o continue; 3749a3450215SRitesh Harjani mb_group_bb_bitmap_free(grinfo); 3750c9de560dSAlex Tomas ext4_lock_group(sb, i); 3751d3df1453SRitesh Harjani count = ext4_mb_cleanup_pa(grinfo); 3752d3df1453SRitesh Harjani if (count) 3753d3df1453SRitesh Harjani mb_debug(sb, "mballoc: %d PAs left\n", 3754d3df1453SRitesh Harjani count); 3755c9de560dSAlex Tomas ext4_unlock_group(sb, i); 3756fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, grinfo); 3757c9de560dSAlex Tomas } 37588df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 3759c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 3760c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 3761df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3762df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 3763c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 3764df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 3765df3da4eaSSuraj Jitindar Singh kvfree(group_info); 3766df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3767c9de560dSAlex Tomas } 376883e80a6eSJan Kara kfree(sbi->s_mb_avg_fragment_size); 376983e80a6eSJan Kara kfree(sbi->s_mb_avg_fragment_size_locks); 3770196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders); 3771196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders_locks); 3772c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 3773c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 3774c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 3775c9de560dSAlex Tomas if (sbi->s_mb_stats) { 37769d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 37779d8b9ec4STheodore Ts'o "mballoc: %u blocks %u reqs (%u success)", 3778c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 3779c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 3780c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 37819d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 3782a6c75eafSHarshad Shirwadkar "mballoc: %u extents scanned, %u groups scanned, %u goal hits, " 37839d8b9ec4STheodore Ts'o "%u 2^N hits, %u breaks, %u lost", 3784c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 3785a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_bal_groups_scanned), 3786c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 3787c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 3788c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 3789c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 37909d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 379167d25186SHarshad Shirwadkar "mballoc: %u generated and it took %llu", 379267d25186SHarshad Shirwadkar atomic_read(&sbi->s_mb_buddies_generated), 379367d25186SHarshad Shirwadkar atomic64_read(&sbi->s_mb_generation_time)); 37949d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 37959d8b9ec4STheodore Ts'o "mballoc: %u preallocated, %u discarded", 3796c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 3797c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 3798c9de560dSAlex Tomas } 3799c9de560dSAlex Tomas 3800730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 3801c9de560dSAlex Tomas 3802c9de560dSAlex Tomas return 0; 3803c9de560dSAlex Tomas } 3804c9de560dSAlex Tomas 380577ca6cdfSLukas Czerner static inline int ext4_issue_discard(struct super_block *sb, 3806a0154344SDaeho Jeong ext4_group_t block_group, ext4_grpblk_t cluster, int count, 3807a0154344SDaeho Jeong struct bio **biop) 38085c521830SJiaying Zhang { 38095c521830SJiaying Zhang ext4_fsblk_t discard_block; 38105c521830SJiaying Zhang 381184130193STheodore Ts'o discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) + 381284130193STheodore Ts'o ext4_group_first_block_no(sb, block_group)); 381384130193STheodore Ts'o count = EXT4_C2B(EXT4_SB(sb), count); 38145c521830SJiaying Zhang trace_ext4_discard_blocks(sb, 38155c521830SJiaying Zhang (unsigned long long) discard_block, count); 3816a0154344SDaeho Jeong if (biop) { 3817a0154344SDaeho Jeong return __blkdev_issue_discard(sb->s_bdev, 3818a0154344SDaeho Jeong (sector_t)discard_block << (sb->s_blocksize_bits - 9), 3819a0154344SDaeho Jeong (sector_t)count << (sb->s_blocksize_bits - 9), 382044abff2cSChristoph Hellwig GFP_NOFS, biop); 3821a0154344SDaeho Jeong } else 382293259636SLukas Czerner return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); 38235c521830SJiaying Zhang } 38245c521830SJiaying Zhang 3825a0154344SDaeho Jeong static void ext4_free_data_in_buddy(struct super_block *sb, 3826a0154344SDaeho Jeong struct ext4_free_data *entry) 3827c9de560dSAlex Tomas { 3828c9de560dSAlex Tomas struct ext4_buddy e4b; 3829c894058dSAneesh Kumar K.V struct ext4_group_info *db; 3830c7f2bafaSKemeng Shi int err, count = 0; 3831c9de560dSAlex Tomas 3832d3df1453SRitesh Harjani mb_debug(sb, "gonna free %u blocks in group %u (0x%p):", 383318aadd47SBobi Jam entry->efd_count, entry->efd_group, entry); 3834c9de560dSAlex Tomas 383518aadd47SBobi Jam err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); 3836c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 3837c9de560dSAlex Tomas BUG_ON(err != 0); 3838c9de560dSAlex Tomas 3839d08854f5STheodore Ts'o spin_lock(&EXT4_SB(sb)->s_md_lock); 3840d08854f5STheodore Ts'o EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count; 3841d08854f5STheodore Ts'o spin_unlock(&EXT4_SB(sb)->s_md_lock); 384218aadd47SBobi Jam 3843c894058dSAneesh Kumar K.V db = e4b.bd_info; 3844c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 384518aadd47SBobi Jam count += entry->efd_count; 384618aadd47SBobi Jam ext4_lock_group(sb, entry->efd_group); 3847c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 384818aadd47SBobi Jam rb_erase(&entry->efd_node, &(db->bb_free_root)); 384918aadd47SBobi Jam mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); 3850c9de560dSAlex Tomas 38513d56b8d2STao Ma /* 38523d56b8d2STao Ma * Clear the trimmed flag for the group so that the next 38533d56b8d2STao Ma * ext4_trim_fs can trim it. 38543d56b8d2STao Ma * If the volume is mounted with -o discard, online discard 38553d56b8d2STao Ma * is supported and the free blocks will be trimmed online. 38563d56b8d2STao Ma */ 38573d56b8d2STao Ma if (!test_opt(sb, DISCARD)) 38583d56b8d2STao Ma EXT4_MB_GRP_CLEAR_TRIMMED(db); 38593d56b8d2STao Ma 3860c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 3861c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 3862c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 3863c894058dSAneesh Kumar K.V */ 386409cbfeafSKirill A. Shutemov put_page(e4b.bd_buddy_page); 386509cbfeafSKirill A. Shutemov put_page(e4b.bd_bitmap_page); 3866c894058dSAneesh Kumar K.V } 386718aadd47SBobi Jam ext4_unlock_group(sb, entry->efd_group); 3868e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3869c9de560dSAlex Tomas 3870c7f2bafaSKemeng Shi mb_debug(sb, "freed %d blocks in 1 structures\n", count); 3871c9de560dSAlex Tomas } 3872c9de560dSAlex Tomas 3873a0154344SDaeho Jeong /* 3874a0154344SDaeho Jeong * This function is called by the jbd2 layer once the commit has finished, 3875a0154344SDaeho Jeong * so we know we can free the blocks that were released with that commit. 3876a0154344SDaeho Jeong */ 3877a0154344SDaeho Jeong void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid) 3878a0154344SDaeho Jeong { 3879a0154344SDaeho Jeong struct ext4_sb_info *sbi = EXT4_SB(sb); 3880a0154344SDaeho Jeong struct ext4_free_data *entry, *tmp; 3881a0154344SDaeho Jeong struct list_head freed_data_list; 3882a0154344SDaeho Jeong struct list_head *cut_pos = NULL; 388355cdd0afSWang Jianchao bool wake; 3884a0154344SDaeho Jeong 3885a0154344SDaeho Jeong INIT_LIST_HEAD(&freed_data_list); 3886a0154344SDaeho Jeong 3887a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 3888a0154344SDaeho Jeong list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) { 3889a0154344SDaeho Jeong if (entry->efd_tid != commit_tid) 3890a0154344SDaeho Jeong break; 3891a0154344SDaeho Jeong cut_pos = &entry->efd_list; 3892a0154344SDaeho Jeong } 3893a0154344SDaeho Jeong if (cut_pos) 3894a0154344SDaeho Jeong list_cut_position(&freed_data_list, &sbi->s_freed_data_list, 3895a0154344SDaeho Jeong cut_pos); 3896a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 3897a0154344SDaeho Jeong 389855cdd0afSWang Jianchao list_for_each_entry(entry, &freed_data_list, efd_list) 3899a0154344SDaeho Jeong ext4_free_data_in_buddy(sb, entry); 390055cdd0afSWang Jianchao 390155cdd0afSWang Jianchao if (test_opt(sb, DISCARD)) { 390255cdd0afSWang Jianchao spin_lock(&sbi->s_md_lock); 390355cdd0afSWang Jianchao wake = list_empty(&sbi->s_discard_list); 390455cdd0afSWang Jianchao list_splice_tail(&freed_data_list, &sbi->s_discard_list); 390555cdd0afSWang Jianchao spin_unlock(&sbi->s_md_lock); 390655cdd0afSWang Jianchao if (wake) 390755cdd0afSWang Jianchao queue_work(system_unbound_wq, &sbi->s_discard_work); 390855cdd0afSWang Jianchao } else { 390955cdd0afSWang Jianchao list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list) 391055cdd0afSWang Jianchao kmem_cache_free(ext4_free_data_cachep, entry); 391155cdd0afSWang Jianchao } 3912a0154344SDaeho Jeong } 3913a0154344SDaeho Jeong 39145dabfc78STheodore Ts'o int __init ext4_init_mballoc(void) 3915c9de560dSAlex Tomas { 391616828088STheodore Ts'o ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, 391716828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3918c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 3919f283529aSRitesh Harjani goto out; 3920c9de560dSAlex Tomas 392116828088STheodore Ts'o ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, 392216828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3923f283529aSRitesh Harjani if (ext4_ac_cachep == NULL) 3924f283529aSRitesh Harjani goto out_pa_free; 3925c894058dSAneesh Kumar K.V 392618aadd47SBobi Jam ext4_free_data_cachep = KMEM_CACHE(ext4_free_data, 392716828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3928f283529aSRitesh Harjani if (ext4_free_data_cachep == NULL) 3929f283529aSRitesh Harjani goto out_ac_free; 3930f283529aSRitesh Harjani 3931c9de560dSAlex Tomas return 0; 3932f283529aSRitesh Harjani 3933f283529aSRitesh Harjani out_ac_free: 3934f283529aSRitesh Harjani kmem_cache_destroy(ext4_ac_cachep); 3935f283529aSRitesh Harjani out_pa_free: 3936f283529aSRitesh Harjani kmem_cache_destroy(ext4_pspace_cachep); 3937f283529aSRitesh Harjani out: 3938f283529aSRitesh Harjani return -ENOMEM; 3939c9de560dSAlex Tomas } 3940c9de560dSAlex Tomas 39415dabfc78STheodore Ts'o void ext4_exit_mballoc(void) 3942c9de560dSAlex Tomas { 39433e03f9caSJesper Dangaard Brouer /* 39443e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 39453e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 39463e03f9caSJesper Dangaard Brouer */ 39473e03f9caSJesper Dangaard Brouer rcu_barrier(); 3948c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 3949256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 395018aadd47SBobi Jam kmem_cache_destroy(ext4_free_data_cachep); 39512892c15dSEric Sandeen ext4_groupinfo_destroy_slabs(); 3952c9de560dSAlex Tomas } 3953c9de560dSAlex Tomas 3954c9de560dSAlex Tomas 3955c9de560dSAlex Tomas /* 395673b2c716SUwe Kleine-König * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps 3957c9de560dSAlex Tomas * Returns 0 if success or error code 3958c9de560dSAlex Tomas */ 39594ddfef7bSEric Sandeen static noinline_for_stack int 39604ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 396153accfa9STheodore Ts'o handle_t *handle, unsigned int reserv_clstrs) 3962c9de560dSAlex Tomas { 3963c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3964c9de560dSAlex Tomas struct ext4_group_desc *gdp; 3965c9de560dSAlex Tomas struct buffer_head *gdp_bh; 3966c9de560dSAlex Tomas struct ext4_sb_info *sbi; 3967c9de560dSAlex Tomas struct super_block *sb; 3968c9de560dSAlex Tomas ext4_fsblk_t block; 3969519deca0SAneesh Kumar K.V int err, len; 3970c9de560dSAlex Tomas 3971c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3972c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 3973c9de560dSAlex Tomas 3974c9de560dSAlex Tomas sb = ac->ac_sb; 3975c9de560dSAlex Tomas sbi = EXT4_SB(sb); 3976c9de560dSAlex Tomas 3977574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 39789008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 3979fb28f9ceSKemeng Shi return PTR_ERR(bitmap_bh); 39809008a58eSDarrick J. Wong } 3981c9de560dSAlex Tomas 39825d601255Sliang xie BUFFER_TRACE(bitmap_bh, "getting write access"); 3983188c299eSJan Kara err = ext4_journal_get_write_access(handle, sb, bitmap_bh, 3984188c299eSJan Kara EXT4_JTR_NONE); 3985c9de560dSAlex Tomas if (err) 3986c9de560dSAlex Tomas goto out_err; 3987c9de560dSAlex Tomas 3988c9de560dSAlex Tomas err = -EIO; 3989c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 3990c9de560dSAlex Tomas if (!gdp) 3991c9de560dSAlex Tomas goto out_err; 3992c9de560dSAlex Tomas 3993a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 3994021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, gdp)); 399503cddb80SAneesh Kumar K.V 39965d601255Sliang xie BUFFER_TRACE(gdp_bh, "get_write_access"); 3997188c299eSJan Kara err = ext4_journal_get_write_access(handle, sb, gdp_bh, EXT4_JTR_NONE); 3998c9de560dSAlex Tomas if (err) 3999c9de560dSAlex Tomas goto out_err; 4000c9de560dSAlex Tomas 4001bda00de7SAkinobu Mita block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4002c9de560dSAlex Tomas 400353accfa9STheodore Ts'o len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 4004ce9f24ccSJan Kara if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { 400512062dddSEric Sandeen ext4_error(sb, "Allocating blocks %llu-%llu which overlap " 40061084f252STheodore Ts'o "fs metadata", block, block+len); 4007519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 4008554a5cccSVegard Nossum * Fix the bitmap and return EFSCORRUPTED 4009519deca0SAneesh Kumar K.V * We leak some of the blocks here. 4010519deca0SAneesh Kumar K.V */ 4011955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 4012123e3016SRitesh Harjani mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 4013519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 4014955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 40150390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 4016519deca0SAneesh Kumar K.V if (!err) 4017554a5cccSVegard Nossum err = -EFSCORRUPTED; 4018519deca0SAneesh Kumar K.V goto out_err; 4019c9de560dSAlex Tomas } 4020955ce5f5SAneesh Kumar K.V 4021955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 4022c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 4023c9de560dSAlex Tomas { 4024c9de560dSAlex Tomas int i; 4025c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 4026c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 4027c9de560dSAlex Tomas bitmap_bh->b_data)); 4028c9de560dSAlex Tomas } 4029c9de560dSAlex Tomas } 4030c9de560dSAlex Tomas #endif 4031123e3016SRitesh Harjani mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 4032c3e94d1dSYongqiang Yang ac->ac_b_ex.fe_len); 40338844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 40348844618dSTheodore Ts'o (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 4035c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 4036021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, 4037cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, 4038560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 4039c9de560dSAlex Tomas } 4040021b65bbSTheodore Ts'o len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; 4041021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, len); 40421df9bde4SKemeng Shi ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh); 4043feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); 4044955ce5f5SAneesh Kumar K.V 4045955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 404657042651STheodore Ts'o percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); 4047d2a17637SMingming Cao /* 40486bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 4049d2a17637SMingming Cao */ 40506bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 40516bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 405257042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 405357042651STheodore Ts'o reserv_clstrs); 4054c9de560dSAlex Tomas 4055772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 4056772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 4057772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 405890ba983fSTheodore Ts'o atomic64_sub(ac->ac_b_ex.fe_len, 40597c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 40607c990728SSuraj Jitindar Singh flex_group)->free_clusters); 4061772cb7c8SJose R. Santos } 4062772cb7c8SJose R. Santos 40630390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 4064c9de560dSAlex Tomas if (err) 4065c9de560dSAlex Tomas goto out_err; 40660390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 4067c9de560dSAlex Tomas 4068c9de560dSAlex Tomas out_err: 406942a10addSAneesh Kumar K.V brelse(bitmap_bh); 4070c9de560dSAlex Tomas return err; 4071c9de560dSAlex Tomas } 4072c9de560dSAlex Tomas 4073c9de560dSAlex Tomas /* 40748016e29fSHarshad Shirwadkar * Idempotent helper for Ext4 fast commit replay path to set the state of 40758016e29fSHarshad Shirwadkar * blocks in bitmaps and update counters. 40768016e29fSHarshad Shirwadkar */ 40778016e29fSHarshad Shirwadkar void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block, 40788016e29fSHarshad Shirwadkar int len, int state) 40798016e29fSHarshad Shirwadkar { 40808016e29fSHarshad Shirwadkar struct buffer_head *bitmap_bh = NULL; 40818016e29fSHarshad Shirwadkar struct ext4_group_desc *gdp; 40828016e29fSHarshad Shirwadkar struct buffer_head *gdp_bh; 40838016e29fSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 40848016e29fSHarshad Shirwadkar ext4_group_t group; 40858016e29fSHarshad Shirwadkar ext4_grpblk_t blkoff; 4086a5c0e2fdSRitesh Harjani int i, err; 40878016e29fSHarshad Shirwadkar int already; 4088bfdc502aSRitesh Harjani unsigned int clen, clen_changed, thisgrp_len; 40898016e29fSHarshad Shirwadkar 4090bfdc502aSRitesh Harjani while (len > 0) { 40918016e29fSHarshad Shirwadkar ext4_get_group_no_and_offset(sb, block, &group, &blkoff); 4092bfdc502aSRitesh Harjani 4093bfdc502aSRitesh Harjani /* 4094bfdc502aSRitesh Harjani * Check to see if we are freeing blocks across a group 4095bfdc502aSRitesh Harjani * boundary. 4096bfdc502aSRitesh Harjani * In case of flex_bg, this can happen that (block, len) may 4097bfdc502aSRitesh Harjani * span across more than one group. In that case we need to 4098bfdc502aSRitesh Harjani * get the corresponding group metadata to work with. 4099bfdc502aSRitesh Harjani * For this we have goto again loop. 4100bfdc502aSRitesh Harjani */ 4101bfdc502aSRitesh Harjani thisgrp_len = min_t(unsigned int, (unsigned int)len, 4102bfdc502aSRitesh Harjani EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff)); 4103bfdc502aSRitesh Harjani clen = EXT4_NUM_B2C(sbi, thisgrp_len); 4104bfdc502aSRitesh Harjani 41058c91c579SRitesh Harjani if (!ext4_sb_block_valid(sb, NULL, block, thisgrp_len)) { 41068c91c579SRitesh Harjani ext4_error(sb, "Marking blocks in system zone - " 41078c91c579SRitesh Harjani "Block = %llu, len = %u", 41088c91c579SRitesh Harjani block, thisgrp_len); 41098c91c579SRitesh Harjani bitmap_bh = NULL; 41108c91c579SRitesh Harjani break; 41118c91c579SRitesh Harjani } 41128c91c579SRitesh Harjani 41138016e29fSHarshad Shirwadkar bitmap_bh = ext4_read_block_bitmap(sb, group); 41148016e29fSHarshad Shirwadkar if (IS_ERR(bitmap_bh)) { 41158016e29fSHarshad Shirwadkar err = PTR_ERR(bitmap_bh); 41168016e29fSHarshad Shirwadkar bitmap_bh = NULL; 4117bfdc502aSRitesh Harjani break; 41188016e29fSHarshad Shirwadkar } 41198016e29fSHarshad Shirwadkar 41208016e29fSHarshad Shirwadkar err = -EIO; 41218016e29fSHarshad Shirwadkar gdp = ext4_get_group_desc(sb, group, &gdp_bh); 41228016e29fSHarshad Shirwadkar if (!gdp) 4123bfdc502aSRitesh Harjani break; 41248016e29fSHarshad Shirwadkar 41258016e29fSHarshad Shirwadkar ext4_lock_group(sb, group); 41268016e29fSHarshad Shirwadkar already = 0; 41278016e29fSHarshad Shirwadkar for (i = 0; i < clen; i++) 4128bfdc502aSRitesh Harjani if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) == 4129bfdc502aSRitesh Harjani !state) 41308016e29fSHarshad Shirwadkar already++; 41318016e29fSHarshad Shirwadkar 4132a5c0e2fdSRitesh Harjani clen_changed = clen - already; 41338016e29fSHarshad Shirwadkar if (state) 4134123e3016SRitesh Harjani mb_set_bits(bitmap_bh->b_data, blkoff, clen); 41358016e29fSHarshad Shirwadkar else 4136bd8247eeSRitesh Harjani mb_clear_bits(bitmap_bh->b_data, blkoff, clen); 41378016e29fSHarshad Shirwadkar if (ext4_has_group_desc_csum(sb) && 41388016e29fSHarshad Shirwadkar (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 41398016e29fSHarshad Shirwadkar gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 41408016e29fSHarshad Shirwadkar ext4_free_group_clusters_set(sb, gdp, 4141bfdc502aSRitesh Harjani ext4_free_clusters_after_init(sb, group, gdp)); 41428016e29fSHarshad Shirwadkar } 41438016e29fSHarshad Shirwadkar if (state) 4144a5c0e2fdSRitesh Harjani clen = ext4_free_group_clusters(sb, gdp) - clen_changed; 41458016e29fSHarshad Shirwadkar else 4146a5c0e2fdSRitesh Harjani clen = ext4_free_group_clusters(sb, gdp) + clen_changed; 41478016e29fSHarshad Shirwadkar 41488016e29fSHarshad Shirwadkar ext4_free_group_clusters_set(sb, gdp, clen); 41491df9bde4SKemeng Shi ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh); 41508016e29fSHarshad Shirwadkar ext4_group_desc_csum_set(sb, group, gdp); 41518016e29fSHarshad Shirwadkar 41528016e29fSHarshad Shirwadkar ext4_unlock_group(sb, group); 41538016e29fSHarshad Shirwadkar 41548016e29fSHarshad Shirwadkar if (sbi->s_log_groups_per_flex) { 41558016e29fSHarshad Shirwadkar ext4_group_t flex_group = ext4_flex_group(sbi, group); 4156a5c0e2fdSRitesh Harjani struct flex_groups *fg = sbi_array_rcu_deref(sbi, 4157a5c0e2fdSRitesh Harjani s_flex_groups, flex_group); 41588016e29fSHarshad Shirwadkar 4159a5c0e2fdSRitesh Harjani if (state) 4160a5c0e2fdSRitesh Harjani atomic64_sub(clen_changed, &fg->free_clusters); 4161a5c0e2fdSRitesh Harjani else 4162a5c0e2fdSRitesh Harjani atomic64_add(clen_changed, &fg->free_clusters); 4163bfdc502aSRitesh Harjani 41648016e29fSHarshad Shirwadkar } 41658016e29fSHarshad Shirwadkar 41668016e29fSHarshad Shirwadkar err = ext4_handle_dirty_metadata(NULL, NULL, bitmap_bh); 41678016e29fSHarshad Shirwadkar if (err) 4168bfdc502aSRitesh Harjani break; 41698016e29fSHarshad Shirwadkar sync_dirty_buffer(bitmap_bh); 41708016e29fSHarshad Shirwadkar err = ext4_handle_dirty_metadata(NULL, NULL, gdp_bh); 41718016e29fSHarshad Shirwadkar sync_dirty_buffer(gdp_bh); 4172bfdc502aSRitesh Harjani if (err) 4173bfdc502aSRitesh Harjani break; 41748016e29fSHarshad Shirwadkar 4175bfdc502aSRitesh Harjani block += thisgrp_len; 4176bfdc502aSRitesh Harjani len -= thisgrp_len; 4177bfdc502aSRitesh Harjani brelse(bitmap_bh); 4178bfdc502aSRitesh Harjani BUG_ON(len < 0); 4179bfdc502aSRitesh Harjani } 4180bfdc502aSRitesh Harjani 4181bfdc502aSRitesh Harjani if (err) 41828016e29fSHarshad Shirwadkar brelse(bitmap_bh); 41838016e29fSHarshad Shirwadkar } 41848016e29fSHarshad Shirwadkar 41858016e29fSHarshad Shirwadkar /* 4186c9de560dSAlex Tomas * here we normalize request for locality group 4187d7a1fee1SDan Ehrenberg * Group request are normalized to s_mb_group_prealloc, which goes to 4188d7a1fee1SDan Ehrenberg * s_strip if we set the same via mount option. 4189d7a1fee1SDan Ehrenberg * s_mb_group_prealloc can be configured via 4190b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 4191c9de560dSAlex Tomas * 4192c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 4193c9de560dSAlex Tomas */ 4194c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 4195c9de560dSAlex Tomas { 4196c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 4197c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 4198c9de560dSAlex Tomas 4199c9de560dSAlex Tomas BUG_ON(lg == NULL); 4200c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 4201d3df1453SRitesh Harjani mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); 4202c9de560dSAlex Tomas } 4203c9de560dSAlex Tomas 420438727786SOjaswin Mujoo /* 420538727786SOjaswin Mujoo * This function returns the next element to look at during inode 420638727786SOjaswin Mujoo * PA rbtree walk. We assume that we have held the inode PA rbtree lock 420738727786SOjaswin Mujoo * (ei->i_prealloc_lock) 420838727786SOjaswin Mujoo * 420938727786SOjaswin Mujoo * new_start The start of the range we want to compare 421038727786SOjaswin Mujoo * cur_start The existing start that we are comparing against 421138727786SOjaswin Mujoo * node The node of the rb_tree 421238727786SOjaswin Mujoo */ 421338727786SOjaswin Mujoo static inline struct rb_node* 421438727786SOjaswin Mujoo ext4_mb_pa_rb_next_iter(ext4_lblk_t new_start, ext4_lblk_t cur_start, struct rb_node *node) 421538727786SOjaswin Mujoo { 421638727786SOjaswin Mujoo if (new_start < cur_start) 421738727786SOjaswin Mujoo return node->rb_left; 421838727786SOjaswin Mujoo else 421938727786SOjaswin Mujoo return node->rb_right; 422038727786SOjaswin Mujoo } 422138727786SOjaswin Mujoo 42227692094aSOjaswin Mujoo static inline void 42237692094aSOjaswin Mujoo ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac, 4224bedc5d34SBaokun Li ext4_lblk_t start, loff_t end) 42257692094aSOjaswin Mujoo { 42267692094aSOjaswin Mujoo struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 42277692094aSOjaswin Mujoo struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 42287692094aSOjaswin Mujoo struct ext4_prealloc_space *tmp_pa; 4229bedc5d34SBaokun Li ext4_lblk_t tmp_pa_start; 4230bedc5d34SBaokun Li loff_t tmp_pa_end; 423138727786SOjaswin Mujoo struct rb_node *iter; 42327692094aSOjaswin Mujoo 423338727786SOjaswin Mujoo read_lock(&ei->i_prealloc_lock); 423438727786SOjaswin Mujoo for (iter = ei->i_prealloc_node.rb_node; iter; 423538727786SOjaswin Mujoo iter = ext4_mb_pa_rb_next_iter(start, tmp_pa_start, iter)) { 423638727786SOjaswin Mujoo tmp_pa = rb_entry(iter, struct ext4_prealloc_space, 423738727786SOjaswin Mujoo pa_node.inode_node); 42387692094aSOjaswin Mujoo tmp_pa_start = tmp_pa->pa_lstart; 4239bedc5d34SBaokun Li tmp_pa_end = pa_logical_end(sbi, tmp_pa); 42407692094aSOjaswin Mujoo 424138727786SOjaswin Mujoo spin_lock(&tmp_pa->pa_lock); 424238727786SOjaswin Mujoo if (tmp_pa->pa_deleted == 0) 42437692094aSOjaswin Mujoo BUG_ON(!(start >= tmp_pa_end || end <= tmp_pa_start)); 42447692094aSOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 42457692094aSOjaswin Mujoo } 424638727786SOjaswin Mujoo read_unlock(&ei->i_prealloc_lock); 42477692094aSOjaswin Mujoo } 42487692094aSOjaswin Mujoo 4249c9de560dSAlex Tomas /* 42500830344cSOjaswin Mujoo * Given an allocation context "ac" and a range "start", "end", check 42510830344cSOjaswin Mujoo * and adjust boundaries if the range overlaps with any of the existing 42520830344cSOjaswin Mujoo * preallocatoins stored in the corresponding inode of the allocation context. 42530830344cSOjaswin Mujoo * 42540830344cSOjaswin Mujoo * Parameters: 42550830344cSOjaswin Mujoo * ac allocation context 42560830344cSOjaswin Mujoo * start start of the new range 42570830344cSOjaswin Mujoo * end end of the new range 42580830344cSOjaswin Mujoo */ 42590830344cSOjaswin Mujoo static inline void 42600830344cSOjaswin Mujoo ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac, 4261bedc5d34SBaokun Li ext4_lblk_t *start, loff_t *end) 42620830344cSOjaswin Mujoo { 42630830344cSOjaswin Mujoo struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 42640830344cSOjaswin Mujoo struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 426538727786SOjaswin Mujoo struct ext4_prealloc_space *tmp_pa = NULL, *left_pa = NULL, *right_pa = NULL; 426638727786SOjaswin Mujoo struct rb_node *iter; 4267bedc5d34SBaokun Li ext4_lblk_t new_start, tmp_pa_start, right_pa_start = -1; 4268bedc5d34SBaokun Li loff_t new_end, tmp_pa_end, left_pa_end = -1; 42690830344cSOjaswin Mujoo 42700830344cSOjaswin Mujoo new_start = *start; 42710830344cSOjaswin Mujoo new_end = *end; 42720830344cSOjaswin Mujoo 427338727786SOjaswin Mujoo /* 427438727786SOjaswin Mujoo * Adjust the normalized range so that it doesn't overlap with any 427538727786SOjaswin Mujoo * existing preallocated blocks(PAs). Make sure to hold the rbtree lock 427638727786SOjaswin Mujoo * so it doesn't change underneath us. 427738727786SOjaswin Mujoo */ 427838727786SOjaswin Mujoo read_lock(&ei->i_prealloc_lock); 42790830344cSOjaswin Mujoo 428038727786SOjaswin Mujoo /* Step 1: find any one immediate neighboring PA of the normalized range */ 428138727786SOjaswin Mujoo for (iter = ei->i_prealloc_node.rb_node; iter; 428238727786SOjaswin Mujoo iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, 428338727786SOjaswin Mujoo tmp_pa_start, iter)) { 428438727786SOjaswin Mujoo tmp_pa = rb_entry(iter, struct ext4_prealloc_space, 428538727786SOjaswin Mujoo pa_node.inode_node); 42860830344cSOjaswin Mujoo tmp_pa_start = tmp_pa->pa_lstart; 4287bedc5d34SBaokun Li tmp_pa_end = pa_logical_end(sbi, tmp_pa); 42880830344cSOjaswin Mujoo 42890830344cSOjaswin Mujoo /* PA must not overlap original request */ 429038727786SOjaswin Mujoo spin_lock(&tmp_pa->pa_lock); 429138727786SOjaswin Mujoo if (tmp_pa->pa_deleted == 0) 42920830344cSOjaswin Mujoo BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end || 42930830344cSOjaswin Mujoo ac->ac_o_ex.fe_logical < tmp_pa_start)); 42940830344cSOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 42950830344cSOjaswin Mujoo } 42960830344cSOjaswin Mujoo 429738727786SOjaswin Mujoo /* 429838727786SOjaswin Mujoo * Step 2: check if the found PA is left or right neighbor and 429938727786SOjaswin Mujoo * get the other neighbor 430038727786SOjaswin Mujoo */ 430138727786SOjaswin Mujoo if (tmp_pa) { 430238727786SOjaswin Mujoo if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) { 430338727786SOjaswin Mujoo struct rb_node *tmp; 430438727786SOjaswin Mujoo 430538727786SOjaswin Mujoo left_pa = tmp_pa; 430638727786SOjaswin Mujoo tmp = rb_next(&left_pa->pa_node.inode_node); 430738727786SOjaswin Mujoo if (tmp) { 430838727786SOjaswin Mujoo right_pa = rb_entry(tmp, 430938727786SOjaswin Mujoo struct ext4_prealloc_space, 431038727786SOjaswin Mujoo pa_node.inode_node); 431138727786SOjaswin Mujoo } 431238727786SOjaswin Mujoo } else { 431338727786SOjaswin Mujoo struct rb_node *tmp; 431438727786SOjaswin Mujoo 431538727786SOjaswin Mujoo right_pa = tmp_pa; 431638727786SOjaswin Mujoo tmp = rb_prev(&right_pa->pa_node.inode_node); 431738727786SOjaswin Mujoo if (tmp) { 431838727786SOjaswin Mujoo left_pa = rb_entry(tmp, 431938727786SOjaswin Mujoo struct ext4_prealloc_space, 432038727786SOjaswin Mujoo pa_node.inode_node); 432138727786SOjaswin Mujoo } 432238727786SOjaswin Mujoo } 432338727786SOjaswin Mujoo } 432438727786SOjaswin Mujoo 432538727786SOjaswin Mujoo /* Step 3: get the non deleted neighbors */ 432638727786SOjaswin Mujoo if (left_pa) { 432738727786SOjaswin Mujoo for (iter = &left_pa->pa_node.inode_node;; 432838727786SOjaswin Mujoo iter = rb_prev(iter)) { 432938727786SOjaswin Mujoo if (!iter) { 433038727786SOjaswin Mujoo left_pa = NULL; 433138727786SOjaswin Mujoo break; 433238727786SOjaswin Mujoo } 433338727786SOjaswin Mujoo 433438727786SOjaswin Mujoo tmp_pa = rb_entry(iter, struct ext4_prealloc_space, 433538727786SOjaswin Mujoo pa_node.inode_node); 433638727786SOjaswin Mujoo left_pa = tmp_pa; 433738727786SOjaswin Mujoo spin_lock(&tmp_pa->pa_lock); 433838727786SOjaswin Mujoo if (tmp_pa->pa_deleted == 0) { 433938727786SOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 434038727786SOjaswin Mujoo break; 43410830344cSOjaswin Mujoo } 43420830344cSOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 43430830344cSOjaswin Mujoo } 434438727786SOjaswin Mujoo } 434538727786SOjaswin Mujoo 434638727786SOjaswin Mujoo if (right_pa) { 434738727786SOjaswin Mujoo for (iter = &right_pa->pa_node.inode_node;; 434838727786SOjaswin Mujoo iter = rb_next(iter)) { 434938727786SOjaswin Mujoo if (!iter) { 435038727786SOjaswin Mujoo right_pa = NULL; 435138727786SOjaswin Mujoo break; 435238727786SOjaswin Mujoo } 435338727786SOjaswin Mujoo 435438727786SOjaswin Mujoo tmp_pa = rb_entry(iter, struct ext4_prealloc_space, 435538727786SOjaswin Mujoo pa_node.inode_node); 435638727786SOjaswin Mujoo right_pa = tmp_pa; 435738727786SOjaswin Mujoo spin_lock(&tmp_pa->pa_lock); 435838727786SOjaswin Mujoo if (tmp_pa->pa_deleted == 0) { 435938727786SOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 436038727786SOjaswin Mujoo break; 436138727786SOjaswin Mujoo } 436238727786SOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 436338727786SOjaswin Mujoo } 436438727786SOjaswin Mujoo } 436538727786SOjaswin Mujoo 436638727786SOjaswin Mujoo if (left_pa) { 4367bedc5d34SBaokun Li left_pa_end = pa_logical_end(sbi, left_pa); 436838727786SOjaswin Mujoo BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical); 436938727786SOjaswin Mujoo } 437038727786SOjaswin Mujoo 437138727786SOjaswin Mujoo if (right_pa) { 437238727786SOjaswin Mujoo right_pa_start = right_pa->pa_lstart; 437338727786SOjaswin Mujoo BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical); 437438727786SOjaswin Mujoo } 437538727786SOjaswin Mujoo 437638727786SOjaswin Mujoo /* Step 4: trim our normalized range to not overlap with the neighbors */ 437738727786SOjaswin Mujoo if (left_pa) { 437838727786SOjaswin Mujoo if (left_pa_end > new_start) 437938727786SOjaswin Mujoo new_start = left_pa_end; 438038727786SOjaswin Mujoo } 438138727786SOjaswin Mujoo 438238727786SOjaswin Mujoo if (right_pa) { 438338727786SOjaswin Mujoo if (right_pa_start < new_end) 438438727786SOjaswin Mujoo new_end = right_pa_start; 438538727786SOjaswin Mujoo } 438638727786SOjaswin Mujoo read_unlock(&ei->i_prealloc_lock); 43870830344cSOjaswin Mujoo 43880830344cSOjaswin Mujoo /* XXX: extra loop to check we really don't overlap preallocations */ 43890830344cSOjaswin Mujoo ext4_mb_pa_assert_overlap(ac, new_start, new_end); 43900830344cSOjaswin Mujoo 43910830344cSOjaswin Mujoo *start = new_start; 43920830344cSOjaswin Mujoo *end = new_end; 43930830344cSOjaswin Mujoo } 43940830344cSOjaswin Mujoo 43950830344cSOjaswin Mujoo /* 4396c9de560dSAlex Tomas * Normalization means making request better in terms of 4397c9de560dSAlex Tomas * size and alignment 4398c9de560dSAlex Tomas */ 43994ddfef7bSEric Sandeen static noinline_for_stack void 44004ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 4401c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4402c9de560dSAlex Tomas { 440353accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4404b07ffe69SKemeng Shi struct ext4_super_block *es = sbi->s_es; 4405c9de560dSAlex Tomas int bsbits, max; 4406bedc5d34SBaokun Li loff_t size, start_off, end; 44071592d2c5SCurt Wohlgemuth loff_t orig_size __maybe_unused; 44085a0790c2SAndi Kleen ext4_lblk_t start; 4409c9de560dSAlex Tomas 4410c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 4411c9de560dSAlex Tomas do not need preallocation */ 4412c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 4413c9de560dSAlex Tomas return; 4414c9de560dSAlex Tomas 4415c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 4416c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 4417c9de560dSAlex Tomas return; 4418c9de560dSAlex Tomas 4419c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 4420c9de560dSAlex Tomas * required (it's a tail, for example) */ 4421c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 4422c9de560dSAlex Tomas return; 4423c9de560dSAlex Tomas 4424c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 4425c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 4426c9de560dSAlex Tomas return ; 4427c9de560dSAlex Tomas } 4428c9de560dSAlex Tomas 4429c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 4430c9de560dSAlex Tomas 4431c9de560dSAlex Tomas /* first, let's learn actual file size 4432c9de560dSAlex Tomas * given current request is allocated */ 443343bbddc0SBaokun Li size = extent_logical_end(sbi, &ac->ac_o_ex); 4434c9de560dSAlex Tomas size = size << bsbits; 4435c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 4436c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 44375a0790c2SAndi Kleen orig_size = size; 4438c9de560dSAlex Tomas 44391930479cSValerie Clement /* max size of free chunks */ 44401930479cSValerie Clement max = 2 << bsbits; 4441c9de560dSAlex Tomas 44421930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 44431930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 4444c9de560dSAlex Tomas 4445c9de560dSAlex Tomas /* first, try to predict filesize */ 4446c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 4447c9de560dSAlex Tomas start_off = 0; 4448c9de560dSAlex Tomas if (size <= 16 * 1024) { 4449c9de560dSAlex Tomas size = 16 * 1024; 4450c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 4451c9de560dSAlex Tomas size = 32 * 1024; 4452c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 4453c9de560dSAlex Tomas size = 64 * 1024; 4454c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 4455c9de560dSAlex Tomas size = 128 * 1024; 4456c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 4457c9de560dSAlex Tomas size = 256 * 1024; 4458c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 4459c9de560dSAlex Tomas size = 512 * 1024; 4460c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 4461c9de560dSAlex Tomas size = 1024 * 1024; 44621930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 4463c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 44641930479cSValerie Clement (21 - bsbits)) << 21; 44651930479cSValerie Clement size = 2 * 1024 * 1024; 44661930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 4467c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 4468c9de560dSAlex Tomas (22 - bsbits)) << 22; 4469c9de560dSAlex Tomas size = 4 * 1024 * 1024; 4470b3916da0SKemeng Shi } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len), 44711930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 4472c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 4473c9de560dSAlex Tomas (23 - bsbits)) << 23; 4474c9de560dSAlex Tomas size = 8 * 1024 * 1024; 4475c9de560dSAlex Tomas } else { 4476c9de560dSAlex Tomas start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; 447791a48aafSKemeng Shi size = (loff_t) EXT4_C2B(sbi, 4478b27b1535SXiaoguang Wang ac->ac_o_ex.fe_len) << bsbits; 4479c9de560dSAlex Tomas } 44805a0790c2SAndi Kleen size = size >> bsbits; 44815a0790c2SAndi Kleen start = start_off >> bsbits; 4482c9de560dSAlex Tomas 4483a08f789dSBaokun Li /* 4484a08f789dSBaokun Li * For tiny groups (smaller than 8MB) the chosen allocation 4485a08f789dSBaokun Li * alignment may be larger than group size. Make sure the 4486a08f789dSBaokun Li * alignment does not move allocation to a different group which 4487a08f789dSBaokun Li * makes mballoc fail assertions later. 4488a08f789dSBaokun Li */ 4489a08f789dSBaokun Li start = max(start, rounddown(ac->ac_o_ex.fe_logical, 4490a08f789dSBaokun Li (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); 4491a08f789dSBaokun Li 4492c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 4493c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 4494c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 4495c9de560dSAlex Tomas start = ar->lleft + 1; 4496c9de560dSAlex Tomas } 4497c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 4498c9de560dSAlex Tomas size -= start + size - ar->lright; 4499c9de560dSAlex Tomas 4500cd648b8aSJan Kara /* 4501cd648b8aSJan Kara * Trim allocation request for filesystems with artificially small 4502cd648b8aSJan Kara * groups. 4503cd648b8aSJan Kara */ 4504cd648b8aSJan Kara if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) 4505cd648b8aSJan Kara size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); 4506cd648b8aSJan Kara 4507c9de560dSAlex Tomas end = start + size; 4508c9de560dSAlex Tomas 45090830344cSOjaswin Mujoo ext4_mb_pa_adjust_overlap(ac, &start, &end); 4510c9de560dSAlex Tomas 4511c9de560dSAlex Tomas size = end - start; 4512c9de560dSAlex Tomas 4513cf4ff938SBaokun Li /* 4514cf4ff938SBaokun Li * In this function "start" and "size" are normalized for better 4515cf4ff938SBaokun Li * alignment and length such that we could preallocate more blocks. 4516cf4ff938SBaokun Li * This normalization is done such that original request of 4517cf4ff938SBaokun Li * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and 4518cf4ff938SBaokun Li * "size" boundaries. 4519cf4ff938SBaokun Li * (Note fe_len can be relaxed since FS block allocation API does not 4520cf4ff938SBaokun Li * provide gurantee on number of contiguous blocks allocation since that 4521cf4ff938SBaokun Li * depends upon free space left, etc). 4522cf4ff938SBaokun Li * In case of inode pa, later we use the allocated blocks 45231221b235SKemeng Shi * [pa_pstart + fe_logical - pa_lstart, fe_len/size] from the preallocated 4524cf4ff938SBaokun Li * range of goal/best blocks [start, size] to put it at the 4525cf4ff938SBaokun Li * ac_o_ex.fe_logical extent of this inode. 4526cf4ff938SBaokun Li * (See ext4_mb_use_inode_pa() for more details) 4527cf4ff938SBaokun Li */ 4528cf4ff938SBaokun Li if (start + size <= ac->ac_o_ex.fe_logical || 4529c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 45309d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, 45319d8b9ec4STheodore Ts'o "start %lu, size %lu, fe_logical %lu", 4532c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 4533c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 4534dfe076c1SDmitry Monakhov BUG(); 4535c9de560dSAlex Tomas } 4536b5b60778SMaurizio Lombardi BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 4537c9de560dSAlex Tomas 4538c9de560dSAlex Tomas /* now prepare goal request */ 4539c9de560dSAlex Tomas 4540c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 4541c9de560dSAlex Tomas * placement or satisfy big request as is */ 4542c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 454353accfa9STheodore Ts'o ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); 45447e170922SOjaswin Mujoo ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; 4545c9de560dSAlex Tomas 4546c9de560dSAlex Tomas /* define goal start in order to merge */ 4547b07ffe69SKemeng Shi if (ar->pright && (ar->lright == (start + size)) && 4548b07ffe69SKemeng Shi ar->pright >= size && 4549b07ffe69SKemeng Shi ar->pright - size >= le32_to_cpu(es->s_first_data_block)) { 4550c9de560dSAlex Tomas /* merge to the right */ 4551c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 4552b07ffe69SKemeng Shi &ac->ac_g_ex.fe_group, 4553b07ffe69SKemeng Shi &ac->ac_g_ex.fe_start); 4554c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 4555c9de560dSAlex Tomas } 4556b07ffe69SKemeng Shi if (ar->pleft && (ar->lleft + 1 == start) && 4557b07ffe69SKemeng Shi ar->pleft + 1 < ext4_blocks_count(es)) { 4558c9de560dSAlex Tomas /* merge to the left */ 4559c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 4560b07ffe69SKemeng Shi &ac->ac_g_ex.fe_group, 4561b07ffe69SKemeng Shi &ac->ac_g_ex.fe_start); 4562c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 4563c9de560dSAlex Tomas } 4564c9de560dSAlex Tomas 4565d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, 4566d3df1453SRitesh Harjani orig_size, start); 4567c9de560dSAlex Tomas } 4568c9de560dSAlex Tomas 4569c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 4570c9de560dSAlex Tomas { 4571c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4572c9de560dSAlex Tomas 4573a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { 4574c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 4575c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 4576291dae47SCurt Wohlgemuth if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) 4577c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 4578fdd9a009SOjaswin Mujoo 4579c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 4580fdd9a009SOjaswin Mujoo for (int i=0; i<EXT4_MB_NUM_CRS; i++) { 4581fdd9a009SOjaswin Mujoo atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]); 4582fdd9a009SOjaswin Mujoo } 4583fdd9a009SOjaswin Mujoo 4584a6c75eafSHarshad Shirwadkar atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); 4585c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 4586c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 4587c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 45887e170922SOjaswin Mujoo /* did we allocate as much as normalizer originally wanted? */ 45897e170922SOjaswin Mujoo if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len) 45903ef5d263SOjaswin Mujoo atomic_inc(&sbi->s_bal_len_goals); 45917e170922SOjaswin Mujoo 4592c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 4593c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 4594c9de560dSAlex Tomas } 4595c9de560dSAlex Tomas 4596296c355cSTheodore Ts'o if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) 4597296c355cSTheodore Ts'o trace_ext4_mballoc_alloc(ac); 4598296c355cSTheodore Ts'o else 4599296c355cSTheodore Ts'o trace_ext4_mballoc_prealloc(ac); 4600c9de560dSAlex Tomas } 4601c9de560dSAlex Tomas 4602c9de560dSAlex Tomas /* 4603b844167eSCurt Wohlgemuth * Called on failure; free up any blocks from the inode PA for this 4604b844167eSCurt Wohlgemuth * context. We don't need this for MB_GROUP_PA because we only change 4605b844167eSCurt Wohlgemuth * pa_free in ext4_mb_release_context(), but on failure, we've already 4606b844167eSCurt Wohlgemuth * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. 4607b844167eSCurt Wohlgemuth */ 4608b844167eSCurt Wohlgemuth static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) 4609b844167eSCurt Wohlgemuth { 4610b844167eSCurt Wohlgemuth struct ext4_prealloc_space *pa = ac->ac_pa; 461186f0afd4STheodore Ts'o struct ext4_buddy e4b; 461286f0afd4STheodore Ts'o int err; 4613b844167eSCurt Wohlgemuth 461486f0afd4STheodore Ts'o if (pa == NULL) { 4615c99d1e6eSTheodore Ts'o if (ac->ac_f_ex.fe_len == 0) 4616c99d1e6eSTheodore Ts'o return; 461786f0afd4STheodore Ts'o err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); 461819b8b035STheodore Ts'o if (WARN_RATELIMIT(err, 461919b8b035STheodore Ts'o "ext4: mb_load_buddy failed (%d)", err)) 462086f0afd4STheodore Ts'o /* 462186f0afd4STheodore Ts'o * This should never happen since we pin the 462286f0afd4STheodore Ts'o * pages in the ext4_allocation_context so 462386f0afd4STheodore Ts'o * ext4_mb_load_buddy() should never fail. 462486f0afd4STheodore Ts'o */ 462586f0afd4STheodore Ts'o return; 462686f0afd4STheodore Ts'o ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 462786f0afd4STheodore Ts'o mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, 462886f0afd4STheodore Ts'o ac->ac_f_ex.fe_len); 462986f0afd4STheodore Ts'o ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 4630c99d1e6eSTheodore Ts'o ext4_mb_unload_buddy(&e4b); 463186f0afd4STheodore Ts'o return; 463286f0afd4STheodore Ts'o } 463336cb0f52SKemeng Shi if (pa->pa_type == MB_INODE_PA) { 463436cb0f52SKemeng Shi spin_lock(&pa->pa_lock); 4635400db9d3SZheng Liu pa->pa_free += ac->ac_b_ex.fe_len; 463636cb0f52SKemeng Shi spin_unlock(&pa->pa_lock); 463736cb0f52SKemeng Shi } 4638b844167eSCurt Wohlgemuth } 4639b844167eSCurt Wohlgemuth 4640b844167eSCurt Wohlgemuth /* 4641c9de560dSAlex Tomas * use blocks preallocated to inode 4642c9de560dSAlex Tomas */ 4643c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 4644c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 4645c9de560dSAlex Tomas { 464653accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4647c9de560dSAlex Tomas ext4_fsblk_t start; 4648c9de560dSAlex Tomas ext4_fsblk_t end; 4649c9de560dSAlex Tomas int len; 4650c9de560dSAlex Tomas 4651c9de560dSAlex Tomas /* found preallocated blocks, use them */ 4652c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 465353accfa9STheodore Ts'o end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), 465453accfa9STheodore Ts'o start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); 465553accfa9STheodore Ts'o len = EXT4_NUM_B2C(sbi, end - start); 4656c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 4657c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 4658c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 4659c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 4660c9de560dSAlex Tomas ac->ac_pa = pa; 4661c9de560dSAlex Tomas 4662c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 466353accfa9STheodore Ts'o BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); 4664c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 466593cdf49fSOjaswin Mujoo BUG_ON(ac->ac_b_ex.fe_len <= 0); 4666c9de560dSAlex Tomas pa->pa_free -= len; 4667c9de560dSAlex Tomas 4668d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); 4669c9de560dSAlex Tomas } 4670c9de560dSAlex Tomas 4671c9de560dSAlex Tomas /* 4672c9de560dSAlex Tomas * use blocks preallocated to locality group 4673c9de560dSAlex Tomas */ 4674c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 4675c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 4676c9de560dSAlex Tomas { 467703cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 46786be2ded1SAneesh Kumar K.V 4679c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 4680c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 4681c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 4682c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 4683c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 4684c9de560dSAlex Tomas ac->ac_pa = pa; 4685c9de560dSAlex Tomas 46861221b235SKemeng Shi /* we don't correct pa_pstart or pa_len here to avoid 468726346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 4688c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 468926346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 469026346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 4691c9de560dSAlex Tomas */ 4692d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", 46931afdc588SKemeng Shi pa->pa_lstart, len, pa); 4694c9de560dSAlex Tomas } 4695c9de560dSAlex Tomas 4696c9de560dSAlex Tomas /* 46975e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 46985e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 46995e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 47005e745b04SAneesh Kumar K.V * from the goal block. 47015e745b04SAneesh Kumar K.V */ 47025e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 47035e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 47045e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 47055e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 47065e745b04SAneesh Kumar K.V { 47075e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 47085e745b04SAneesh Kumar K.V 47095e745b04SAneesh Kumar K.V if (cpa == NULL) { 47105e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 47115e745b04SAneesh Kumar K.V return pa; 47125e745b04SAneesh Kumar K.V } 471379211c8eSAndrew Morton cur_distance = abs(goal_block - cpa->pa_pstart); 471479211c8eSAndrew Morton new_distance = abs(goal_block - pa->pa_pstart); 47155e745b04SAneesh Kumar K.V 47165a54b2f1SColy Li if (cur_distance <= new_distance) 47175e745b04SAneesh Kumar K.V return cpa; 47185e745b04SAneesh Kumar K.V 47195e745b04SAneesh Kumar K.V /* drop the previous reference */ 47205e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 47215e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 47225e745b04SAneesh Kumar K.V return pa; 47235e745b04SAneesh Kumar K.V } 47245e745b04SAneesh Kumar K.V 47255e745b04SAneesh Kumar K.V /* 47261eff5904SKemeng Shi * check if found pa meets EXT4_MB_HINT_GOAL_ONLY 47271eff5904SKemeng Shi */ 47281eff5904SKemeng Shi static bool 47291eff5904SKemeng Shi ext4_mb_pa_goal_check(struct ext4_allocation_context *ac, 47301eff5904SKemeng Shi struct ext4_prealloc_space *pa) 47311eff5904SKemeng Shi { 47321eff5904SKemeng Shi struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 47331eff5904SKemeng Shi ext4_fsblk_t start; 47341eff5904SKemeng Shi 47351eff5904SKemeng Shi if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))) 47361eff5904SKemeng Shi return true; 47371eff5904SKemeng Shi 47381eff5904SKemeng Shi /* 47391eff5904SKemeng Shi * If EXT4_MB_HINT_GOAL_ONLY is set, ac_g_ex will not be adjusted 47401eff5904SKemeng Shi * in ext4_mb_normalize_request and will keep same with ac_o_ex 47411eff5904SKemeng Shi * from ext4_mb_initialize_context. Choose ac_g_ex here to keep 47421eff5904SKemeng Shi * consistent with ext4_mb_find_by_goal. 47431eff5904SKemeng Shi */ 47441eff5904SKemeng Shi start = pa->pa_pstart + 47451eff5904SKemeng Shi (ac->ac_g_ex.fe_logical - pa->pa_lstart); 47461eff5904SKemeng Shi if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start) 47471eff5904SKemeng Shi return false; 47481eff5904SKemeng Shi 47491eff5904SKemeng Shi if (ac->ac_g_ex.fe_len > pa->pa_len - 47501eff5904SKemeng Shi EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart)) 47511eff5904SKemeng Shi return false; 47521eff5904SKemeng Shi 47531eff5904SKemeng Shi return true; 47541eff5904SKemeng Shi } 47551eff5904SKemeng Shi 47561eff5904SKemeng Shi /* 4757c9de560dSAlex Tomas * search goal blocks in preallocated space 4758c9de560dSAlex Tomas */ 47594fca8f07SRitesh Harjani static noinline_for_stack bool 47604ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 4761c9de560dSAlex Tomas { 476253accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 47636be2ded1SAneesh Kumar K.V int order, i; 4764c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 4765c9de560dSAlex Tomas struct ext4_locality_group *lg; 47669d3de7eeSOjaswin Mujoo struct ext4_prealloc_space *tmp_pa = NULL, *cpa = NULL; 476738727786SOjaswin Mujoo struct rb_node *iter; 47685e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 4769c9de560dSAlex Tomas 4770c9de560dSAlex Tomas /* only data can be preallocated */ 4771c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 47724fca8f07SRitesh Harjani return false; 4773c9de560dSAlex Tomas 47749d3de7eeSOjaswin Mujoo /* 47759d3de7eeSOjaswin Mujoo * first, try per-file preallocation by searching the inode pa rbtree. 47769d3de7eeSOjaswin Mujoo * 47779d3de7eeSOjaswin Mujoo * Here, we can't do a direct traversal of the tree because 47789d3de7eeSOjaswin Mujoo * ext4_mb_discard_group_preallocation() can paralelly mark the pa 47799d3de7eeSOjaswin Mujoo * deleted and that can cause direct traversal to skip some entries. 47809d3de7eeSOjaswin Mujoo */ 478138727786SOjaswin Mujoo read_lock(&ei->i_prealloc_lock); 47829d3de7eeSOjaswin Mujoo 47839d3de7eeSOjaswin Mujoo if (RB_EMPTY_ROOT(&ei->i_prealloc_node)) { 47849d3de7eeSOjaswin Mujoo goto try_group_pa; 47859d3de7eeSOjaswin Mujoo } 47869d3de7eeSOjaswin Mujoo 47879d3de7eeSOjaswin Mujoo /* 47889d3de7eeSOjaswin Mujoo * Step 1: Find a pa with logical start immediately adjacent to the 47899d3de7eeSOjaswin Mujoo * original logical start. This could be on the left or right. 47909d3de7eeSOjaswin Mujoo * 47919d3de7eeSOjaswin Mujoo * (tmp_pa->pa_lstart never changes so we can skip locking for it). 47929d3de7eeSOjaswin Mujoo */ 479338727786SOjaswin Mujoo for (iter = ei->i_prealloc_node.rb_node; iter; 479438727786SOjaswin Mujoo iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, 47959d3de7eeSOjaswin Mujoo tmp_pa->pa_lstart, iter)) { 479638727786SOjaswin Mujoo tmp_pa = rb_entry(iter, struct ext4_prealloc_space, 479738727786SOjaswin Mujoo pa_node.inode_node); 47989d3de7eeSOjaswin Mujoo } 4799c9de560dSAlex Tomas 48009d3de7eeSOjaswin Mujoo /* 48019d3de7eeSOjaswin Mujoo * Step 2: The adjacent pa might be to the right of logical start, find 48029d3de7eeSOjaswin Mujoo * the left adjacent pa. After this step we'd have a valid tmp_pa whose 48039d3de7eeSOjaswin Mujoo * logical start is towards the left of original request's logical start 48049d3de7eeSOjaswin Mujoo */ 48059d3de7eeSOjaswin Mujoo if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) { 48069d3de7eeSOjaswin Mujoo struct rb_node *tmp; 48079d3de7eeSOjaswin Mujoo tmp = rb_prev(&tmp_pa->pa_node.inode_node); 4808bcf43499SOjaswin Mujoo 48099d3de7eeSOjaswin Mujoo if (tmp) { 48109d3de7eeSOjaswin Mujoo tmp_pa = rb_entry(tmp, struct ext4_prealloc_space, 48119d3de7eeSOjaswin Mujoo pa_node.inode_node); 48129d3de7eeSOjaswin Mujoo } else { 48139d3de7eeSOjaswin Mujoo /* 48149d3de7eeSOjaswin Mujoo * If there is no adjacent pa to the left then finding 48159d3de7eeSOjaswin Mujoo * an overlapping pa is not possible hence stop searching 48169d3de7eeSOjaswin Mujoo * inode pa tree 48179d3de7eeSOjaswin Mujoo */ 48189d3de7eeSOjaswin Mujoo goto try_group_pa; 48199d3de7eeSOjaswin Mujoo } 48209d3de7eeSOjaswin Mujoo } 48219d3de7eeSOjaswin Mujoo 48229d3de7eeSOjaswin Mujoo BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); 48239d3de7eeSOjaswin Mujoo 48249d3de7eeSOjaswin Mujoo /* 48259d3de7eeSOjaswin Mujoo * Step 3: If the left adjacent pa is deleted, keep moving left to find 48269d3de7eeSOjaswin Mujoo * the first non deleted adjacent pa. After this step we should have a 48279d3de7eeSOjaswin Mujoo * valid tmp_pa which is guaranteed to be non deleted. 48289d3de7eeSOjaswin Mujoo */ 48299d3de7eeSOjaswin Mujoo for (iter = &tmp_pa->pa_node.inode_node;; iter = rb_prev(iter)) { 48309d3de7eeSOjaswin Mujoo if (!iter) { 48319d3de7eeSOjaswin Mujoo /* 48329d3de7eeSOjaswin Mujoo * no non deleted left adjacent pa, so stop searching 48339d3de7eeSOjaswin Mujoo * inode pa tree 48349d3de7eeSOjaswin Mujoo */ 48359d3de7eeSOjaswin Mujoo goto try_group_pa; 48369d3de7eeSOjaswin Mujoo } 48379d3de7eeSOjaswin Mujoo tmp_pa = rb_entry(iter, struct ext4_prealloc_space, 48389d3de7eeSOjaswin Mujoo pa_node.inode_node); 48399d3de7eeSOjaswin Mujoo spin_lock(&tmp_pa->pa_lock); 48409d3de7eeSOjaswin Mujoo if (tmp_pa->pa_deleted == 0) { 48419d3de7eeSOjaswin Mujoo /* 48429d3de7eeSOjaswin Mujoo * We will keep holding the pa_lock from 48439d3de7eeSOjaswin Mujoo * this point on because we don't want group discard 48449d3de7eeSOjaswin Mujoo * to delete this pa underneath us. Since group 48459d3de7eeSOjaswin Mujoo * discard is anyways an ENOSPC operation it 48469d3de7eeSOjaswin Mujoo * should be okay for it to wait a few more cycles. 48479d3de7eeSOjaswin Mujoo */ 48489d3de7eeSOjaswin Mujoo break; 48499d3de7eeSOjaswin Mujoo } else { 48509d3de7eeSOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 48519d3de7eeSOjaswin Mujoo } 48529d3de7eeSOjaswin Mujoo } 48539d3de7eeSOjaswin Mujoo 48549d3de7eeSOjaswin Mujoo BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); 48559d3de7eeSOjaswin Mujoo BUG_ON(tmp_pa->pa_deleted == 1); 48569d3de7eeSOjaswin Mujoo 48579d3de7eeSOjaswin Mujoo /* 48589d3de7eeSOjaswin Mujoo * Step 4: We now have the non deleted left adjacent pa. Only this 48599d3de7eeSOjaswin Mujoo * pa can possibly satisfy the request hence check if it overlaps 48609d3de7eeSOjaswin Mujoo * original logical start and stop searching if it doesn't. 48619d3de7eeSOjaswin Mujoo */ 486243bbddc0SBaokun Li if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) { 48639d3de7eeSOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 48649d3de7eeSOjaswin Mujoo goto try_group_pa; 48659d3de7eeSOjaswin Mujoo } 4866c9de560dSAlex Tomas 4867fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 486812e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && 4869bcf43499SOjaswin Mujoo (tmp_pa->pa_pstart + EXT4_C2B(sbi, tmp_pa->pa_len) > 4870e86a7182SOjaswin Mujoo EXT4_MAX_BLOCK_FILE_PHYS)) { 4871e86a7182SOjaswin Mujoo /* 48729d3de7eeSOjaswin Mujoo * Since PAs don't overlap, we won't find any other PA to 48739d3de7eeSOjaswin Mujoo * satisfy this. 4874e86a7182SOjaswin Mujoo */ 48759d3de7eeSOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 48769d3de7eeSOjaswin Mujoo goto try_group_pa; 4877e86a7182SOjaswin Mujoo } 4878fb0a387dSEric Sandeen 48799d3de7eeSOjaswin Mujoo if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) { 4880bcf43499SOjaswin Mujoo atomic_inc(&tmp_pa->pa_count); 4881bcf43499SOjaswin Mujoo ext4_mb_use_inode_pa(ac, tmp_pa); 4882bcf43499SOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 488338727786SOjaswin Mujoo read_unlock(&ei->i_prealloc_lock); 48844fca8f07SRitesh Harjani return true; 48859d3de7eeSOjaswin Mujoo } else { 48869d3de7eeSOjaswin Mujoo /* 48879d3de7eeSOjaswin Mujoo * We found a valid overlapping pa but couldn't use it because 48889d3de7eeSOjaswin Mujoo * it had no free blocks. This should ideally never happen 48899d3de7eeSOjaswin Mujoo * because: 48909d3de7eeSOjaswin Mujoo * 48919d3de7eeSOjaswin Mujoo * 1. When a new inode pa is added to rbtree it must have 48929d3de7eeSOjaswin Mujoo * pa_free > 0 since otherwise we won't actually need 48939d3de7eeSOjaswin Mujoo * preallocation. 48949d3de7eeSOjaswin Mujoo * 48959d3de7eeSOjaswin Mujoo * 2. An inode pa that is in the rbtree can only have it's 48969d3de7eeSOjaswin Mujoo * pa_free become zero when another thread calls: 48979d3de7eeSOjaswin Mujoo * ext4_mb_new_blocks 48989d3de7eeSOjaswin Mujoo * ext4_mb_use_preallocated 48999d3de7eeSOjaswin Mujoo * ext4_mb_use_inode_pa 49009d3de7eeSOjaswin Mujoo * 49019d3de7eeSOjaswin Mujoo * 3. Further, after the above calls make pa_free == 0, we will 49029d3de7eeSOjaswin Mujoo * immediately remove it from the rbtree in: 49039d3de7eeSOjaswin Mujoo * ext4_mb_new_blocks 49049d3de7eeSOjaswin Mujoo * ext4_mb_release_context 49059d3de7eeSOjaswin Mujoo * ext4_mb_put_pa 49069d3de7eeSOjaswin Mujoo * 49079d3de7eeSOjaswin Mujoo * 4. Since the pa_free becoming 0 and pa_free getting removed 49089d3de7eeSOjaswin Mujoo * from tree both happen in ext4_mb_new_blocks, which is always 49099d3de7eeSOjaswin Mujoo * called with i_data_sem held for data allocations, we can be 49109d3de7eeSOjaswin Mujoo * sure that another process will never see a pa in rbtree with 49119d3de7eeSOjaswin Mujoo * pa_free == 0. 49129d3de7eeSOjaswin Mujoo */ 49139d3de7eeSOjaswin Mujoo WARN_ON_ONCE(tmp_pa->pa_free == 0); 4914c9de560dSAlex Tomas } 4915bcf43499SOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 49169d3de7eeSOjaswin Mujoo try_group_pa: 491738727786SOjaswin Mujoo read_unlock(&ei->i_prealloc_lock); 4918c9de560dSAlex Tomas 4919c9de560dSAlex Tomas /* can we use group allocation? */ 4920c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 49214fca8f07SRitesh Harjani return false; 4922c9de560dSAlex Tomas 4923c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 4924c9de560dSAlex Tomas lg = ac->ac_lg; 4925c9de560dSAlex Tomas if (lg == NULL) 49264fca8f07SRitesh Harjani return false; 49276be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 49286be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 49296be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 49306be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 4931c9de560dSAlex Tomas 4932bda00de7SAkinobu Mita goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); 49335e745b04SAneesh Kumar K.V /* 49345e745b04SAneesh Kumar K.V * search for the prealloc space that is having 49355e745b04SAneesh Kumar K.V * minimal distance from the goal block. 49365e745b04SAneesh Kumar K.V */ 49376be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 4938c9de560dSAlex Tomas rcu_read_lock(); 4939bcf43499SOjaswin Mujoo list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[i], 4940a8e38fd3SOjaswin Mujoo pa_node.lg_list) { 4941bcf43499SOjaswin Mujoo spin_lock(&tmp_pa->pa_lock); 4942bcf43499SOjaswin Mujoo if (tmp_pa->pa_deleted == 0 && 4943bcf43499SOjaswin Mujoo tmp_pa->pa_free >= ac->ac_o_ex.fe_len) { 49445e745b04SAneesh Kumar K.V 49455e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 4946bcf43499SOjaswin Mujoo tmp_pa, cpa); 49475e745b04SAneesh Kumar K.V } 4948bcf43499SOjaswin Mujoo spin_unlock(&tmp_pa->pa_lock); 49495e745b04SAneesh Kumar K.V } 49505e745b04SAneesh Kumar K.V rcu_read_unlock(); 49515e745b04SAneesh Kumar K.V } 49525e745b04SAneesh Kumar K.V if (cpa) { 49535e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 49544fca8f07SRitesh Harjani return true; 4955c9de560dSAlex Tomas } 49564fca8f07SRitesh Harjani return false; 4957c9de560dSAlex Tomas } 4958c9de560dSAlex Tomas 4959c9de560dSAlex Tomas /* 49607a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 49617a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 49627a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 4963955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 49647a2fcbf7SAneesh Kumar K.V */ 49657a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 49667a2fcbf7SAneesh Kumar K.V ext4_group_t group) 49677a2fcbf7SAneesh Kumar K.V { 49687a2fcbf7SAneesh Kumar K.V struct rb_node *n; 49697a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 49707a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 49717a2fcbf7SAneesh Kumar K.V 49727a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 49735354b2afSTheodore Ts'o if (!grp) 49745354b2afSTheodore Ts'o return; 49757a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 49767a2fcbf7SAneesh Kumar K.V 49777a2fcbf7SAneesh Kumar K.V while (n) { 497818aadd47SBobi Jam entry = rb_entry(n, struct ext4_free_data, efd_node); 4979123e3016SRitesh Harjani mb_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count); 49807a2fcbf7SAneesh Kumar K.V n = rb_next(n); 49817a2fcbf7SAneesh Kumar K.V } 49827a2fcbf7SAneesh Kumar K.V return; 49837a2fcbf7SAneesh Kumar K.V } 49847a2fcbf7SAneesh Kumar K.V 49857a2fcbf7SAneesh Kumar K.V /* 4986c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 4987c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 4988955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 4989c9de560dSAlex Tomas */ 4990089ceeccSEric Sandeen static noinline_for_stack 4991089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 4992c9de560dSAlex Tomas ext4_group_t group) 4993c9de560dSAlex Tomas { 4994c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 4995c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4996c9de560dSAlex Tomas struct list_head *cur; 4997c9de560dSAlex Tomas ext4_group_t groupnr; 4998c9de560dSAlex Tomas ext4_grpblk_t start; 4999c9de560dSAlex Tomas int preallocated = 0; 5000c9de560dSAlex Tomas int len; 5001c9de560dSAlex Tomas 50025354b2afSTheodore Ts'o if (!grp) 50035354b2afSTheodore Ts'o return; 50045354b2afSTheodore Ts'o 5005c9de560dSAlex Tomas /* all form of preallocation discards first load group, 5006c9de560dSAlex Tomas * so the only competing code is preallocation use. 5007c9de560dSAlex Tomas * we don't need any locking here 5008c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 5009c9de560dSAlex Tomas * otherwise we could leave used blocks available for 5010c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 5011c9de560dSAlex Tomas * is dropping preallocation 5012c9de560dSAlex Tomas */ 5013c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 5014c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 5015c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 5016c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 5017c9de560dSAlex Tomas &groupnr, &start); 5018c9de560dSAlex Tomas len = pa->pa_len; 5019c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5020c9de560dSAlex Tomas if (unlikely(len == 0)) 5021c9de560dSAlex Tomas continue; 5022c9de560dSAlex Tomas BUG_ON(groupnr != group); 5023123e3016SRitesh Harjani mb_set_bits(bitmap, start, len); 5024c9de560dSAlex Tomas preallocated += len; 5025c9de560dSAlex Tomas } 5026d3df1453SRitesh Harjani mb_debug(sb, "preallocated %d for group %u\n", preallocated, group); 5027c9de560dSAlex Tomas } 5028c9de560dSAlex Tomas 502927bc446eSbrookxu static void ext4_mb_mark_pa_deleted(struct super_block *sb, 503027bc446eSbrookxu struct ext4_prealloc_space *pa) 503127bc446eSbrookxu { 503227bc446eSbrookxu struct ext4_inode_info *ei; 503327bc446eSbrookxu 503427bc446eSbrookxu if (pa->pa_deleted) { 503527bc446eSbrookxu ext4_warning(sb, "deleted pa, type:%d, pblk:%llu, lblk:%u, len:%d\n", 503627bc446eSbrookxu pa->pa_type, pa->pa_pstart, pa->pa_lstart, 503727bc446eSbrookxu pa->pa_len); 503827bc446eSbrookxu return; 503927bc446eSbrookxu } 504027bc446eSbrookxu 504127bc446eSbrookxu pa->pa_deleted = 1; 504227bc446eSbrookxu 504327bc446eSbrookxu if (pa->pa_type == MB_INODE_PA) { 504427bc446eSbrookxu ei = EXT4_I(pa->pa_inode); 504527bc446eSbrookxu atomic_dec(&ei->i_prealloc_active); 504627bc446eSbrookxu } 504727bc446eSbrookxu } 504827bc446eSbrookxu 504982089725SOjaswin Mujoo static inline void ext4_mb_pa_free(struct ext4_prealloc_space *pa) 5050c9de560dSAlex Tomas { 505182089725SOjaswin Mujoo BUG_ON(!pa); 50524e8d2139SJunho Ryu BUG_ON(atomic_read(&pa->pa_count)); 50534e8d2139SJunho Ryu BUG_ON(pa->pa_deleted == 0); 5054c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 5055c9de560dSAlex Tomas } 5056c9de560dSAlex Tomas 505782089725SOjaswin Mujoo static void ext4_mb_pa_callback(struct rcu_head *head) 505882089725SOjaswin Mujoo { 505982089725SOjaswin Mujoo struct ext4_prealloc_space *pa; 506082089725SOjaswin Mujoo 506182089725SOjaswin Mujoo pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 506282089725SOjaswin Mujoo ext4_mb_pa_free(pa); 506382089725SOjaswin Mujoo } 506482089725SOjaswin Mujoo 5065c9de560dSAlex Tomas /* 5066c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 5067c9de560dSAlex Tomas * if this was the last reference and the space is consumed 5068c9de560dSAlex Tomas */ 5069c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 5070c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 5071c9de560dSAlex Tomas { 5072a9df9a49STheodore Ts'o ext4_group_t grp; 5073d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 507438727786SOjaswin Mujoo struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 5075c9de560dSAlex Tomas 5076c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 5077c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 50784e8d2139SJunho Ryu if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { 50794e8d2139SJunho Ryu spin_unlock(&pa->pa_lock); 50804e8d2139SJunho Ryu return; 50814e8d2139SJunho Ryu } 50824e8d2139SJunho Ryu 5083c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 5084c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5085c9de560dSAlex Tomas return; 5086c9de560dSAlex Tomas } 5087c9de560dSAlex Tomas 508827bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 5089c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5090c9de560dSAlex Tomas 5091d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 5092cc0fb9adSAneesh Kumar K.V /* 5093cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 5094cc0fb9adSAneesh Kumar K.V * next group when pa is used up 5095cc0fb9adSAneesh Kumar K.V */ 5096cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 5097d33a1976SEric Sandeen grp_blk--; 5098d33a1976SEric Sandeen 5099bd86298eSLukas Czerner grp = ext4_get_group_number(sb, grp_blk); 5100c9de560dSAlex Tomas 5101c9de560dSAlex Tomas /* 5102c9de560dSAlex Tomas * possible race: 5103c9de560dSAlex Tomas * 5104c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 5105c9de560dSAlex Tomas * find block B in PA 5106c9de560dSAlex Tomas * copy on-disk bitmap to buddy 5107c9de560dSAlex Tomas * mark B in on-disk bitmap 5108c9de560dSAlex Tomas * drop PA from group 5109c9de560dSAlex Tomas * mark all PAs in buddy 5110c9de560dSAlex Tomas * 5111c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 5112c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 5113c9de560dSAlex Tomas * against that pair 5114c9de560dSAlex Tomas */ 5115c9de560dSAlex Tomas ext4_lock_group(sb, grp); 5116c9de560dSAlex Tomas list_del(&pa->pa_group_list); 5117c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 5118c9de560dSAlex Tomas 5119a8e38fd3SOjaswin Mujoo if (pa->pa_type == MB_INODE_PA) { 512038727786SOjaswin Mujoo write_lock(pa->pa_node_lock.inode_lock); 512138727786SOjaswin Mujoo rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); 512238727786SOjaswin Mujoo write_unlock(pa->pa_node_lock.inode_lock); 512338727786SOjaswin Mujoo ext4_mb_pa_free(pa); 5124a8e38fd3SOjaswin Mujoo } else { 5125a8e38fd3SOjaswin Mujoo spin_lock(pa->pa_node_lock.lg_lock); 5126a8e38fd3SOjaswin Mujoo list_del_rcu(&pa->pa_node.lg_list); 5127a8e38fd3SOjaswin Mujoo spin_unlock(pa->pa_node_lock.lg_lock); 512838727786SOjaswin Mujoo call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 512938727786SOjaswin Mujoo } 5130a8e38fd3SOjaswin Mujoo } 5131c9de560dSAlex Tomas 513238727786SOjaswin Mujoo static void ext4_mb_pa_rb_insert(struct rb_root *root, struct rb_node *new) 513338727786SOjaswin Mujoo { 513438727786SOjaswin Mujoo struct rb_node **iter = &root->rb_node, *parent = NULL; 513538727786SOjaswin Mujoo struct ext4_prealloc_space *iter_pa, *new_pa; 513638727786SOjaswin Mujoo ext4_lblk_t iter_start, new_start; 513738727786SOjaswin Mujoo 513838727786SOjaswin Mujoo while (*iter) { 513938727786SOjaswin Mujoo iter_pa = rb_entry(*iter, struct ext4_prealloc_space, 514038727786SOjaswin Mujoo pa_node.inode_node); 514138727786SOjaswin Mujoo new_pa = rb_entry(new, struct ext4_prealloc_space, 514238727786SOjaswin Mujoo pa_node.inode_node); 514338727786SOjaswin Mujoo iter_start = iter_pa->pa_lstart; 514438727786SOjaswin Mujoo new_start = new_pa->pa_lstart; 514538727786SOjaswin Mujoo 514638727786SOjaswin Mujoo parent = *iter; 514738727786SOjaswin Mujoo if (new_start < iter_start) 514838727786SOjaswin Mujoo iter = &((*iter)->rb_left); 514938727786SOjaswin Mujoo else 515038727786SOjaswin Mujoo iter = &((*iter)->rb_right); 515138727786SOjaswin Mujoo } 515238727786SOjaswin Mujoo 515338727786SOjaswin Mujoo rb_link_node(new, parent, iter); 515438727786SOjaswin Mujoo rb_insert_color(new, root); 5155c9de560dSAlex Tomas } 5156c9de560dSAlex Tomas 5157c9de560dSAlex Tomas /* 5158c9de560dSAlex Tomas * creates new preallocated space for given inode 5159c9de560dSAlex Tomas */ 516053f86b17SRitesh Harjani static noinline_for_stack void 51614ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 5162c9de560dSAlex Tomas { 5163c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 516453accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 5165c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 5166c9de560dSAlex Tomas struct ext4_group_info *grp; 5167c9de560dSAlex Tomas struct ext4_inode_info *ei; 5168c9de560dSAlex Tomas 5169c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 5170c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 5171c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 5172c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 517353f86b17SRitesh Harjani BUG_ON(ac->ac_pa == NULL); 5174c9de560dSAlex Tomas 517553f86b17SRitesh Harjani pa = ac->ac_pa; 5176c9de560dSAlex Tomas 51777e170922SOjaswin Mujoo if (ac->ac_b_ex.fe_len < ac->ac_orig_goal_len) { 5178bc056e71SBaokun Li struct ext4_free_extent ex = { 5179bc056e71SBaokun Li .fe_logical = ac->ac_g_ex.fe_logical, 5180bc056e71SBaokun Li .fe_len = ac->ac_orig_goal_len, 5181bc056e71SBaokun Li }; 5182bc056e71SBaokun Li loff_t orig_goal_end = extent_logical_end(sbi, &ex); 5183c9de560dSAlex Tomas 5184c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 5185c9de560dSAlex Tomas * so, found space must get proper lstart 5186c9de560dSAlex Tomas * to cover original request */ 5187c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 5188c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 5189c9de560dSAlex Tomas 519093cdf49fSOjaswin Mujoo /* 519193cdf49fSOjaswin Mujoo * Use the below logic for adjusting best extent as it keeps 519293cdf49fSOjaswin Mujoo * fragmentation in check while ensuring logical range of best 519393cdf49fSOjaswin Mujoo * extent doesn't overflow out of goal extent: 519493cdf49fSOjaswin Mujoo * 51957e170922SOjaswin Mujoo * 1. Check if best ex can be kept at end of goal (before 51967e170922SOjaswin Mujoo * cr_best_avail trimmed it) and still cover original start 519793cdf49fSOjaswin Mujoo * 2. Else, check if best ex can be kept at start of goal and 519893cdf49fSOjaswin Mujoo * still cover original start 519993cdf49fSOjaswin Mujoo * 3. Else, keep the best ex at start of original request. 520093cdf49fSOjaswin Mujoo */ 5201bc056e71SBaokun Li ex.fe_len = ac->ac_b_ex.fe_len; 5202bc056e71SBaokun Li 5203bc056e71SBaokun Li ex.fe_logical = orig_goal_end - EXT4_C2B(sbi, ex.fe_len); 5204bc056e71SBaokun Li if (ac->ac_o_ex.fe_logical >= ex.fe_logical) 520593cdf49fSOjaswin Mujoo goto adjust_bex; 5206c9de560dSAlex Tomas 5207bc056e71SBaokun Li ex.fe_logical = ac->ac_g_ex.fe_logical; 5208bc056e71SBaokun Li if (ac->ac_o_ex.fe_logical < extent_logical_end(sbi, &ex)) 520993cdf49fSOjaswin Mujoo goto adjust_bex; 5210c9de560dSAlex Tomas 5211bc056e71SBaokun Li ex.fe_logical = ac->ac_o_ex.fe_logical; 521293cdf49fSOjaswin Mujoo adjust_bex: 5213bc056e71SBaokun Li ac->ac_b_ex.fe_logical = ex.fe_logical; 5214c9de560dSAlex Tomas 5215c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 5216c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 5217bc056e71SBaokun Li BUG_ON(extent_logical_end(sbi, &ex) > orig_goal_end); 5218c9de560dSAlex Tomas } 5219c9de560dSAlex Tomas 5220c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 5221c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 5222c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 5223c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 5224c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 5225d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 5226c9de560dSAlex Tomas pa->pa_deleted = 0; 5227cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 5228c9de560dSAlex Tomas 5229d3df1453SRitesh Harjani mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, 5230d3df1453SRitesh Harjani pa->pa_len, pa->pa_lstart); 52319bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 5232c9de560dSAlex Tomas 523353accfa9STheodore Ts'o atomic_add(pa->pa_free, &sbi->s_mb_preallocated); 5234abc075d4SKemeng Shi ext4_mb_use_inode_pa(ac, pa); 5235c9de560dSAlex Tomas 5236c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 5237c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 52385354b2afSTheodore Ts'o if (!grp) 52395354b2afSTheodore Ts'o return; 5240c9de560dSAlex Tomas 5241a8e38fd3SOjaswin Mujoo pa->pa_node_lock.inode_lock = &ei->i_prealloc_lock; 5242c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 5243c9de560dSAlex Tomas 5244c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 5245c9de560dSAlex Tomas 524638727786SOjaswin Mujoo write_lock(pa->pa_node_lock.inode_lock); 524738727786SOjaswin Mujoo ext4_mb_pa_rb_insert(&ei->i_prealloc_node, &pa->pa_node.inode_node); 524838727786SOjaswin Mujoo write_unlock(pa->pa_node_lock.inode_lock); 524927bc446eSbrookxu atomic_inc(&ei->i_prealloc_active); 5250c9de560dSAlex Tomas } 5251c9de560dSAlex Tomas 5252c9de560dSAlex Tomas /* 5253c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 5254c9de560dSAlex Tomas */ 525553f86b17SRitesh Harjani static noinline_for_stack void 52564ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 5257c9de560dSAlex Tomas { 5258c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 5259c9de560dSAlex Tomas struct ext4_locality_group *lg; 5260c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 5261c9de560dSAlex Tomas struct ext4_group_info *grp; 5262c9de560dSAlex Tomas 5263c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 5264c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 5265c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 5266c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 526753f86b17SRitesh Harjani BUG_ON(ac->ac_pa == NULL); 5268c9de560dSAlex Tomas 526953f86b17SRitesh Harjani pa = ac->ac_pa; 5270c9de560dSAlex Tomas 5271c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 5272c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 5273c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 5274c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 5275c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 5276a8e38fd3SOjaswin Mujoo INIT_LIST_HEAD(&pa->pa_node.lg_list); 5277d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 5278c9de560dSAlex Tomas pa->pa_deleted = 0; 5279cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 5280c9de560dSAlex Tomas 5281d3df1453SRitesh Harjani mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, 5282d3df1453SRitesh Harjani pa->pa_len, pa->pa_lstart); 52839bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 5284c9de560dSAlex Tomas 5285c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 5286c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 5287c9de560dSAlex Tomas 5288c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 52895354b2afSTheodore Ts'o if (!grp) 52905354b2afSTheodore Ts'o return; 5291c9de560dSAlex Tomas lg = ac->ac_lg; 5292c9de560dSAlex Tomas BUG_ON(lg == NULL); 5293c9de560dSAlex Tomas 5294a8e38fd3SOjaswin Mujoo pa->pa_node_lock.lg_lock = &lg->lg_prealloc_lock; 5295c9de560dSAlex Tomas pa->pa_inode = NULL; 5296c9de560dSAlex Tomas 5297c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 5298c9de560dSAlex Tomas 52996be2ded1SAneesh Kumar K.V /* 53006be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 53016be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 53026be2ded1SAneesh Kumar K.V */ 5303c9de560dSAlex Tomas } 5304c9de560dSAlex Tomas 530553f86b17SRitesh Harjani static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 5306c9de560dSAlex Tomas { 5307c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 530853f86b17SRitesh Harjani ext4_mb_new_group_pa(ac); 5309c9de560dSAlex Tomas else 531053f86b17SRitesh Harjani ext4_mb_new_inode_pa(ac); 5311c9de560dSAlex Tomas } 5312c9de560dSAlex Tomas 5313c9de560dSAlex Tomas /* 5314c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 5315c9de560dSAlex Tomas * in-core bitmap and buddy. 5316c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 5317c9de560dSAlex Tomas * nobody else can find/use it. 5318c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 5319c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 5320c9de560dSAlex Tomas */ 53214ddfef7bSEric Sandeen static noinline_for_stack int 53224ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 53233e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 5324c9de560dSAlex Tomas { 5325c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 5326c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 5327498e5f24STheodore Ts'o unsigned int end; 5328498e5f24STheodore Ts'o unsigned int next; 5329c9de560dSAlex Tomas ext4_group_t group; 5330c9de560dSAlex Tomas ext4_grpblk_t bit; 5331ba80b101STheodore Ts'o unsigned long long grp_blk_start; 5332c9de560dSAlex Tomas int free = 0; 5333c9de560dSAlex Tomas 5334c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 5335c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 533653accfa9STheodore Ts'o grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); 5337c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 5338c9de560dSAlex Tomas end = bit + pa->pa_len; 5339c9de560dSAlex Tomas 5340c9de560dSAlex Tomas while (bit < end) { 5341ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 5342c9de560dSAlex Tomas if (bit >= end) 5343c9de560dSAlex Tomas break; 5344ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 5345d3df1453SRitesh Harjani mb_debug(sb, "free preallocated %u/%u in group %u\n", 53465a0790c2SAndi Kleen (unsigned) ext4_group_first_block_no(sb, group) + bit, 53475a0790c2SAndi Kleen (unsigned) next - bit, (unsigned) group); 5348c9de560dSAlex Tomas free += next - bit; 5349c9de560dSAlex Tomas 53503e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); 535153accfa9STheodore Ts'o trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + 535253accfa9STheodore Ts'o EXT4_C2B(sbi, bit)), 5353a9c667f8SLukas Czerner next - bit); 5354c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 5355c9de560dSAlex Tomas bit = next + 1; 5356c9de560dSAlex Tomas } 5357c9de560dSAlex Tomas if (free != pa->pa_free) { 53589d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_CRIT, 535936bad423SRitesh Harjani "pa %p: logic %lu, phys. %lu, len %d", 5360c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 5361c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 536236bad423SRitesh Harjani pa->pa_len); 5363e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", 536426346ff6SAneesh Kumar K.V free, pa->pa_free); 5365e56eb659SAneesh Kumar K.V /* 5366e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 5367e56eb659SAneesh Kumar K.V * from the bitmap and continue. 5368e56eb659SAneesh Kumar K.V */ 5369c9de560dSAlex Tomas } 5370c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 5371c9de560dSAlex Tomas 5372863c37fcSzhong jiang return 0; 5373c9de560dSAlex Tomas } 5374c9de560dSAlex Tomas 53754ddfef7bSEric Sandeen static noinline_for_stack int 53764ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 53773e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 5378c9de560dSAlex Tomas { 5379c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 5380c9de560dSAlex Tomas ext4_group_t group; 5381c9de560dSAlex Tomas ext4_grpblk_t bit; 5382c9de560dSAlex Tomas 538360e07cf5SYongqiang Yang trace_ext4_mb_release_group_pa(sb, pa); 5384c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 5385c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 5386463808f2STheodore Ts'o if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) { 5387463808f2STheodore Ts'o ext4_warning(sb, "bad group: expected %u, group %u, pa_start %llu", 5388463808f2STheodore Ts'o e4b->bd_group, group, pa->pa_pstart); 5389463808f2STheodore Ts'o return 0; 5390463808f2STheodore Ts'o } 5391c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 5392c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 53933e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); 5394c9de560dSAlex Tomas 5395c9de560dSAlex Tomas return 0; 5396c9de560dSAlex Tomas } 5397c9de560dSAlex Tomas 5398c9de560dSAlex Tomas /* 5399c9de560dSAlex Tomas * releases all preallocations in given group 5400c9de560dSAlex Tomas * 5401c9de560dSAlex Tomas * first, we need to decide discard policy: 5402c9de560dSAlex Tomas * - when do we discard 5403c9de560dSAlex Tomas * 1) ENOSPC 5404c9de560dSAlex Tomas * - how many do we discard 5405c9de560dSAlex Tomas * 1) how many requested 5406c9de560dSAlex Tomas */ 54074ddfef7bSEric Sandeen static noinline_for_stack int 54084ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 54098c80fb31SChunguang Xu ext4_group_t group, int *busy) 5410c9de560dSAlex Tomas { 5411c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 5412c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 5413c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 5414c9de560dSAlex Tomas struct list_head list; 5415c9de560dSAlex Tomas struct ext4_buddy e4b; 541638727786SOjaswin Mujoo struct ext4_inode_info *ei; 5417c9de560dSAlex Tomas int err; 54188c80fb31SChunguang Xu int free = 0; 5419c9de560dSAlex Tomas 54205354b2afSTheodore Ts'o if (!grp) 54215354b2afSTheodore Ts'o return 0; 5422d3df1453SRitesh Harjani mb_debug(sb, "discard preallocation for group %u\n", group); 5423c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 5424bbc4ec77SRitesh Harjani goto out_dbg; 5425c9de560dSAlex Tomas 5426574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 54279008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 54289008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 542954d3adbcSTheodore Ts'o ext4_error_err(sb, -err, 543054d3adbcSTheodore Ts'o "Error %d reading block bitmap for %u", 54319008a58eSDarrick J. Wong err, group); 5432bbc4ec77SRitesh Harjani goto out_dbg; 5433c9de560dSAlex Tomas } 5434c9de560dSAlex Tomas 5435c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 5436ce89f46cSAneesh Kumar K.V if (err) { 54379651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 54389651e6b2SKonstantin Khlebnikov err, group); 5439ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 5440bbc4ec77SRitesh Harjani goto out_dbg; 5441ce89f46cSAneesh Kumar K.V } 5442c9de560dSAlex Tomas 5443c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 5444c9de560dSAlex Tomas ext4_lock_group(sb, group); 5445c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 5446c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 5447c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 5448c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 5449c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 54508c80fb31SChunguang Xu *busy = 1; 5451c9de560dSAlex Tomas continue; 5452c9de560dSAlex Tomas } 5453c9de560dSAlex Tomas if (pa->pa_deleted) { 5454c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5455c9de560dSAlex Tomas continue; 5456c9de560dSAlex Tomas } 5457c9de560dSAlex Tomas 5458c9de560dSAlex Tomas /* seems this one can be freed ... */ 545927bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 5460c9de560dSAlex Tomas 546170022da8SYe Bin if (!free) 546270022da8SYe Bin this_cpu_inc(discard_pa_seq); 546370022da8SYe Bin 5464c9de560dSAlex Tomas /* we can trust pa_free ... */ 5465c9de560dSAlex Tomas free += pa->pa_free; 5466c9de560dSAlex Tomas 5467c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5468c9de560dSAlex Tomas 5469c9de560dSAlex Tomas list_del(&pa->pa_group_list); 5470c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 5471c9de560dSAlex Tomas } 5472c9de560dSAlex Tomas 5473c9de560dSAlex Tomas /* now free all selected PAs */ 5474c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 5475c9de560dSAlex Tomas 5476c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 5477a8e38fd3SOjaswin Mujoo if (pa->pa_type == MB_GROUP_PA) { 5478a8e38fd3SOjaswin Mujoo spin_lock(pa->pa_node_lock.lg_lock); 5479a8e38fd3SOjaswin Mujoo list_del_rcu(&pa->pa_node.lg_list); 5480a8e38fd3SOjaswin Mujoo spin_unlock(pa->pa_node_lock.lg_lock); 5481a8e38fd3SOjaswin Mujoo } else { 548238727786SOjaswin Mujoo write_lock(pa->pa_node_lock.inode_lock); 548338727786SOjaswin Mujoo ei = EXT4_I(pa->pa_inode); 548438727786SOjaswin Mujoo rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); 548538727786SOjaswin Mujoo write_unlock(pa->pa_node_lock.inode_lock); 5486a8e38fd3SOjaswin Mujoo } 5487c9de560dSAlex Tomas 5488c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 548938727786SOjaswin Mujoo 549038727786SOjaswin Mujoo if (pa->pa_type == MB_GROUP_PA) { 549138727786SOjaswin Mujoo ext4_mb_release_group_pa(&e4b, pa); 5492c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 549338727786SOjaswin Mujoo } else { 549438727786SOjaswin Mujoo ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 549538727786SOjaswin Mujoo ext4_mb_pa_free(pa); 549638727786SOjaswin Mujoo } 5497c9de560dSAlex Tomas } 5498c9de560dSAlex Tomas 5499c9de560dSAlex Tomas ext4_unlock_group(sb, group); 5500e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 5501c9de560dSAlex Tomas put_bh(bitmap_bh); 5502bbc4ec77SRitesh Harjani out_dbg: 5503d3df1453SRitesh Harjani mb_debug(sb, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n", 55048c80fb31SChunguang Xu free, group, grp->bb_free); 55058c80fb31SChunguang Xu return free; 5506c9de560dSAlex Tomas } 5507c9de560dSAlex Tomas 5508c9de560dSAlex Tomas /* 5509c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 5510c9de560dSAlex Tomas * 5511c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 5512c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 5513c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 5514c9de560dSAlex Tomas * 5515c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 5516c9de560dSAlex Tomas */ 551727bc446eSbrookxu void ext4_discard_preallocations(struct inode *inode, unsigned int needed) 5518c9de560dSAlex Tomas { 5519c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 5520c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 5521c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 5522c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 5523c9de560dSAlex Tomas ext4_group_t group = 0; 5524c9de560dSAlex Tomas struct list_head list; 5525c9de560dSAlex Tomas struct ext4_buddy e4b; 552638727786SOjaswin Mujoo struct rb_node *iter; 5527c9de560dSAlex Tomas int err; 5528c9de560dSAlex Tomas 5529c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 5530c9de560dSAlex Tomas return; 5531c9de560dSAlex Tomas } 5532c9de560dSAlex Tomas 55338016e29fSHarshad Shirwadkar if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) 55348016e29fSHarshad Shirwadkar return; 55358016e29fSHarshad Shirwadkar 5536d3df1453SRitesh Harjani mb_debug(sb, "discard preallocation for inode %lu\n", 5537d3df1453SRitesh Harjani inode->i_ino); 553827bc446eSbrookxu trace_ext4_discard_preallocations(inode, 553927bc446eSbrookxu atomic_read(&ei->i_prealloc_active), needed); 5540c9de560dSAlex Tomas 5541c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 5542c9de560dSAlex Tomas 554327bc446eSbrookxu if (needed == 0) 554427bc446eSbrookxu needed = UINT_MAX; 554527bc446eSbrookxu 5546c9de560dSAlex Tomas repeat: 5547c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 554838727786SOjaswin Mujoo write_lock(&ei->i_prealloc_lock); 554938727786SOjaswin Mujoo for (iter = rb_first(&ei->i_prealloc_node); iter && needed; 555038727786SOjaswin Mujoo iter = rb_next(iter)) { 555138727786SOjaswin Mujoo pa = rb_entry(iter, struct ext4_prealloc_space, 555238727786SOjaswin Mujoo pa_node.inode_node); 5553a8e38fd3SOjaswin Mujoo BUG_ON(pa->pa_node_lock.inode_lock != &ei->i_prealloc_lock); 555438727786SOjaswin Mujoo 5555c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 5556c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 5557c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 5558c9de560dSAlex Tomas * use preallocation while we're discarding it */ 5559c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 556038727786SOjaswin Mujoo write_unlock(&ei->i_prealloc_lock); 55619d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, 55629d8b9ec4STheodore Ts'o "uh-oh! used pa while discarding"); 5563c9de560dSAlex Tomas WARN_ON(1); 5564c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 5565c9de560dSAlex Tomas goto repeat; 5566c9de560dSAlex Tomas 5567c9de560dSAlex Tomas } 5568c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 556927bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 5570c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 557138727786SOjaswin Mujoo rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); 5572c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 557327bc446eSbrookxu needed--; 5574c9de560dSAlex Tomas continue; 5575c9de560dSAlex Tomas } 5576c9de560dSAlex Tomas 5577c9de560dSAlex Tomas /* someone is deleting pa right now */ 5578c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 557938727786SOjaswin Mujoo write_unlock(&ei->i_prealloc_lock); 5580c9de560dSAlex Tomas 5581c9de560dSAlex Tomas /* we have to wait here because pa_deleted 5582c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 5583c9de560dSAlex Tomas * the list. as we might be called from 5584c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 5585c9de560dSAlex Tomas * and concurrent thread which is unlinking 5586c9de560dSAlex Tomas * pa from inode's list may access already 5587c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 5588c9de560dSAlex Tomas 5589c9de560dSAlex Tomas /* XXX: if this happens too often, we can 5590c9de560dSAlex Tomas * add a flag to force wait only in case 5591c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 5592c9de560dSAlex Tomas * regular truncate */ 5593c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 5594c9de560dSAlex Tomas goto repeat; 5595c9de560dSAlex Tomas } 559638727786SOjaswin Mujoo write_unlock(&ei->i_prealloc_lock); 5597c9de560dSAlex Tomas 5598c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 5599cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 5600bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 5601c9de560dSAlex Tomas 56029651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 56039651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 5604ce89f46cSAneesh Kumar K.V if (err) { 560554d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 56069651e6b2SKonstantin Khlebnikov err, group); 5607ce89f46cSAneesh Kumar K.V continue; 5608ce89f46cSAneesh Kumar K.V } 5609c9de560dSAlex Tomas 5610574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 56119008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 56129008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 561354d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", 56149008a58eSDarrick J. Wong err, group); 5615e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 5616ce89f46cSAneesh Kumar K.V continue; 5617c9de560dSAlex Tomas } 5618c9de560dSAlex Tomas 5619c9de560dSAlex Tomas ext4_lock_group(sb, group); 5620c9de560dSAlex Tomas list_del(&pa->pa_group_list); 56213e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 5622c9de560dSAlex Tomas ext4_unlock_group(sb, group); 5623c9de560dSAlex Tomas 5624e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 5625c9de560dSAlex Tomas put_bh(bitmap_bh); 5626c9de560dSAlex Tomas 5627c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 562838727786SOjaswin Mujoo ext4_mb_pa_free(pa); 5629c9de560dSAlex Tomas } 5630c9de560dSAlex Tomas } 5631c9de560dSAlex Tomas 563253f86b17SRitesh Harjani static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) 563353f86b17SRitesh Harjani { 563453f86b17SRitesh Harjani struct ext4_prealloc_space *pa; 563553f86b17SRitesh Harjani 563653f86b17SRitesh Harjani BUG_ON(ext4_pspace_cachep == NULL); 563753f86b17SRitesh Harjani pa = kmem_cache_zalloc(ext4_pspace_cachep, GFP_NOFS); 563853f86b17SRitesh Harjani if (!pa) 563953f86b17SRitesh Harjani return -ENOMEM; 564053f86b17SRitesh Harjani atomic_set(&pa->pa_count, 1); 564153f86b17SRitesh Harjani ac->ac_pa = pa; 564253f86b17SRitesh Harjani return 0; 564353f86b17SRitesh Harjani } 564453f86b17SRitesh Harjani 564582089725SOjaswin Mujoo static void ext4_mb_pa_put_free(struct ext4_allocation_context *ac) 564653f86b17SRitesh Harjani { 564753f86b17SRitesh Harjani struct ext4_prealloc_space *pa = ac->ac_pa; 564853f86b17SRitesh Harjani 564953f86b17SRitesh Harjani BUG_ON(!pa); 565053f86b17SRitesh Harjani ac->ac_pa = NULL; 565153f86b17SRitesh Harjani WARN_ON(!atomic_dec_and_test(&pa->pa_count)); 565282089725SOjaswin Mujoo /* 565382089725SOjaswin Mujoo * current function is only called due to an error or due to 565482089725SOjaswin Mujoo * len of found blocks < len of requested blocks hence the PA has not 565582089725SOjaswin Mujoo * been added to grp->bb_prealloc_list. So we don't need to lock it 565682089725SOjaswin Mujoo */ 565782089725SOjaswin Mujoo pa->pa_deleted = 1; 565882089725SOjaswin Mujoo ext4_mb_pa_free(pa); 565953f86b17SRitesh Harjani } 566053f86b17SRitesh Harjani 56616ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 5662e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 5663c9de560dSAlex Tomas { 5664e68cf40cSRitesh Harjani ext4_group_t i, ngroups; 5665c9de560dSAlex Tomas 566695257987SJan Kara if (ext4_forced_shutdown(sb)) 5667e3570639SEric Sandeen return; 5668e3570639SEric Sandeen 56698df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 5670d3df1453SRitesh Harjani mb_debug(sb, "groups: "); 56718df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 5672c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 5673c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 5674c9de560dSAlex Tomas ext4_grpblk_t start; 5675c9de560dSAlex Tomas struct list_head *cur; 56765354b2afSTheodore Ts'o 56775354b2afSTheodore Ts'o if (!grp) 56785354b2afSTheodore Ts'o continue; 5679c9de560dSAlex Tomas ext4_lock_group(sb, i); 5680c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 5681c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 5682c9de560dSAlex Tomas pa_group_list); 5683c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 5684c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 5685c9de560dSAlex Tomas NULL, &start); 5686c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5687d3df1453SRitesh Harjani mb_debug(sb, "PA:%u:%d:%d\n", i, start, 5688d3df1453SRitesh Harjani pa->pa_len); 5689c9de560dSAlex Tomas } 569060bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 5691d3df1453SRitesh Harjani mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free, 5692d3df1453SRitesh Harjani grp->bb_fragments); 5693c9de560dSAlex Tomas } 5694c9de560dSAlex Tomas } 5695e68cf40cSRitesh Harjani 5696e68cf40cSRitesh Harjani static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 5697e68cf40cSRitesh Harjani { 5698e68cf40cSRitesh Harjani struct super_block *sb = ac->ac_sb; 5699e68cf40cSRitesh Harjani 570095257987SJan Kara if (ext4_forced_shutdown(sb)) 5701e68cf40cSRitesh Harjani return; 5702e68cf40cSRitesh Harjani 5703d3df1453SRitesh Harjani mb_debug(sb, "Can't allocate:" 5704e68cf40cSRitesh Harjani " Allocation context details:"); 5705d3df1453SRitesh Harjani mb_debug(sb, "status %u flags 0x%x", 5706e68cf40cSRitesh Harjani ac->ac_status, ac->ac_flags); 5707d3df1453SRitesh Harjani mb_debug(sb, "orig %lu/%lu/%lu@%lu, " 5708e68cf40cSRitesh Harjani "goal %lu/%lu/%lu@%lu, " 5709e68cf40cSRitesh Harjani "best %lu/%lu/%lu@%lu cr %d", 5710e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_group, 5711e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_start, 5712e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_len, 5713e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_logical, 5714e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_group, 5715e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_start, 5716e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_len, 5717e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_logical, 5718e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_group, 5719e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_start, 5720e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_len, 5721e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_logical, 5722e68cf40cSRitesh Harjani (int)ac->ac_criteria); 5723d3df1453SRitesh Harjani mb_debug(sb, "%u found", ac->ac_found); 5724569f196fSRitesh Harjani mb_debug(sb, "used pa: %s, ", ac->ac_pa ? "yes" : "no"); 5725569f196fSRitesh Harjani if (ac->ac_pa) 5726569f196fSRitesh Harjani mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ? 5727569f196fSRitesh Harjani "group pa" : "inode pa"); 5728e68cf40cSRitesh Harjani ext4_mb_show_pa(sb); 5729e68cf40cSRitesh Harjani } 5730c9de560dSAlex Tomas #else 5731e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 5732e68cf40cSRitesh Harjani { 5733e68cf40cSRitesh Harjani return; 5734e68cf40cSRitesh Harjani } 5735c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 5736c9de560dSAlex Tomas { 5737e68cf40cSRitesh Harjani ext4_mb_show_pa(ac->ac_sb); 5738c9de560dSAlex Tomas return; 5739c9de560dSAlex Tomas } 5740c9de560dSAlex Tomas #endif 5741c9de560dSAlex Tomas 5742c9de560dSAlex Tomas /* 5743c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 5744c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 5745c9de560dSAlex Tomas * allocation which ever is larger 5746c9de560dSAlex Tomas * 5747b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 5748c9de560dSAlex Tomas */ 5749c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 5750c9de560dSAlex Tomas { 5751c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 5752c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 5753c9de560dSAlex Tomas loff_t size, isize; 5754a9f2a293SJan Kara bool inode_pa_eligible, group_pa_eligible; 5755c9de560dSAlex Tomas 5756c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 5757c9de560dSAlex Tomas return; 5758c9de560dSAlex Tomas 57594ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 57604ba74d00STheodore Ts'o return; 57614ba74d00STheodore Ts'o 5762a9f2a293SJan Kara group_pa_eligible = sbi->s_mb_group_prealloc > 0; 5763a9f2a293SJan Kara inode_pa_eligible = true; 576443bbddc0SBaokun Li size = extent_logical_end(sbi, &ac->ac_o_ex); 576550797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 576650797481STheodore Ts'o >> bsbits; 5767c9de560dSAlex Tomas 5768a9f2a293SJan Kara /* No point in using inode preallocation for closed files */ 576982dd124cSNikolay Borisov if ((size == isize) && !ext4_fs_is_busy(sbi) && 5770a9f2a293SJan Kara !inode_is_open_for_write(ac->ac_inode)) 5771a9f2a293SJan Kara inode_pa_eligible = false; 577250797481STheodore Ts'o 577371780577STheodore Ts'o size = max(size, isize); 5774a9f2a293SJan Kara /* Don't use group allocation for large files */ 5775a9f2a293SJan Kara if (size > sbi->s_mb_stream_request) 5776a9f2a293SJan Kara group_pa_eligible = false; 5777a9f2a293SJan Kara 5778a9f2a293SJan Kara if (!group_pa_eligible) { 5779a9f2a293SJan Kara if (inode_pa_eligible) 57804ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 5781a9f2a293SJan Kara else 5782a9f2a293SJan Kara ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 5783c9de560dSAlex Tomas return; 57844ba74d00STheodore Ts'o } 5785c9de560dSAlex Tomas 5786c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 5787c9de560dSAlex Tomas /* 5788c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 5789c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 5790c9de560dSAlex Tomas * request from multiple CPUs. 5791c9de560dSAlex Tomas */ 5792a0b6bc63SChristoph Lameter ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); 5793c9de560dSAlex Tomas 5794c9de560dSAlex Tomas /* we're going to use group allocation */ 5795c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 5796c9de560dSAlex Tomas 5797c9de560dSAlex Tomas /* serialize all allocations in the group */ 5798c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 5799c9de560dSAlex Tomas } 5800c9de560dSAlex Tomas 5801d73eff68SGuoqing Jiang static noinline_for_stack void 58024ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 5803c9de560dSAlex Tomas struct ext4_allocation_request *ar) 5804c9de560dSAlex Tomas { 5805c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 5806c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 5807c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 5808c9de560dSAlex Tomas ext4_group_t group; 5809498e5f24STheodore Ts'o unsigned int len; 5810498e5f24STheodore Ts'o ext4_fsblk_t goal; 5811c9de560dSAlex Tomas ext4_grpblk_t block; 5812c9de560dSAlex Tomas 5813c9de560dSAlex Tomas /* we can't allocate > group size */ 5814c9de560dSAlex Tomas len = ar->len; 5815c9de560dSAlex Tomas 5816c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 581740ae3487STheodore Ts'o if (len >= EXT4_CLUSTERS_PER_GROUP(sb)) 581840ae3487STheodore Ts'o len = EXT4_CLUSTERS_PER_GROUP(sb); 5819c9de560dSAlex Tomas 5820c9de560dSAlex Tomas /* start searching from the goal */ 5821c9de560dSAlex Tomas goal = ar->goal; 5822c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 5823c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 5824c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 5825c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 5826c9de560dSAlex Tomas 5827c9de560dSAlex Tomas /* set up allocation goals */ 5828f5a44db5STheodore Ts'o ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); 5829c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 5830c9de560dSAlex Tomas ac->ac_sb = sb; 5831c9de560dSAlex Tomas ac->ac_inode = ar->inode; 583253accfa9STheodore Ts'o ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; 5833c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 5834c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 5835c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 583653accfa9STheodore Ts'o ac->ac_g_ex = ac->ac_o_ex; 58377e170922SOjaswin Mujoo ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; 5838c9de560dSAlex Tomas ac->ac_flags = ar->flags; 5839c9de560dSAlex Tomas 58403cb77bd2Sbrookxu /* we have to define context: we'll work with a file or 5841c9de560dSAlex Tomas * locality group. this is a policy, actually */ 5842c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 5843c9de560dSAlex Tomas 5844d3df1453SRitesh Harjani mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " 5845c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 5846c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 5847c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 5848c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 5849c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 585082dd124cSNikolay Borisov inode_is_open_for_write(ar->inode) ? "" : "non-"); 5851c9de560dSAlex Tomas } 5852c9de560dSAlex Tomas 58536be2ded1SAneesh Kumar K.V static noinline_for_stack void 58546be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 58556be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 58566be2ded1SAneesh Kumar K.V int order, int total_entries) 58576be2ded1SAneesh Kumar K.V { 58586be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 58596be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 58606be2ded1SAneesh Kumar K.V struct list_head discard_list; 58616be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 58626be2ded1SAneesh Kumar K.V 5863d3df1453SRitesh Harjani mb_debug(sb, "discard locality group preallocation\n"); 58646be2ded1SAneesh Kumar K.V 58656be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 58666be2ded1SAneesh Kumar K.V 58676be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 58686be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 5869a8e38fd3SOjaswin Mujoo pa_node.lg_list, 587092e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 58716be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 58726be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 58736be2ded1SAneesh Kumar K.V /* 58746be2ded1SAneesh Kumar K.V * This is the pa that we just used 58756be2ded1SAneesh Kumar K.V * for block allocation. So don't 58766be2ded1SAneesh Kumar K.V * free that 58776be2ded1SAneesh Kumar K.V */ 58786be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 58796be2ded1SAneesh Kumar K.V continue; 58806be2ded1SAneesh Kumar K.V } 58816be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 58826be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 58836be2ded1SAneesh Kumar K.V continue; 58846be2ded1SAneesh Kumar K.V } 58856be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 5886cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 58876be2ded1SAneesh Kumar K.V 58886be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 588927bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 58906be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 58916be2ded1SAneesh Kumar K.V 5892a8e38fd3SOjaswin Mujoo list_del_rcu(&pa->pa_node.lg_list); 58936be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 58946be2ded1SAneesh Kumar K.V 58956be2ded1SAneesh Kumar K.V total_entries--; 58966be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 58976be2ded1SAneesh Kumar K.V /* 58986be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 58996be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 59006be2ded1SAneesh Kumar K.V * mak sure we don't call discard 59016be2ded1SAneesh Kumar K.V * soon for this list. 59026be2ded1SAneesh Kumar K.V */ 59036be2ded1SAneesh Kumar K.V break; 59046be2ded1SAneesh Kumar K.V } 59056be2ded1SAneesh Kumar K.V } 59066be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 59076be2ded1SAneesh Kumar K.V 59086be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 59099651e6b2SKonstantin Khlebnikov int err; 59106be2ded1SAneesh Kumar K.V 5911bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 59129651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 59139651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 59149651e6b2SKonstantin Khlebnikov if (err) { 591554d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 59169651e6b2SKonstantin Khlebnikov err, group); 59176be2ded1SAneesh Kumar K.V continue; 59186be2ded1SAneesh Kumar K.V } 59196be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 59206be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 59213e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 59226be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 59236be2ded1SAneesh Kumar K.V 5924e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 59256be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 59266be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 59276be2ded1SAneesh Kumar K.V } 59286be2ded1SAneesh Kumar K.V } 59296be2ded1SAneesh Kumar K.V 59306be2ded1SAneesh Kumar K.V /* 59316be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 59326be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 59336be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 59346be2ded1SAneesh Kumar K.V * 59356be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 59366be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 59376be2ded1SAneesh Kumar K.V */ 59386be2ded1SAneesh Kumar K.V 59396be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 59406be2ded1SAneesh Kumar K.V { 59416be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 59426be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 59436be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 59446be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 59456be2ded1SAneesh Kumar K.V 59466be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 59476be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 59486be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 59496be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 59506be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 5951f1167009SNiu Yawei spin_lock(&lg->lg_prealloc_lock); 59526be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 5953a8e38fd3SOjaswin Mujoo pa_node.lg_list, 595492e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 59556be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 59566be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 5957e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 59586be2ded1SAneesh Kumar K.V continue; 59596be2ded1SAneesh Kumar K.V } 59606be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 59616be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 5962a8e38fd3SOjaswin Mujoo list_add_tail_rcu(&pa->pa_node.lg_list, 5963a8e38fd3SOjaswin Mujoo &tmp_pa->pa_node.lg_list); 59646be2ded1SAneesh Kumar K.V added = 1; 59656be2ded1SAneesh Kumar K.V /* 59666be2ded1SAneesh Kumar K.V * we want to count the total 59676be2ded1SAneesh Kumar K.V * number of entries in the list 59686be2ded1SAneesh Kumar K.V */ 59696be2ded1SAneesh Kumar K.V } 59706be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 59716be2ded1SAneesh Kumar K.V lg_prealloc_count++; 59726be2ded1SAneesh Kumar K.V } 59736be2ded1SAneesh Kumar K.V if (!added) 5974a8e38fd3SOjaswin Mujoo list_add_tail_rcu(&pa->pa_node.lg_list, 59756be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 5976f1167009SNiu Yawei spin_unlock(&lg->lg_prealloc_lock); 59776be2ded1SAneesh Kumar K.V 59786be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 59796be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 59806be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 59816be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 59826be2ded1SAneesh Kumar K.V return; 59836be2ded1SAneesh Kumar K.V } 59846be2ded1SAneesh Kumar K.V return ; 59856be2ded1SAneesh Kumar K.V } 59866be2ded1SAneesh Kumar K.V 5987c9de560dSAlex Tomas /* 5988c9de560dSAlex Tomas * release all resource we used in allocation 5989c9de560dSAlex Tomas */ 5990c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 5991c9de560dSAlex Tomas { 599253accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 59936be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 59946be2ded1SAneesh Kumar K.V if (pa) { 5995cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 5996c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 59976be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 599853accfa9STheodore Ts'o pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 599953accfa9STheodore Ts'o pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 60006be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 60016be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 60026be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 600366d5e027Sbrookxu 60046be2ded1SAneesh Kumar K.V /* 60056be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 60066be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 60076be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 600844183d42SAmir Goldstein * doesn't grow big. 60096be2ded1SAneesh Kumar K.V */ 601066d5e027Sbrookxu if (likely(pa->pa_free)) { 6011a8e38fd3SOjaswin Mujoo spin_lock(pa->pa_node_lock.lg_lock); 6012a8e38fd3SOjaswin Mujoo list_del_rcu(&pa->pa_node.lg_list); 6013a8e38fd3SOjaswin Mujoo spin_unlock(pa->pa_node_lock.lg_lock); 60146be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 6015c9de560dSAlex Tomas } 601666d5e027Sbrookxu } 601727bc446eSbrookxu 60186be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 6019c9de560dSAlex Tomas } 6020c9de560dSAlex Tomas if (ac->ac_bitmap_page) 602109cbfeafSKirill A. Shutemov put_page(ac->ac_bitmap_page); 6022c9de560dSAlex Tomas if (ac->ac_buddy_page) 602309cbfeafSKirill A. Shutemov put_page(ac->ac_buddy_page); 6024c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 6025c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 6026c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 6027c9de560dSAlex Tomas return 0; 6028c9de560dSAlex Tomas } 6029c9de560dSAlex Tomas 6030c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 6031c9de560dSAlex Tomas { 60328df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 6033c9de560dSAlex Tomas int ret; 60348c80fb31SChunguang Xu int freed = 0, busy = 0; 60358c80fb31SChunguang Xu int retry = 0; 6036c9de560dSAlex Tomas 60379bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 60388c80fb31SChunguang Xu 60398c80fb31SChunguang Xu if (needed == 0) 60408c80fb31SChunguang Xu needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; 60418c80fb31SChunguang Xu repeat: 60428df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 60438c80fb31SChunguang Xu ret = ext4_mb_discard_group_preallocations(sb, i, &busy); 6044c9de560dSAlex Tomas freed += ret; 6045c9de560dSAlex Tomas needed -= ret; 60468c80fb31SChunguang Xu cond_resched(); 60478c80fb31SChunguang Xu } 60488c80fb31SChunguang Xu 60498c80fb31SChunguang Xu if (needed > 0 && busy && ++retry < 3) { 60508c80fb31SChunguang Xu busy = 0; 60518c80fb31SChunguang Xu goto repeat; 6052c9de560dSAlex Tomas } 6053c9de560dSAlex Tomas 6054c9de560dSAlex Tomas return freed; 6055c9de560dSAlex Tomas } 6056c9de560dSAlex Tomas 6057cf5e2ca6SRitesh Harjani static bool ext4_mb_discard_preallocations_should_retry(struct super_block *sb, 605807b5b8e1SRitesh Harjani struct ext4_allocation_context *ac, u64 *seq) 6059cf5e2ca6SRitesh Harjani { 6060cf5e2ca6SRitesh Harjani int freed; 606107b5b8e1SRitesh Harjani u64 seq_retry = 0; 606207b5b8e1SRitesh Harjani bool ret = false; 6063cf5e2ca6SRitesh Harjani 6064cf5e2ca6SRitesh Harjani freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 606507b5b8e1SRitesh Harjani if (freed) { 606607b5b8e1SRitesh Harjani ret = true; 606707b5b8e1SRitesh Harjani goto out_dbg; 606807b5b8e1SRitesh Harjani } 606907b5b8e1SRitesh Harjani seq_retry = ext4_get_discard_pa_seq_sum(); 607099377830SRitesh Harjani if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { 607199377830SRitesh Harjani ac->ac_flags |= EXT4_MB_STRICT_CHECK; 607207b5b8e1SRitesh Harjani *seq = seq_retry; 607307b5b8e1SRitesh Harjani ret = true; 607407b5b8e1SRitesh Harjani } 607507b5b8e1SRitesh Harjani 607607b5b8e1SRitesh Harjani out_dbg: 607707b5b8e1SRitesh Harjani mb_debug(sb, "freed %d, retry ? %s\n", freed, ret ? "yes" : "no"); 607807b5b8e1SRitesh Harjani return ret; 6079cf5e2ca6SRitesh Harjani } 6080cf5e2ca6SRitesh Harjani 6081ad78b5efSKemeng Shi /* 6082ad78b5efSKemeng Shi * Simple allocator for Ext4 fast commit replay path. It searches for blocks 6083ad78b5efSKemeng Shi * linearly starting at the goal block and also excludes the blocks which 6084ad78b5efSKemeng Shi * are going to be in use after fast commit replay. 6085ad78b5efSKemeng Shi */ 6086ad78b5efSKemeng Shi static ext4_fsblk_t 6087ad78b5efSKemeng Shi ext4_mb_new_blocks_simple(struct ext4_allocation_request *ar, int *errp) 6088ad78b5efSKemeng Shi { 6089ad78b5efSKemeng Shi struct buffer_head *bitmap_bh; 6090ad78b5efSKemeng Shi struct super_block *sb = ar->inode->i_sb; 6091ad78b5efSKemeng Shi struct ext4_sb_info *sbi = EXT4_SB(sb); 6092ad78b5efSKemeng Shi ext4_group_t group, nr; 6093ad78b5efSKemeng Shi ext4_grpblk_t blkoff; 6094ad78b5efSKemeng Shi ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 6095ad78b5efSKemeng Shi ext4_grpblk_t i = 0; 6096ad78b5efSKemeng Shi ext4_fsblk_t goal, block; 6097ad78b5efSKemeng Shi struct ext4_super_block *es = EXT4_SB(sb)->s_es; 6098ad78b5efSKemeng Shi 6099ad78b5efSKemeng Shi goal = ar->goal; 6100ad78b5efSKemeng Shi if (goal < le32_to_cpu(es->s_first_data_block) || 6101ad78b5efSKemeng Shi goal >= ext4_blocks_count(es)) 6102ad78b5efSKemeng Shi goal = le32_to_cpu(es->s_first_data_block); 6103ad78b5efSKemeng Shi 6104ad78b5efSKemeng Shi ar->len = 0; 6105ad78b5efSKemeng Shi ext4_get_group_no_and_offset(sb, goal, &group, &blkoff); 6106ad78b5efSKemeng Shi for (nr = ext4_get_groups_count(sb); nr > 0; nr--) { 6107ad78b5efSKemeng Shi bitmap_bh = ext4_read_block_bitmap(sb, group); 6108ad78b5efSKemeng Shi if (IS_ERR(bitmap_bh)) { 6109ad78b5efSKemeng Shi *errp = PTR_ERR(bitmap_bh); 6110ad78b5efSKemeng Shi pr_warn("Failed to read block bitmap\n"); 6111ad78b5efSKemeng Shi return 0; 6112ad78b5efSKemeng Shi } 6113ad78b5efSKemeng Shi 6114ad78b5efSKemeng Shi while (1) { 6115ad78b5efSKemeng Shi i = mb_find_next_zero_bit(bitmap_bh->b_data, max, 6116ad78b5efSKemeng Shi blkoff); 6117ad78b5efSKemeng Shi if (i >= max) 6118ad78b5efSKemeng Shi break; 6119ad78b5efSKemeng Shi if (ext4_fc_replay_check_excluded(sb, 6120ad78b5efSKemeng Shi ext4_group_first_block_no(sb, group) + 6121ad78b5efSKemeng Shi EXT4_C2B(sbi, i))) { 6122ad78b5efSKemeng Shi blkoff = i + 1; 6123ad78b5efSKemeng Shi } else 6124ad78b5efSKemeng Shi break; 6125ad78b5efSKemeng Shi } 6126ad78b5efSKemeng Shi brelse(bitmap_bh); 6127ad78b5efSKemeng Shi if (i < max) 6128ad78b5efSKemeng Shi break; 6129ad78b5efSKemeng Shi 6130ad78b5efSKemeng Shi if (++group >= ext4_get_groups_count(sb)) 6131ad78b5efSKemeng Shi group = 0; 6132ad78b5efSKemeng Shi 6133ad78b5efSKemeng Shi blkoff = 0; 6134ad78b5efSKemeng Shi } 6135ad78b5efSKemeng Shi 6136ad78b5efSKemeng Shi if (i >= max) { 6137ad78b5efSKemeng Shi *errp = -ENOSPC; 6138ad78b5efSKemeng Shi return 0; 6139ad78b5efSKemeng Shi } 6140ad78b5efSKemeng Shi 6141ad78b5efSKemeng Shi block = ext4_group_first_block_no(sb, group) + EXT4_C2B(sbi, i); 6142ad78b5efSKemeng Shi ext4_mb_mark_bb(sb, block, 1, 1); 6143ad78b5efSKemeng Shi ar->len = 1; 6144ad78b5efSKemeng Shi 6145ad78b5efSKemeng Shi return block; 6146ad78b5efSKemeng Shi } 61478016e29fSHarshad Shirwadkar 6148c9de560dSAlex Tomas /* 6149c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 6150c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 6151c9de560dSAlex Tomas * to usual allocation 6152c9de560dSAlex Tomas */ 6153c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 6154c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 6155c9de560dSAlex Tomas { 6156256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 6157c9de560dSAlex Tomas struct ext4_sb_info *sbi; 6158c9de560dSAlex Tomas struct super_block *sb; 6159c9de560dSAlex Tomas ext4_fsblk_t block = 0; 616060e58e0fSMingming Cao unsigned int inquota = 0; 616153accfa9STheodore Ts'o unsigned int reserv_clstrs = 0; 616280fa46d6STheodore Ts'o int retries = 0; 616307b5b8e1SRitesh Harjani u64 seq; 6164c9de560dSAlex Tomas 6165b10a44c3STheodore Ts'o might_sleep(); 6166c9de560dSAlex Tomas sb = ar->inode->i_sb; 6167c9de560dSAlex Tomas sbi = EXT4_SB(sb); 6168c9de560dSAlex Tomas 61699bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 61708016e29fSHarshad Shirwadkar if (sbi->s_mount_state & EXT4_FC_REPLAY) 6171ad78b5efSKemeng Shi return ext4_mb_new_blocks_simple(ar, errp); 6172ba80b101STheodore Ts'o 617345dc63e7SDmitry Monakhov /* Allow to use superuser reservation for quota file */ 617402749a4cSTahsin Erdogan if (ext4_is_quota_file(ar->inode)) 617545dc63e7SDmitry Monakhov ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; 617645dc63e7SDmitry Monakhov 6177e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) { 617860e58e0fSMingming Cao /* Without delayed allocation we need to verify 617960e58e0fSMingming Cao * there is enough free blocks to do block allocation 618060e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 6181d2a17637SMingming Cao */ 618255f020dbSAllison Henderson while (ar->len && 6183e7d5f315STheodore Ts'o ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { 618455f020dbSAllison Henderson 6185030ba6bcSAneesh Kumar K.V /* let others to free the space */ 6186bb8b20edSLukas Czerner cond_resched(); 6187030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 6188030ba6bcSAneesh Kumar K.V } 6189030ba6bcSAneesh Kumar K.V if (!ar->len) { 6190bbc4ec77SRitesh Harjani ext4_mb_show_pa(sb); 619107031431SMingming Cao *errp = -ENOSPC; 619207031431SMingming Cao return 0; 619307031431SMingming Cao } 619453accfa9STheodore Ts'o reserv_clstrs = ar->len; 619555f020dbSAllison Henderson if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { 619653accfa9STheodore Ts'o dquot_alloc_block_nofail(ar->inode, 619753accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len)); 619855f020dbSAllison Henderson } else { 619955f020dbSAllison Henderson while (ar->len && 620053accfa9STheodore Ts'o dquot_alloc_block(ar->inode, 620153accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len))) { 620255f020dbSAllison Henderson 6203c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 6204c9de560dSAlex Tomas ar->len--; 6205c9de560dSAlex Tomas } 620655f020dbSAllison Henderson } 620760e58e0fSMingming Cao inquota = ar->len; 6208c9de560dSAlex Tomas if (ar->len == 0) { 6209c9de560dSAlex Tomas *errp = -EDQUOT; 62106c7a120aSAditya Kali goto out; 6211c9de560dSAlex Tomas } 621260e58e0fSMingming Cao } 6213d2a17637SMingming Cao 621485556c9aSWei Yongjun ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); 6215833576b3STheodore Ts'o if (!ac) { 6216363d4251SShen Feng ar->len = 0; 6217256bdb49SEric Sandeen *errp = -ENOMEM; 62186c7a120aSAditya Kali goto out; 6219256bdb49SEric Sandeen } 6220256bdb49SEric Sandeen 6221d73eff68SGuoqing Jiang ext4_mb_initialize_context(ac, ar); 6222c9de560dSAlex Tomas 6223256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 622481198536SRitesh Harjani seq = this_cpu_read(discard_pa_seq); 6225256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 6226256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 6227256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 622853f86b17SRitesh Harjani 622953f86b17SRitesh Harjani *errp = ext4_mb_pa_alloc(ac); 623053f86b17SRitesh Harjani if (*errp) 623153f86b17SRitesh Harjani goto errout; 6232c9de560dSAlex Tomas repeat: 6233c9de560dSAlex Tomas /* allocate space in core */ 62346c7a120aSAditya Kali *errp = ext4_mb_regular_allocator(ac); 623553f86b17SRitesh Harjani /* 623653f86b17SRitesh Harjani * pa allocated above is added to grp->bb_prealloc_list only 623753f86b17SRitesh Harjani * when we were able to allocate some block i.e. when 623853f86b17SRitesh Harjani * ac->ac_status == AC_STATUS_FOUND. 623953f86b17SRitesh Harjani * And error from above mean ac->ac_status != AC_STATUS_FOUND 624053f86b17SRitesh Harjani * So we have to free this pa here itself. 624153f86b17SRitesh Harjani */ 62422c00ef3eSAlexey Khoroshilov if (*errp) { 624382089725SOjaswin Mujoo ext4_mb_pa_put_free(ac); 62442c00ef3eSAlexey Khoroshilov ext4_discard_allocated_blocks(ac); 62452c00ef3eSAlexey Khoroshilov goto errout; 62462c00ef3eSAlexey Khoroshilov } 624753f86b17SRitesh Harjani if (ac->ac_status == AC_STATUS_FOUND && 624853f86b17SRitesh Harjani ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) 624982089725SOjaswin Mujoo ext4_mb_pa_put_free(ac); 6250c9de560dSAlex Tomas } 6251256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 625253accfa9STheodore Ts'o *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); 6253554a5cccSVegard Nossum if (*errp) { 6254b844167eSCurt Wohlgemuth ext4_discard_allocated_blocks(ac); 62556d138cedSEric Sandeen goto errout; 62566d138cedSEric Sandeen } else { 6257256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 6258256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 6259519deca0SAneesh Kumar K.V } 6260c9de560dSAlex Tomas } else { 626180fa46d6STheodore Ts'o if (++retries < 3 && 626280fa46d6STheodore Ts'o ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) 6263c9de560dSAlex Tomas goto repeat; 626453f86b17SRitesh Harjani /* 626553f86b17SRitesh Harjani * If block allocation fails then the pa allocated above 626653f86b17SRitesh Harjani * needs to be freed here itself. 626753f86b17SRitesh Harjani */ 626882089725SOjaswin Mujoo ext4_mb_pa_put_free(ac); 6269c9de560dSAlex Tomas *errp = -ENOSPC; 62706c7a120aSAditya Kali } 62716c7a120aSAditya Kali 62726c7a120aSAditya Kali if (*errp) { 6273aaae558dSKemeng Shi errout: 6274256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 6275c9de560dSAlex Tomas ar->len = 0; 6276256bdb49SEric Sandeen ext4_mb_show_ac(ac); 6277c9de560dSAlex Tomas } 6278256bdb49SEric Sandeen ext4_mb_release_context(ac); 6279363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 6280aaae558dSKemeng Shi out: 628160e58e0fSMingming Cao if (inquota && ar->len < inquota) 628253accfa9STheodore Ts'o dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); 62830087d9fbSAneesh Kumar K.V if (!ar->len) { 6284e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) 62850087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 628657042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 628753accfa9STheodore Ts'o reserv_clstrs); 62880087d9fbSAneesh Kumar K.V } 6289c9de560dSAlex Tomas 62909bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 6291ba80b101STheodore Ts'o 6292c9de560dSAlex Tomas return block; 6293c9de560dSAlex Tomas } 6294c9de560dSAlex Tomas 6295c894058dSAneesh Kumar K.V /* 6296c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 6297c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 6298c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 6299c894058dSAneesh Kumar K.V */ 6300a0154344SDaeho Jeong static void ext4_try_merge_freed_extent(struct ext4_sb_info *sbi, 6301a0154344SDaeho Jeong struct ext4_free_data *entry, 6302a0154344SDaeho Jeong struct ext4_free_data *new_entry, 6303a0154344SDaeho Jeong struct rb_root *entry_rb_root) 6304c894058dSAneesh Kumar K.V { 6305a0154344SDaeho Jeong if ((entry->efd_tid != new_entry->efd_tid) || 6306a0154344SDaeho Jeong (entry->efd_group != new_entry->efd_group)) 6307a0154344SDaeho Jeong return; 6308a0154344SDaeho Jeong if (entry->efd_start_cluster + entry->efd_count == 6309a0154344SDaeho Jeong new_entry->efd_start_cluster) { 6310a0154344SDaeho Jeong new_entry->efd_start_cluster = entry->efd_start_cluster; 6311a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 6312a0154344SDaeho Jeong } else if (new_entry->efd_start_cluster + new_entry->efd_count == 6313a0154344SDaeho Jeong entry->efd_start_cluster) { 6314a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 6315a0154344SDaeho Jeong } else 6316a0154344SDaeho Jeong return; 6317a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 6318a0154344SDaeho Jeong list_del(&entry->efd_list); 6319a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 6320a0154344SDaeho Jeong rb_erase(&entry->efd_node, entry_rb_root); 6321a0154344SDaeho Jeong kmem_cache_free(ext4_free_data_cachep, entry); 6322c894058dSAneesh Kumar K.V } 6323c894058dSAneesh Kumar K.V 632485b67ffbSKemeng Shi static noinline_for_stack void 63254ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 63267a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 6327c9de560dSAlex Tomas { 6328e29136f8STheodore Ts'o ext4_group_t group = e4b->bd_group; 632984130193STheodore Ts'o ext4_grpblk_t cluster; 6330d08854f5STheodore Ts'o ext4_grpblk_t clusters = new_entry->efd_count; 63317a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 6332c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 6333c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 6334c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 6335c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 6336c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 6337c894058dSAneesh Kumar K.V 63380390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 6339c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 6340c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 6341c9de560dSAlex Tomas 634218aadd47SBobi Jam new_node = &new_entry->efd_node; 634318aadd47SBobi Jam cluster = new_entry->efd_start_cluster; 6344c9de560dSAlex Tomas 6345c894058dSAneesh Kumar K.V if (!*n) { 6346c894058dSAneesh Kumar K.V /* first free block exent. We need to 6347c894058dSAneesh Kumar K.V protect buddy cache from being freed, 6348c9de560dSAlex Tomas * otherwise we'll refresh it from 6349c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 6350c9de560dSAlex Tomas * blocks */ 635109cbfeafSKirill A. Shutemov get_page(e4b->bd_buddy_page); 635209cbfeafSKirill A. Shutemov get_page(e4b->bd_bitmap_page); 6353c894058dSAneesh Kumar K.V } 6354c894058dSAneesh Kumar K.V while (*n) { 6355c894058dSAneesh Kumar K.V parent = *n; 635618aadd47SBobi Jam entry = rb_entry(parent, struct ext4_free_data, efd_node); 635718aadd47SBobi Jam if (cluster < entry->efd_start_cluster) 6358c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 635918aadd47SBobi Jam else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) 6360c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 6361c894058dSAneesh Kumar K.V else { 6362e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 636384130193STheodore Ts'o ext4_group_first_block_no(sb, group) + 636484130193STheodore Ts'o EXT4_C2B(sbi, cluster), 6365e29136f8STheodore Ts'o "Block already on to-be-freed list"); 6366cca41553SChunguang Xu kmem_cache_free(ext4_free_data_cachep, new_entry); 636785b67ffbSKemeng Shi return; 6368c9de560dSAlex Tomas } 6369c9de560dSAlex Tomas } 6370c9de560dSAlex Tomas 6371c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 6372c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 6373c894058dSAneesh Kumar K.V 6374c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 6375c894058dSAneesh Kumar K.V node = rb_prev(new_node); 6376c894058dSAneesh Kumar K.V if (node) { 637718aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 6378a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 6379a0154344SDaeho Jeong &(db->bb_free_root)); 6380c9de560dSAlex Tomas } 6381c894058dSAneesh Kumar K.V 6382c894058dSAneesh Kumar K.V node = rb_next(new_node); 6383c894058dSAneesh Kumar K.V if (node) { 638418aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 6385a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 6386a0154344SDaeho Jeong &(db->bb_free_root)); 6387c894058dSAneesh Kumar K.V } 6388a0154344SDaeho Jeong 6389d08854f5STheodore Ts'o spin_lock(&sbi->s_md_lock); 6390a0154344SDaeho Jeong list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list); 6391d08854f5STheodore Ts'o sbi->s_mb_free_pending += clusters; 6392d08854f5STheodore Ts'o spin_unlock(&sbi->s_md_lock); 6393c9de560dSAlex Tomas } 6394c9de560dSAlex Tomas 63958016e29fSHarshad Shirwadkar static void ext4_free_blocks_simple(struct inode *inode, ext4_fsblk_t block, 63968016e29fSHarshad Shirwadkar unsigned long count) 63978016e29fSHarshad Shirwadkar { 63988016e29fSHarshad Shirwadkar struct buffer_head *bitmap_bh; 63998016e29fSHarshad Shirwadkar struct super_block *sb = inode->i_sb; 64008016e29fSHarshad Shirwadkar struct ext4_group_desc *gdp; 64018016e29fSHarshad Shirwadkar struct buffer_head *gdp_bh; 64028016e29fSHarshad Shirwadkar ext4_group_t group; 64038016e29fSHarshad Shirwadkar ext4_grpblk_t blkoff; 64048016e29fSHarshad Shirwadkar int already_freed = 0, err, i; 64058016e29fSHarshad Shirwadkar 64068016e29fSHarshad Shirwadkar ext4_get_group_no_and_offset(sb, block, &group, &blkoff); 64078016e29fSHarshad Shirwadkar bitmap_bh = ext4_read_block_bitmap(sb, group); 64088016e29fSHarshad Shirwadkar if (IS_ERR(bitmap_bh)) { 64098016e29fSHarshad Shirwadkar pr_warn("Failed to read block bitmap\n"); 64108016e29fSHarshad Shirwadkar return; 64118016e29fSHarshad Shirwadkar } 64128016e29fSHarshad Shirwadkar gdp = ext4_get_group_desc(sb, group, &gdp_bh); 64138016e29fSHarshad Shirwadkar if (!gdp) 64141b5c9d34SKemeng Shi goto err_out; 64158016e29fSHarshad Shirwadkar 64168016e29fSHarshad Shirwadkar for (i = 0; i < count; i++) { 64178016e29fSHarshad Shirwadkar if (!mb_test_bit(blkoff + i, bitmap_bh->b_data)) 64188016e29fSHarshad Shirwadkar already_freed++; 64198016e29fSHarshad Shirwadkar } 64208016e29fSHarshad Shirwadkar mb_clear_bits(bitmap_bh->b_data, blkoff, count); 64218016e29fSHarshad Shirwadkar err = ext4_handle_dirty_metadata(NULL, NULL, bitmap_bh); 64228016e29fSHarshad Shirwadkar if (err) 64231b5c9d34SKemeng Shi goto err_out; 64248016e29fSHarshad Shirwadkar ext4_free_group_clusters_set( 64258016e29fSHarshad Shirwadkar sb, gdp, ext4_free_group_clusters(sb, gdp) + 64268016e29fSHarshad Shirwadkar count - already_freed); 64271df9bde4SKemeng Shi ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh); 64288016e29fSHarshad Shirwadkar ext4_group_desc_csum_set(sb, group, gdp); 64298016e29fSHarshad Shirwadkar ext4_handle_dirty_metadata(NULL, NULL, gdp_bh); 64308016e29fSHarshad Shirwadkar sync_dirty_buffer(bitmap_bh); 64318016e29fSHarshad Shirwadkar sync_dirty_buffer(gdp_bh); 64321b5c9d34SKemeng Shi 64331b5c9d34SKemeng Shi err_out: 64348016e29fSHarshad Shirwadkar brelse(bitmap_bh); 64358016e29fSHarshad Shirwadkar } 64368016e29fSHarshad Shirwadkar 643744338711STheodore Ts'o /** 64388ac3939dSRitesh Harjani * ext4_mb_clear_bb() -- helper function for freeing blocks. 64398ac3939dSRitesh Harjani * Used by ext4_free_blocks() 644044338711STheodore Ts'o * @handle: handle for this transaction 644144338711STheodore Ts'o * @inode: inode 6442c60990b3STheodore Ts'o * @block: starting physical block to be freed 6443c60990b3STheodore Ts'o * @count: number of blocks to be freed 64445def1360SYongqiang Yang * @flags: flags used by ext4_free_blocks 6445c9de560dSAlex Tomas */ 64468ac3939dSRitesh Harjani static void ext4_mb_clear_bb(handle_t *handle, struct inode *inode, 64478ac3939dSRitesh Harjani ext4_fsblk_t block, unsigned long count, 64488ac3939dSRitesh Harjani int flags) 6449c9de560dSAlex Tomas { 645026346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 6451c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 6452c9de560dSAlex Tomas struct ext4_group_desc *gdp; 64535354b2afSTheodore Ts'o struct ext4_group_info *grp; 6454498e5f24STheodore Ts'o unsigned int overflow; 6455c9de560dSAlex Tomas ext4_grpblk_t bit; 6456c9de560dSAlex Tomas struct buffer_head *gd_bh; 6457c9de560dSAlex Tomas ext4_group_t block_group; 6458c9de560dSAlex Tomas struct ext4_sb_info *sbi; 6459c9de560dSAlex Tomas struct ext4_buddy e4b; 646084130193STheodore Ts'o unsigned int count_clusters; 6461c9de560dSAlex Tomas int err = 0; 6462c9de560dSAlex Tomas int ret; 6463c9de560dSAlex Tomas 64648016e29fSHarshad Shirwadkar sbi = EXT4_SB(sb); 64658016e29fSHarshad Shirwadkar 64661e1c2b86SLukas Czerner if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 64671e1c2b86SLukas Czerner !ext4_inode_block_valid(inode, block, count)) { 64681e1c2b86SLukas Czerner ext4_error(sb, "Freeing blocks in system zone - " 64691e1c2b86SLukas Czerner "Block = %llu, count = %lu", block, count); 64701e1c2b86SLukas Czerner /* err = 0. ext4_std_error should be a no op */ 64711e1c2b86SLukas Czerner goto error_return; 64721e1c2b86SLukas Czerner } 64731e1c2b86SLukas Czerner flags |= EXT4_FREE_BLOCKS_VALIDATED; 64741e1c2b86SLukas Czerner 6475c9de560dSAlex Tomas do_more: 6476c9de560dSAlex Tomas overflow = 0; 6477c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 6478c9de560dSAlex Tomas 64795354b2afSTheodore Ts'o grp = ext4_get_group_info(sb, block_group); 64805354b2afSTheodore Ts'o if (unlikely(!grp || EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 6481163a203dSDarrick J. Wong return; 6482163a203dSDarrick J. Wong 6483c9de560dSAlex Tomas /* 6484c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 6485c9de560dSAlex Tomas * boundary. 6486c9de560dSAlex Tomas */ 648784130193STheodore Ts'o if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { 648884130193STheodore Ts'o overflow = EXT4_C2B(sbi, bit) + count - 648984130193STheodore Ts'o EXT4_BLOCKS_PER_GROUP(sb); 6490c9de560dSAlex Tomas count -= overflow; 64911e1c2b86SLukas Czerner /* The range changed so it's no longer validated */ 64921e1c2b86SLukas Czerner flags &= ~EXT4_FREE_BLOCKS_VALIDATED; 6493c9de560dSAlex Tomas } 6494810da240SLukas Czerner count_clusters = EXT4_NUM_B2C(sbi, count); 6495574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 64969008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 64979008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 64989008a58eSDarrick J. Wong bitmap_bh = NULL; 6499c9de560dSAlex Tomas goto error_return; 6500ce89f46cSAneesh Kumar K.V } 6501c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 6502ce89f46cSAneesh Kumar K.V if (!gdp) { 6503ce89f46cSAneesh Kumar K.V err = -EIO; 6504c9de560dSAlex Tomas goto error_return; 6505ce89f46cSAneesh Kumar K.V } 6506c9de560dSAlex Tomas 65071e1c2b86SLukas Czerner if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 65081e1c2b86SLukas Czerner !ext4_inode_block_valid(inode, block, count)) { 650912062dddSEric Sandeen ext4_error(sb, "Freeing blocks in system zone - " 65100610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 6511519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 6512519deca0SAneesh Kumar K.V goto error_return; 6513c9de560dSAlex Tomas } 6514c9de560dSAlex Tomas 6515c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 6516188c299eSJan Kara err = ext4_journal_get_write_access(handle, sb, bitmap_bh, 6517188c299eSJan Kara EXT4_JTR_NONE); 6518c9de560dSAlex Tomas if (err) 6519c9de560dSAlex Tomas goto error_return; 6520c9de560dSAlex Tomas 6521c9de560dSAlex Tomas /* 6522c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 6523c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 6524c9de560dSAlex Tomas * using it 6525c9de560dSAlex Tomas */ 6526c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 6527188c299eSJan Kara err = ext4_journal_get_write_access(handle, sb, gd_bh, EXT4_JTR_NONE); 6528c9de560dSAlex Tomas if (err) 6529c9de560dSAlex Tomas goto error_return; 6530c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 6531c9de560dSAlex Tomas { 6532c9de560dSAlex Tomas int i; 653384130193STheodore Ts'o for (i = 0; i < count_clusters; i++) 6534c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 6535c9de560dSAlex Tomas } 6536c9de560dSAlex Tomas #endif 653784130193STheodore Ts'o trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters); 6538c9de560dSAlex Tomas 6539adb7ef60SKonstantin Khlebnikov /* __GFP_NOFAIL: retry infinitely, ignore TIF_MEMDIE and memcg limit. */ 6540adb7ef60SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b, 6541adb7ef60SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 6542920313a7SAneesh Kumar K.V if (err) 6543920313a7SAneesh Kumar K.V goto error_return; 6544e6362609STheodore Ts'o 6545f96c450dSDaeho Jeong /* 6546f96c450dSDaeho Jeong * We need to make sure we don't reuse the freed block until after the 6547f96c450dSDaeho Jeong * transaction is committed. We make an exception if the inode is to be 6548f96c450dSDaeho Jeong * written in writeback mode since writeback mode has weak data 6549f96c450dSDaeho Jeong * consistency guarantees. 6550f96c450dSDaeho Jeong */ 6551f96c450dSDaeho Jeong if (ext4_handle_valid(handle) && 6552f96c450dSDaeho Jeong ((flags & EXT4_FREE_BLOCKS_METADATA) || 6553f96c450dSDaeho Jeong !ext4_should_writeback_data(inode))) { 65547a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 65557a2fcbf7SAneesh Kumar K.V /* 65567444a072SMichal Hocko * We use __GFP_NOFAIL because ext4_free_blocks() is not allowed 65577444a072SMichal Hocko * to fail. 65587a2fcbf7SAneesh Kumar K.V */ 65597444a072SMichal Hocko new_entry = kmem_cache_alloc(ext4_free_data_cachep, 65607444a072SMichal Hocko GFP_NOFS|__GFP_NOFAIL); 656118aadd47SBobi Jam new_entry->efd_start_cluster = bit; 656218aadd47SBobi Jam new_entry->efd_group = block_group; 656318aadd47SBobi Jam new_entry->efd_count = count_clusters; 656418aadd47SBobi Jam new_entry->efd_tid = handle->h_transaction->t_tid; 6565955ce5f5SAneesh Kumar K.V 65667a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 656784130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 65687a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 6569c9de560dSAlex Tomas } else { 65707a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 65717a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 65727a2fcbf7SAneesh Kumar K.V * them with group lock_held 65737a2fcbf7SAneesh Kumar K.V */ 6574d71c1ae2SLukas Czerner if (test_opt(sb, DISCARD)) { 6575247c3d21SKemeng Shi err = ext4_issue_discard(sb, block_group, bit, 6576247c3d21SKemeng Shi count_clusters, NULL); 6577d71c1ae2SLukas Czerner if (err && err != -EOPNOTSUPP) 6578d71c1ae2SLukas Czerner ext4_msg(sb, KERN_WARNING, "discard request in" 6579a00b482bSRitesh Harjani " group:%u block:%d count:%lu failed" 6580d71c1ae2SLukas Czerner " with %d", block_group, bit, count, 6581d71c1ae2SLukas Czerner err); 65828f9ff189SLukas Czerner } else 65838f9ff189SLukas Czerner EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info); 6584d71c1ae2SLukas Czerner 6585955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 658684130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 658784130193STheodore Ts'o mb_free_blocks(inode, &e4b, bit, count_clusters); 6588c9de560dSAlex Tomas } 6589c9de560dSAlex Tomas 6590021b65bbSTheodore Ts'o ret = ext4_free_group_clusters(sb, gdp) + count_clusters; 6591021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, ret); 65921df9bde4SKemeng Shi ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh); 6593feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, gdp); 6594955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 6595c9de560dSAlex Tomas 6596772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 6597772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 659890ba983fSTheodore Ts'o atomic64_add(count_clusters, 65997c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 66007c990728SSuraj Jitindar Singh flex_group)->free_clusters); 6601772cb7c8SJose R. Santos } 6602772cb7c8SJose R. Santos 66039fe67149SEric Whitney /* 66049fe67149SEric Whitney * on a bigalloc file system, defer the s_freeclusters_counter 66059fe67149SEric Whitney * update to the caller (ext4_remove_space and friends) so they 66069fe67149SEric Whitney * can determine if a cluster freed here should be rereserved 66079fe67149SEric Whitney */ 66089fe67149SEric Whitney if (!(flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)) { 66097b415bf6SAditya Kali if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 66107b415bf6SAditya Kali dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); 66119fe67149SEric Whitney percpu_counter_add(&sbi->s_freeclusters_counter, 66129fe67149SEric Whitney count_clusters); 66139fe67149SEric Whitney } 66147d734532SJan Kara 66157d734532SJan Kara ext4_mb_unload_buddy(&e4b); 66167b415bf6SAditya Kali 66177a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 66187a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 66197a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 66207a2fcbf7SAneesh Kumar K.V 6621c9de560dSAlex Tomas /* And the group descriptor block */ 6622c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 66230390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 6624c9de560dSAlex Tomas if (!err) 6625c9de560dSAlex Tomas err = ret; 6626c9de560dSAlex Tomas 6627c9de560dSAlex Tomas if (overflow && !err) { 6628c9de560dSAlex Tomas block += count; 6629c9de560dSAlex Tomas count = overflow; 6630c9de560dSAlex Tomas put_bh(bitmap_bh); 66311e1c2b86SLukas Czerner /* The range changed so it's no longer validated */ 66321e1c2b86SLukas Czerner flags &= ~EXT4_FREE_BLOCKS_VALIDATED; 6633c9de560dSAlex Tomas goto do_more; 6634c9de560dSAlex Tomas } 6635c9de560dSAlex Tomas error_return: 6636c9de560dSAlex Tomas brelse(bitmap_bh); 6637c9de560dSAlex Tomas ext4_std_error(sb, err); 6638c9de560dSAlex Tomas return; 6639c9de560dSAlex Tomas } 66407360d173SLukas Czerner 66417360d173SLukas Czerner /** 66428ac3939dSRitesh Harjani * ext4_free_blocks() -- Free given blocks and update quota 66438ac3939dSRitesh Harjani * @handle: handle for this transaction 66448ac3939dSRitesh Harjani * @inode: inode 66458ac3939dSRitesh Harjani * @bh: optional buffer of the block to be freed 66468ac3939dSRitesh Harjani * @block: starting physical block to be freed 66478ac3939dSRitesh Harjani * @count: number of blocks to be freed 66488ac3939dSRitesh Harjani * @flags: flags used by ext4_free_blocks 66498ac3939dSRitesh Harjani */ 66508ac3939dSRitesh Harjani void ext4_free_blocks(handle_t *handle, struct inode *inode, 66518ac3939dSRitesh Harjani struct buffer_head *bh, ext4_fsblk_t block, 66528ac3939dSRitesh Harjani unsigned long count, int flags) 66538ac3939dSRitesh Harjani { 66548ac3939dSRitesh Harjani struct super_block *sb = inode->i_sb; 66558ac3939dSRitesh Harjani unsigned int overflow; 66568ac3939dSRitesh Harjani struct ext4_sb_info *sbi; 66578ac3939dSRitesh Harjani 66588ac3939dSRitesh Harjani sbi = EXT4_SB(sb); 66598ac3939dSRitesh Harjani 66608ac3939dSRitesh Harjani if (bh) { 66618ac3939dSRitesh Harjani if (block) 66628ac3939dSRitesh Harjani BUG_ON(block != bh->b_blocknr); 66638ac3939dSRitesh Harjani else 66648ac3939dSRitesh Harjani block = bh->b_blocknr; 66658ac3939dSRitesh Harjani } 66668ac3939dSRitesh Harjani 666711b6890bSKemeng Shi if (sbi->s_mount_state & EXT4_FC_REPLAY) { 66682ec6d0a5SKemeng Shi ext4_free_blocks_simple(inode, block, EXT4_NUM_B2C(sbi, count)); 666911b6890bSKemeng Shi return; 667011b6890bSKemeng Shi } 667111b6890bSKemeng Shi 667211b6890bSKemeng Shi might_sleep(); 667311b6890bSKemeng Shi 66748ac3939dSRitesh Harjani if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 66758ac3939dSRitesh Harjani !ext4_inode_block_valid(inode, block, count)) { 66768ac3939dSRitesh Harjani ext4_error(sb, "Freeing blocks not in datazone - " 66778ac3939dSRitesh Harjani "block = %llu, count = %lu", block, count); 66788ac3939dSRitesh Harjani return; 66798ac3939dSRitesh Harjani } 66801e1c2b86SLukas Czerner flags |= EXT4_FREE_BLOCKS_VALIDATED; 66818ac3939dSRitesh Harjani 66828ac3939dSRitesh Harjani ext4_debug("freeing block %llu\n", block); 66838ac3939dSRitesh Harjani trace_ext4_free_blocks(inode, block, count, flags); 66848ac3939dSRitesh Harjani 66858ac3939dSRitesh Harjani if (bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 66868ac3939dSRitesh Harjani BUG_ON(count > 1); 66878ac3939dSRitesh Harjani 66888ac3939dSRitesh Harjani ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, 66898ac3939dSRitesh Harjani inode, bh, block); 66908ac3939dSRitesh Harjani } 66918ac3939dSRitesh Harjani 66928ac3939dSRitesh Harjani /* 66938ac3939dSRitesh Harjani * If the extent to be freed does not begin on a cluster 66948ac3939dSRitesh Harjani * boundary, we need to deal with partial clusters at the 66958ac3939dSRitesh Harjani * beginning and end of the extent. Normally we will free 66968ac3939dSRitesh Harjani * blocks at the beginning or the end unless we are explicitly 66978ac3939dSRitesh Harjani * requested to avoid doing so. 66988ac3939dSRitesh Harjani */ 66998ac3939dSRitesh Harjani overflow = EXT4_PBLK_COFF(sbi, block); 67008ac3939dSRitesh Harjani if (overflow) { 67018ac3939dSRitesh Harjani if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) { 67028ac3939dSRitesh Harjani overflow = sbi->s_cluster_ratio - overflow; 67038ac3939dSRitesh Harjani block += overflow; 67048ac3939dSRitesh Harjani if (count > overflow) 67058ac3939dSRitesh Harjani count -= overflow; 67068ac3939dSRitesh Harjani else 67078ac3939dSRitesh Harjani return; 67088ac3939dSRitesh Harjani } else { 67098ac3939dSRitesh Harjani block -= overflow; 67108ac3939dSRitesh Harjani count += overflow; 67118ac3939dSRitesh Harjani } 67121e1c2b86SLukas Czerner /* The range changed so it's no longer validated */ 67131e1c2b86SLukas Czerner flags &= ~EXT4_FREE_BLOCKS_VALIDATED; 67148ac3939dSRitesh Harjani } 67158ac3939dSRitesh Harjani overflow = EXT4_LBLK_COFF(sbi, count); 67168ac3939dSRitesh Harjani if (overflow) { 67178ac3939dSRitesh Harjani if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) { 67188ac3939dSRitesh Harjani if (count > overflow) 67198ac3939dSRitesh Harjani count -= overflow; 67208ac3939dSRitesh Harjani else 67218ac3939dSRitesh Harjani return; 67228ac3939dSRitesh Harjani } else 67238ac3939dSRitesh Harjani count += sbi->s_cluster_ratio - overflow; 67241e1c2b86SLukas Czerner /* The range changed so it's no longer validated */ 67251e1c2b86SLukas Czerner flags &= ~EXT4_FREE_BLOCKS_VALIDATED; 67268ac3939dSRitesh Harjani } 67278ac3939dSRitesh Harjani 67288ac3939dSRitesh Harjani if (!bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 67298ac3939dSRitesh Harjani int i; 67308ac3939dSRitesh Harjani int is_metadata = flags & EXT4_FREE_BLOCKS_METADATA; 67318ac3939dSRitesh Harjani 67328ac3939dSRitesh Harjani for (i = 0; i < count; i++) { 67338ac3939dSRitesh Harjani cond_resched(); 67348ac3939dSRitesh Harjani if (is_metadata) 67358ac3939dSRitesh Harjani bh = sb_find_get_block(inode->i_sb, block + i); 67368ac3939dSRitesh Harjani ext4_forget(handle, is_metadata, inode, bh, block + i); 67378ac3939dSRitesh Harjani } 67388ac3939dSRitesh Harjani } 67398ac3939dSRitesh Harjani 67408ac3939dSRitesh Harjani ext4_mb_clear_bb(handle, inode, block, count, flags); 67418ac3939dSRitesh Harjani return; 67428ac3939dSRitesh Harjani } 67438ac3939dSRitesh Harjani 67448ac3939dSRitesh Harjani /** 67450529155eSYongqiang Yang * ext4_group_add_blocks() -- Add given blocks to an existing group 67462846e820SAmir Goldstein * @handle: handle to this transaction 67472846e820SAmir Goldstein * @sb: super block 67484907cb7bSAnatol Pomozov * @block: start physical block to add to the block group 67492846e820SAmir Goldstein * @count: number of blocks to free 67502846e820SAmir Goldstein * 6751e73a347bSAmir Goldstein * This marks the blocks as free in the bitmap and buddy. 67522846e820SAmir Goldstein */ 6753cc7365dfSYongqiang Yang int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, 67542846e820SAmir Goldstein ext4_fsblk_t block, unsigned long count) 67552846e820SAmir Goldstein { 67562846e820SAmir Goldstein struct buffer_head *bitmap_bh = NULL; 67572846e820SAmir Goldstein struct buffer_head *gd_bh; 67582846e820SAmir Goldstein ext4_group_t block_group; 67592846e820SAmir Goldstein ext4_grpblk_t bit; 67602846e820SAmir Goldstein unsigned int i; 67612846e820SAmir Goldstein struct ext4_group_desc *desc; 67622846e820SAmir Goldstein struct ext4_sb_info *sbi = EXT4_SB(sb); 6763e73a347bSAmir Goldstein struct ext4_buddy e4b; 6764d77147ffSharshads int err = 0, ret, free_clusters_count; 6765d77147ffSharshads ext4_grpblk_t clusters_freed; 6766d77147ffSharshads ext4_fsblk_t first_cluster = EXT4_B2C(sbi, block); 6767d77147ffSharshads ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); 6768d77147ffSharshads unsigned long cluster_count = last_cluster - first_cluster + 1; 67692846e820SAmir Goldstein 67702846e820SAmir Goldstein ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); 67712846e820SAmir Goldstein 67724740b830SYongqiang Yang if (count == 0) 67734740b830SYongqiang Yang return 0; 67744740b830SYongqiang Yang 67752846e820SAmir Goldstein ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 67762846e820SAmir Goldstein /* 67772846e820SAmir Goldstein * Check to see if we are freeing blocks across a group 67782846e820SAmir Goldstein * boundary. 67792846e820SAmir Goldstein */ 6780d77147ffSharshads if (bit + cluster_count > EXT4_CLUSTERS_PER_GROUP(sb)) { 6781d77147ffSharshads ext4_warning(sb, "too many blocks added to group %u", 6782cc7365dfSYongqiang Yang block_group); 6783cc7365dfSYongqiang Yang err = -EINVAL; 67842846e820SAmir Goldstein goto error_return; 6785cc7365dfSYongqiang Yang } 67862cd05cc3STheodore Ts'o 67872846e820SAmir Goldstein bitmap_bh = ext4_read_block_bitmap(sb, block_group); 67889008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 67899008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 67909008a58eSDarrick J. Wong bitmap_bh = NULL; 67912846e820SAmir Goldstein goto error_return; 6792cc7365dfSYongqiang Yang } 6793cc7365dfSYongqiang Yang 67942846e820SAmir Goldstein desc = ext4_get_group_desc(sb, block_group, &gd_bh); 6795cc7365dfSYongqiang Yang if (!desc) { 6796cc7365dfSYongqiang Yang err = -EIO; 67972846e820SAmir Goldstein goto error_return; 6798cc7365dfSYongqiang Yang } 67992846e820SAmir Goldstein 6800a00b482bSRitesh Harjani if (!ext4_sb_block_valid(sb, NULL, block, count)) { 68012846e820SAmir Goldstein ext4_error(sb, "Adding blocks in system zones - " 68022846e820SAmir Goldstein "Block = %llu, count = %lu", 68032846e820SAmir Goldstein block, count); 6804cc7365dfSYongqiang Yang err = -EINVAL; 68052846e820SAmir Goldstein goto error_return; 68062846e820SAmir Goldstein } 68072846e820SAmir Goldstein 68082cd05cc3STheodore Ts'o BUFFER_TRACE(bitmap_bh, "getting write access"); 6809188c299eSJan Kara err = ext4_journal_get_write_access(handle, sb, bitmap_bh, 6810188c299eSJan Kara EXT4_JTR_NONE); 68112846e820SAmir Goldstein if (err) 68122846e820SAmir Goldstein goto error_return; 68132846e820SAmir Goldstein 68142846e820SAmir Goldstein /* 68152846e820SAmir Goldstein * We are about to modify some metadata. Call the journal APIs 68162846e820SAmir Goldstein * to unshare ->b_data if a currently-committing transaction is 68172846e820SAmir Goldstein * using it 68182846e820SAmir Goldstein */ 68192846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "get_write_access"); 6820188c299eSJan Kara err = ext4_journal_get_write_access(handle, sb, gd_bh, EXT4_JTR_NONE); 68212846e820SAmir Goldstein if (err) 68222846e820SAmir Goldstein goto error_return; 6823e73a347bSAmir Goldstein 6824d77147ffSharshads for (i = 0, clusters_freed = 0; i < cluster_count; i++) { 68252846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "clear bit"); 6826e73a347bSAmir Goldstein if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { 68272846e820SAmir Goldstein ext4_error(sb, "bit already cleared for block %llu", 68282846e820SAmir Goldstein (ext4_fsblk_t)(block + i)); 68292846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "bit already cleared"); 68302846e820SAmir Goldstein } else { 6831d77147ffSharshads clusters_freed++; 68322846e820SAmir Goldstein } 68332846e820SAmir Goldstein } 6834e73a347bSAmir Goldstein 6835e73a347bSAmir Goldstein err = ext4_mb_load_buddy(sb, block_group, &e4b); 6836e73a347bSAmir Goldstein if (err) 6837e73a347bSAmir Goldstein goto error_return; 6838e73a347bSAmir Goldstein 6839e73a347bSAmir Goldstein /* 6840e73a347bSAmir Goldstein * need to update group_info->bb_free and bitmap 6841e73a347bSAmir Goldstein * with group lock held. generate_buddy look at 6842e73a347bSAmir Goldstein * them with group lock_held 6843e73a347bSAmir Goldstein */ 68442846e820SAmir Goldstein ext4_lock_group(sb, block_group); 6845d77147ffSharshads mb_clear_bits(bitmap_bh->b_data, bit, cluster_count); 6846d77147ffSharshads mb_free_blocks(NULL, &e4b, bit, cluster_count); 6847d77147ffSharshads free_clusters_count = clusters_freed + 6848d77147ffSharshads ext4_free_group_clusters(sb, desc); 6849d77147ffSharshads ext4_free_group_clusters_set(sb, desc, free_clusters_count); 68501df9bde4SKemeng Shi ext4_block_bitmap_csum_set(sb, desc, bitmap_bh); 6851feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, desc); 68522846e820SAmir Goldstein ext4_unlock_group(sb, block_group); 685357042651STheodore Ts'o percpu_counter_add(&sbi->s_freeclusters_counter, 6854d77147ffSharshads clusters_freed); 68552846e820SAmir Goldstein 68562846e820SAmir Goldstein if (sbi->s_log_groups_per_flex) { 68572846e820SAmir Goldstein ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 6858d77147ffSharshads atomic64_add(clusters_freed, 68597c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 68607c990728SSuraj Jitindar Singh flex_group)->free_clusters); 68612846e820SAmir Goldstein } 6862e73a347bSAmir Goldstein 6863e73a347bSAmir Goldstein ext4_mb_unload_buddy(&e4b); 68642846e820SAmir Goldstein 68652846e820SAmir Goldstein /* We dirtied the bitmap block */ 68662846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 68672846e820SAmir Goldstein err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 68682846e820SAmir Goldstein 68692846e820SAmir Goldstein /* And the group descriptor block */ 68702846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 68712846e820SAmir Goldstein ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 68722846e820SAmir Goldstein if (!err) 68732846e820SAmir Goldstein err = ret; 68742846e820SAmir Goldstein 68752846e820SAmir Goldstein error_return: 68762846e820SAmir Goldstein brelse(bitmap_bh); 68772846e820SAmir Goldstein ext4_std_error(sb, err); 6878cc7365dfSYongqiang Yang return err; 68792846e820SAmir Goldstein } 68802846e820SAmir Goldstein 68812846e820SAmir Goldstein /** 68827360d173SLukas Czerner * ext4_trim_extent -- function to TRIM one single free extent in the group 68837360d173SLukas Czerner * @sb: super block for the file system 68847360d173SLukas Czerner * @start: starting block of the free extent in the alloc. group 68857360d173SLukas Czerner * @count: number of blocks to TRIM 68867360d173SLukas Czerner * @e4b: ext4 buddy for the group 68877360d173SLukas Czerner * 68887360d173SLukas Czerner * Trim "count" blocks starting at "start" in the "group". To assure that no 68897360d173SLukas Czerner * one will allocate those blocks, mark it as used in buddy bitmap. This must 68907360d173SLukas Czerner * be called with under the group lock. 68917360d173SLukas Czerner */ 6892bd2eea8dSWang Jianchao static int ext4_trim_extent(struct super_block *sb, 6893bd2eea8dSWang Jianchao int start, int count, struct ext4_buddy *e4b) 6894e2cbd587Sjon ernst __releases(bitlock) 6895e2cbd587Sjon ernst __acquires(bitlock) 68967360d173SLukas Czerner { 68977360d173SLukas Czerner struct ext4_free_extent ex; 6898bd2eea8dSWang Jianchao ext4_group_t group = e4b->bd_group; 6899d71c1ae2SLukas Czerner int ret = 0; 69007360d173SLukas Czerner 6901b3d4c2b1STao Ma trace_ext4_trim_extent(sb, group, start, count); 6902b3d4c2b1STao Ma 69037360d173SLukas Czerner assert_spin_locked(ext4_group_lock_ptr(sb, group)); 69047360d173SLukas Czerner 69057360d173SLukas Czerner ex.fe_start = start; 69067360d173SLukas Czerner ex.fe_group = group; 69077360d173SLukas Czerner ex.fe_len = count; 69087360d173SLukas Czerner 69097360d173SLukas Czerner /* 69107360d173SLukas Czerner * Mark blocks used, so no one can reuse them while 69117360d173SLukas Czerner * being trimmed. 69127360d173SLukas Czerner */ 69137360d173SLukas Czerner mb_mark_used(e4b, &ex); 69147360d173SLukas Czerner ext4_unlock_group(sb, group); 6915a0154344SDaeho Jeong ret = ext4_issue_discard(sb, group, start, count, NULL); 69167360d173SLukas Czerner ext4_lock_group(sb, group); 69177360d173SLukas Czerner mb_free_blocks(NULL, e4b, start, ex.fe_len); 6918d71c1ae2SLukas Czerner return ret; 69197360d173SLukas Czerner } 69207360d173SLukas Czerner 69216920b391SWang Jianchao static int ext4_try_to_trim_range(struct super_block *sb, 69226920b391SWang Jianchao struct ext4_buddy *e4b, ext4_grpblk_t start, 69236920b391SWang Jianchao ext4_grpblk_t max, ext4_grpblk_t minblocks) 6924a5fda113STheodore Ts'o __acquires(ext4_group_lock_ptr(sb, e4b->bd_group)) 6925a5fda113STheodore Ts'o __releases(ext4_group_lock_ptr(sb, e4b->bd_group)) 69266920b391SWang Jianchao { 69276920b391SWang Jianchao ext4_grpblk_t next, count, free_count; 69286920b391SWang Jianchao void *bitmap; 69296920b391SWang Jianchao 69306920b391SWang Jianchao bitmap = e4b->bd_bitmap; 69316920b391SWang Jianchao start = (e4b->bd_info->bb_first_free > start) ? 69326920b391SWang Jianchao e4b->bd_info->bb_first_free : start; 69336920b391SWang Jianchao count = 0; 69346920b391SWang Jianchao free_count = 0; 69356920b391SWang Jianchao 69366920b391SWang Jianchao while (start <= max) { 69376920b391SWang Jianchao start = mb_find_next_zero_bit(bitmap, max + 1, start); 69386920b391SWang Jianchao if (start > max) 69396920b391SWang Jianchao break; 69406920b391SWang Jianchao next = mb_find_next_bit(bitmap, max + 1, start); 69416920b391SWang Jianchao 69426920b391SWang Jianchao if ((next - start) >= minblocks) { 6943afcc4e32SLukas Bulwahn int ret = ext4_trim_extent(sb, start, next - start, e4b); 6944afcc4e32SLukas Bulwahn 69456920b391SWang Jianchao if (ret && ret != -EOPNOTSUPP) 69466920b391SWang Jianchao break; 69476920b391SWang Jianchao count += next - start; 69486920b391SWang Jianchao } 69496920b391SWang Jianchao free_count += next - start; 69506920b391SWang Jianchao start = next + 1; 69516920b391SWang Jianchao 69526920b391SWang Jianchao if (fatal_signal_pending(current)) { 69536920b391SWang Jianchao count = -ERESTARTSYS; 69546920b391SWang Jianchao break; 69556920b391SWang Jianchao } 69566920b391SWang Jianchao 69576920b391SWang Jianchao if (need_resched()) { 69586920b391SWang Jianchao ext4_unlock_group(sb, e4b->bd_group); 69596920b391SWang Jianchao cond_resched(); 69606920b391SWang Jianchao ext4_lock_group(sb, e4b->bd_group); 69616920b391SWang Jianchao } 69626920b391SWang Jianchao 69636920b391SWang Jianchao if ((e4b->bd_info->bb_free - free_count) < minblocks) 69646920b391SWang Jianchao break; 69656920b391SWang Jianchao } 69666920b391SWang Jianchao 69676920b391SWang Jianchao return count; 69686920b391SWang Jianchao } 69696920b391SWang Jianchao 69707360d173SLukas Czerner /** 69717360d173SLukas Czerner * ext4_trim_all_free -- function to trim all free space in alloc. group 69727360d173SLukas Czerner * @sb: super block for file system 697322612283STao Ma * @group: group to be trimmed 69747360d173SLukas Czerner * @start: first group block to examine 69757360d173SLukas Czerner * @max: last group block to examine 69767360d173SLukas Czerner * @minblocks: minimum extent block count 6977d63c00eaSDmitry Monakhov * @set_trimmed: set the trimmed flag if at least one block is trimmed 69787360d173SLukas Czerner * 69797360d173SLukas Czerner * ext4_trim_all_free walks through group's block bitmap searching for free 69807360d173SLukas Czerner * extents. When the free extent is found, mark it as used in group buddy 69817360d173SLukas Czerner * bitmap. Then issue a TRIM command on this extent and free the extent in 6982b6f5558cSWang Jianchao * the group buddy bitmap. 69837360d173SLukas Czerner */ 69840b75a840SLukas Czerner static ext4_grpblk_t 698578944086SLukas Czerner ext4_trim_all_free(struct super_block *sb, ext4_group_t group, 698678944086SLukas Czerner ext4_grpblk_t start, ext4_grpblk_t max, 6987d63c00eaSDmitry Monakhov ext4_grpblk_t minblocks, bool set_trimmed) 69887360d173SLukas Czerner { 698978944086SLukas Czerner struct ext4_buddy e4b; 69906920b391SWang Jianchao int ret; 69917360d173SLukas Czerner 6992b3d4c2b1STao Ma trace_ext4_trim_all_free(sb, group, start, max); 6993b3d4c2b1STao Ma 699478944086SLukas Czerner ret = ext4_mb_load_buddy(sb, group, &e4b); 699578944086SLukas Czerner if (ret) { 69969651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 69979651e6b2SKonstantin Khlebnikov ret, group); 699878944086SLukas Czerner return ret; 699978944086SLukas Czerner } 700028739eeaSLukas Czerner 700128739eeaSLukas Czerner ext4_lock_group(sb, group); 70023d56b8d2STao Ma 70036920b391SWang Jianchao if (!EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) || 70042327fb2eSLukas Czerner minblocks < EXT4_SB(sb)->s_last_trim_minblks) { 70056920b391SWang Jianchao ret = ext4_try_to_trim_range(sb, &e4b, start, max, minblocks); 7006d63c00eaSDmitry Monakhov if (ret >= 0 && set_trimmed) 70073d56b8d2STao Ma EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info); 70086920b391SWang Jianchao } else { 70096920b391SWang Jianchao ret = 0; 7010d71c1ae2SLukas Czerner } 70116920b391SWang Jianchao 70127360d173SLukas Czerner ext4_unlock_group(sb, group); 701378944086SLukas Czerner ext4_mb_unload_buddy(&e4b); 70147360d173SLukas Czerner 70157360d173SLukas Czerner ext4_debug("trimmed %d blocks in the group %d\n", 70166920b391SWang Jianchao ret, group); 70177360d173SLukas Czerner 7018d71c1ae2SLukas Czerner return ret; 70197360d173SLukas Czerner } 70207360d173SLukas Czerner 70217360d173SLukas Czerner /** 70227360d173SLukas Czerner * ext4_trim_fs() -- trim ioctl handle function 70237360d173SLukas Czerner * @sb: superblock for filesystem 70247360d173SLukas Czerner * @range: fstrim_range structure 70257360d173SLukas Czerner * 70267360d173SLukas Czerner * start: First Byte to trim 70277360d173SLukas Czerner * len: number of Bytes to trim from start 70287360d173SLukas Czerner * minlen: minimum extent length in Bytes 70297360d173SLukas Czerner * ext4_trim_fs goes through all allocation groups containing Bytes from 70307360d173SLukas Czerner * start to start+len. For each such a group ext4_trim_all_free function 70317360d173SLukas Czerner * is invoked to trim all free space. 70327360d173SLukas Czerner */ 70337360d173SLukas Czerner int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) 70347360d173SLukas Czerner { 70357b47ef52SChristoph Hellwig unsigned int discard_granularity = bdev_discard_granularity(sb->s_bdev); 703678944086SLukas Czerner struct ext4_group_info *grp; 7037913eed83SLukas Czerner ext4_group_t group, first_group, last_group; 70387137d7a4STheodore Ts'o ext4_grpblk_t cnt = 0, first_cluster, last_cluster; 7039913eed83SLukas Czerner uint64_t start, end, minlen, trimmed = 0; 70400f0a25bfSJan Kara ext4_fsblk_t first_data_blk = 70410f0a25bfSJan Kara le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 7042913eed83SLukas Czerner ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); 7043d63c00eaSDmitry Monakhov bool whole_group, eof = false; 70447360d173SLukas Czerner int ret = 0; 70457360d173SLukas Czerner 70467360d173SLukas Czerner start = range->start >> sb->s_blocksize_bits; 7047913eed83SLukas Czerner end = start + (range->len >> sb->s_blocksize_bits) - 1; 7048aaf7d73eSLukas Czerner minlen = EXT4_NUM_B2C(EXT4_SB(sb), 7049aaf7d73eSLukas Czerner range->minlen >> sb->s_blocksize_bits); 70507360d173SLukas Czerner 70515de35e8dSLukas Czerner if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) || 70525de35e8dSLukas Czerner start >= max_blks || 70535de35e8dSLukas Czerner range->len < sb->s_blocksize) 70547360d173SLukas Czerner return -EINVAL; 7055173b6e38SJan Kara /* No point to try to trim less than discard granularity */ 70567b47ef52SChristoph Hellwig if (range->minlen < discard_granularity) { 7057173b6e38SJan Kara minlen = EXT4_NUM_B2C(EXT4_SB(sb), 70587b47ef52SChristoph Hellwig discard_granularity >> sb->s_blocksize_bits); 7059173b6e38SJan Kara if (minlen > EXT4_CLUSTERS_PER_GROUP(sb)) 7060173b6e38SJan Kara goto out; 7061173b6e38SJan Kara } 7062d63c00eaSDmitry Monakhov if (end >= max_blks - 1) { 7063913eed83SLukas Czerner end = max_blks - 1; 7064d63c00eaSDmitry Monakhov eof = true; 7065d63c00eaSDmitry Monakhov } 7066913eed83SLukas Czerner if (end <= first_data_blk) 706722f10457STao Ma goto out; 7068913eed83SLukas Czerner if (start < first_data_blk) 70690f0a25bfSJan Kara start = first_data_blk; 70707360d173SLukas Czerner 7071913eed83SLukas Czerner /* Determine first and last group to examine based on start and end */ 70727360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, 70737137d7a4STheodore Ts'o &first_group, &first_cluster); 7074913eed83SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end, 70757137d7a4STheodore Ts'o &last_group, &last_cluster); 70767360d173SLukas Czerner 7077913eed83SLukas Czerner /* end now represents the last cluster to discard in this group */ 7078913eed83SLukas Czerner end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 7079d63c00eaSDmitry Monakhov whole_group = true; 70807360d173SLukas Czerner 70817360d173SLukas Czerner for (group = first_group; group <= last_group; group++) { 708278944086SLukas Czerner grp = ext4_get_group_info(sb, group); 70835354b2afSTheodore Ts'o if (!grp) 70845354b2afSTheodore Ts'o continue; 708578944086SLukas Czerner /* We only do this if the grp has never been initialized */ 708678944086SLukas Czerner if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 7087adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, GFP_NOFS); 708878944086SLukas Czerner if (ret) 70897360d173SLukas Czerner break; 70907360d173SLukas Czerner } 70917360d173SLukas Czerner 70920ba08517STao Ma /* 7093913eed83SLukas Czerner * For all the groups except the last one, last cluster will 7094913eed83SLukas Czerner * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to 7095913eed83SLukas Czerner * change it for the last group, note that last_cluster is 7096913eed83SLukas Czerner * already computed earlier by ext4_get_group_no_and_offset() 70970ba08517STao Ma */ 7098d63c00eaSDmitry Monakhov if (group == last_group) { 7099913eed83SLukas Czerner end = last_cluster; 7100d63c00eaSDmitry Monakhov whole_group = eof ? true : end == EXT4_CLUSTERS_PER_GROUP(sb) - 1; 7101d63c00eaSDmitry Monakhov } 710278944086SLukas Czerner if (grp->bb_free >= minlen) { 71037137d7a4STheodore Ts'o cnt = ext4_trim_all_free(sb, group, first_cluster, 7104d63c00eaSDmitry Monakhov end, minlen, whole_group); 71057360d173SLukas Czerner if (cnt < 0) { 71067360d173SLukas Czerner ret = cnt; 71077360d173SLukas Czerner break; 71087360d173SLukas Czerner } 71097360d173SLukas Czerner trimmed += cnt; 711021e7fd22SLukas Czerner } 7111913eed83SLukas Czerner 7112913eed83SLukas Czerner /* 7113913eed83SLukas Czerner * For every group except the first one, we are sure 7114913eed83SLukas Czerner * that the first cluster to discard will be cluster #0. 7115913eed83SLukas Czerner */ 71167137d7a4STheodore Ts'o first_cluster = 0; 71177360d173SLukas Czerner } 71187360d173SLukas Czerner 71193d56b8d2STao Ma if (!ret) 71202327fb2eSLukas Czerner EXT4_SB(sb)->s_last_trim_minblks = minlen; 71213d56b8d2STao Ma 712222f10457STao Ma out: 7123aaf7d73eSLukas Czerner range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; 71247360d173SLukas Czerner return ret; 71257360d173SLukas Czerner } 71260c9ec4beSDarrick J. Wong 71270c9ec4beSDarrick J. Wong /* Iterate all the free extents in the group. */ 71280c9ec4beSDarrick J. Wong int 71290c9ec4beSDarrick J. Wong ext4_mballoc_query_range( 71300c9ec4beSDarrick J. Wong struct super_block *sb, 71310c9ec4beSDarrick J. Wong ext4_group_t group, 71320c9ec4beSDarrick J. Wong ext4_grpblk_t start, 71330c9ec4beSDarrick J. Wong ext4_grpblk_t end, 71340c9ec4beSDarrick J. Wong ext4_mballoc_query_range_fn formatter, 71350c9ec4beSDarrick J. Wong void *priv) 71360c9ec4beSDarrick J. Wong { 71370c9ec4beSDarrick J. Wong void *bitmap; 71380c9ec4beSDarrick J. Wong ext4_grpblk_t next; 71390c9ec4beSDarrick J. Wong struct ext4_buddy e4b; 71400c9ec4beSDarrick J. Wong int error; 71410c9ec4beSDarrick J. Wong 71420c9ec4beSDarrick J. Wong error = ext4_mb_load_buddy(sb, group, &e4b); 71430c9ec4beSDarrick J. Wong if (error) 71440c9ec4beSDarrick J. Wong return error; 71450c9ec4beSDarrick J. Wong bitmap = e4b.bd_bitmap; 71460c9ec4beSDarrick J. Wong 71470c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 71480c9ec4beSDarrick J. Wong 71490c9ec4beSDarrick J. Wong start = (e4b.bd_info->bb_first_free > start) ? 71500c9ec4beSDarrick J. Wong e4b.bd_info->bb_first_free : start; 71510c9ec4beSDarrick J. Wong if (end >= EXT4_CLUSTERS_PER_GROUP(sb)) 71520c9ec4beSDarrick J. Wong end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 71530c9ec4beSDarrick J. Wong 71540c9ec4beSDarrick J. Wong while (start <= end) { 71550c9ec4beSDarrick J. Wong start = mb_find_next_zero_bit(bitmap, end + 1, start); 71560c9ec4beSDarrick J. Wong if (start > end) 71570c9ec4beSDarrick J. Wong break; 71580c9ec4beSDarrick J. Wong next = mb_find_next_bit(bitmap, end + 1, start); 71590c9ec4beSDarrick J. Wong 71600c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 71610c9ec4beSDarrick J. Wong error = formatter(sb, group, start, next - start, priv); 71620c9ec4beSDarrick J. Wong if (error) 71630c9ec4beSDarrick J. Wong goto out_unload; 71640c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 71650c9ec4beSDarrick J. Wong 71660c9ec4beSDarrick J. Wong start = next + 1; 71670c9ec4beSDarrick J. Wong } 71680c9ec4beSDarrick J. Wong 71690c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 71700c9ec4beSDarrick J. Wong out_unload: 71710c9ec4beSDarrick J. Wong ext4_mb_unload_buddy(&e4b); 71720c9ec4beSDarrick J. Wong 71730c9ec4beSDarrick J. Wong return error; 71740c9ec4beSDarrick J. Wong } 7175