1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 28fa3ed80SDennis Zhou #ifndef _MM_PERCPU_INTERNAL_H 38fa3ed80SDennis Zhou #define _MM_PERCPU_INTERNAL_H 48fa3ed80SDennis Zhou 58fa3ed80SDennis Zhou #include <linux/types.h> 68fa3ed80SDennis Zhou #include <linux/percpu.h> 78fa3ed80SDennis Zhou 8ca460b3cSDennis Zhou (Facebook) /* 9ca460b3cSDennis Zhou (Facebook) * pcpu_block_md is the metadata block struct. 10ca460b3cSDennis Zhou (Facebook) * Each chunk's bitmap is split into a number of full blocks. 11ca460b3cSDennis Zhou (Facebook) * All units are in terms of bits. 12382b88e9SDennis Zhou * 13382b88e9SDennis Zhou * The scan hint is the largest known contiguous area before the contig hint. 14382b88e9SDennis Zhou * It is not necessarily the actual largest contig hint though. There is an 15382b88e9SDennis Zhou * invariant that the scan_hint_start > contig_hint_start iff 16382b88e9SDennis Zhou * scan_hint == contig_hint. This is necessary because when scanning forward, 17382b88e9SDennis Zhou * we don't know if a new contig hint would be better than the current one. 18ca460b3cSDennis Zhou (Facebook) */ 19ca460b3cSDennis Zhou (Facebook) struct pcpu_block_md { 20382b88e9SDennis Zhou int scan_hint; /* scan hint for block */ 21382b88e9SDennis Zhou int scan_hint_start; /* block relative starting 22382b88e9SDennis Zhou position of the scan hint */ 23ca460b3cSDennis Zhou (Facebook) int contig_hint; /* contig hint for block */ 24ca460b3cSDennis Zhou (Facebook) int contig_hint_start; /* block relative starting 25ca460b3cSDennis Zhou (Facebook) position of the contig hint */ 26ca460b3cSDennis Zhou (Facebook) int left_free; /* size of free space along 27ca460b3cSDennis Zhou (Facebook) the left side of the block */ 28ca460b3cSDennis Zhou (Facebook) int right_free; /* size of free space along 29ca460b3cSDennis Zhou (Facebook) the right side of the block */ 30ca460b3cSDennis Zhou (Facebook) int first_free; /* block position of first free */ 31047924c9SDennis Zhou int nr_bits; /* total bits responsible for */ 32ca460b3cSDennis Zhou (Facebook) }; 33ca460b3cSDennis Zhou (Facebook) 348fa3ed80SDennis Zhou struct pcpu_chunk { 3530a5b536SDennis Zhou #ifdef CONFIG_PERCPU_STATS 3630a5b536SDennis Zhou int nr_alloc; /* # of allocations */ 3730a5b536SDennis Zhou size_t max_alloc_size; /* largest allocation size */ 3830a5b536SDennis Zhou #endif 3930a5b536SDennis Zhou 408fa3ed80SDennis Zhou struct list_head list; /* linked to pcpu_slot lists */ 4140064aecSDennis Zhou (Facebook) int free_bytes; /* free bytes in the chunk */ 42*92c14cabSDennis Zhou struct pcpu_block_md chunk_md; 438fa3ed80SDennis Zhou void *base_addr; /* base address of this chunk */ 448fa3ed80SDennis Zhou 4540064aecSDennis Zhou (Facebook) unsigned long *alloc_map; /* allocation map */ 4640064aecSDennis Zhou (Facebook) unsigned long *bound_map; /* boundary map */ 47ca460b3cSDennis Zhou (Facebook) struct pcpu_block_md *md_blocks; /* metadata blocks */ 488fa3ed80SDennis Zhou 498fa3ed80SDennis Zhou void *data; /* chunk data */ 508fa3ed80SDennis Zhou bool immutable; /* no [de]population allowed */ 51e2266705SDennis Zhou (Facebook) int start_offset; /* the overlap with the previous 52e2266705SDennis Zhou (Facebook) region to have a page aligned 53e2266705SDennis Zhou (Facebook) base_addr */ 546b9d7c8eSDennis Zhou (Facebook) int end_offset; /* additional area required to 556b9d7c8eSDennis Zhou (Facebook) have the region end page 566b9d7c8eSDennis Zhou (Facebook) aligned */ 57c0ebfdc3SDennis Zhou (Facebook) 58c0ebfdc3SDennis Zhou (Facebook) int nr_pages; /* # of pages served by this chunk */ 598fa3ed80SDennis Zhou int nr_populated; /* # of populated pages */ 600cecf50cSDennis Zhou (Facebook) int nr_empty_pop_pages; /* # of empty populated pages */ 618fa3ed80SDennis Zhou unsigned long populated[]; /* populated bitmap */ 628fa3ed80SDennis Zhou }; 638fa3ed80SDennis Zhou 648fa3ed80SDennis Zhou extern spinlock_t pcpu_lock; 658fa3ed80SDennis Zhou 668fa3ed80SDennis Zhou extern struct list_head *pcpu_slot; 678fa3ed80SDennis Zhou extern int pcpu_nr_slots; 686b9b6f39SDennis Zhou (Facebook) extern int pcpu_nr_empty_pop_pages; 698fa3ed80SDennis Zhou 708fa3ed80SDennis Zhou extern struct pcpu_chunk *pcpu_first_chunk; 718fa3ed80SDennis Zhou extern struct pcpu_chunk *pcpu_reserved_chunk; 728fa3ed80SDennis Zhou 7340064aecSDennis Zhou (Facebook) /** 74ca460b3cSDennis Zhou (Facebook) * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 75ca460b3cSDennis Zhou (Facebook) * @chunk: chunk of interest 76ca460b3cSDennis Zhou (Facebook) * 77ca460b3cSDennis Zhou (Facebook) * This conversion is from the number of physical pages that the chunk 78ca460b3cSDennis Zhou (Facebook) * serves to the number of bitmap blocks used. 79ca460b3cSDennis Zhou (Facebook) */ 80ca460b3cSDennis Zhou (Facebook) static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 81ca460b3cSDennis Zhou (Facebook) { 82ca460b3cSDennis Zhou (Facebook) return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 83ca460b3cSDennis Zhou (Facebook) } 84ca460b3cSDennis Zhou (Facebook) 85ca460b3cSDennis Zhou (Facebook) /** 8640064aecSDennis Zhou (Facebook) * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 8740064aecSDennis Zhou (Facebook) * @pages: number of physical pages 8840064aecSDennis Zhou (Facebook) * 8940064aecSDennis Zhou (Facebook) * This conversion is from physical pages to the number of bits 9040064aecSDennis Zhou (Facebook) * required in the bitmap. 9140064aecSDennis Zhou (Facebook) */ 9240064aecSDennis Zhou (Facebook) static inline int pcpu_nr_pages_to_map_bits(int pages) 9340064aecSDennis Zhou (Facebook) { 9440064aecSDennis Zhou (Facebook) return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 9540064aecSDennis Zhou (Facebook) } 9640064aecSDennis Zhou (Facebook) 9740064aecSDennis Zhou (Facebook) /** 9840064aecSDennis Zhou (Facebook) * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 9940064aecSDennis Zhou (Facebook) * @chunk: chunk of interest 10040064aecSDennis Zhou (Facebook) * 10140064aecSDennis Zhou (Facebook) * This conversion is from the number of physical pages that the chunk 10240064aecSDennis Zhou (Facebook) * serves to the number of bits in the bitmap. 10340064aecSDennis Zhou (Facebook) */ 10440064aecSDennis Zhou (Facebook) static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 10540064aecSDennis Zhou (Facebook) { 10640064aecSDennis Zhou (Facebook) return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 10740064aecSDennis Zhou (Facebook) } 10840064aecSDennis Zhou (Facebook) 10930a5b536SDennis Zhou #ifdef CONFIG_PERCPU_STATS 11030a5b536SDennis Zhou 11130a5b536SDennis Zhou #include <linux/spinlock.h> 11230a5b536SDennis Zhou 11330a5b536SDennis Zhou struct percpu_stats { 11430a5b536SDennis Zhou u64 nr_alloc; /* lifetime # of allocations */ 11530a5b536SDennis Zhou u64 nr_dealloc; /* lifetime # of deallocations */ 11630a5b536SDennis Zhou u64 nr_cur_alloc; /* current # of allocations */ 11730a5b536SDennis Zhou u64 nr_max_alloc; /* max # of live allocations */ 11830a5b536SDennis Zhou u32 nr_chunks; /* current # of live chunks */ 11930a5b536SDennis Zhou u32 nr_max_chunks; /* max # of live chunks */ 12030a5b536SDennis Zhou size_t min_alloc_size; /* min allocaiton size */ 12130a5b536SDennis Zhou size_t max_alloc_size; /* max allocation size */ 12230a5b536SDennis Zhou }; 12330a5b536SDennis Zhou 12430a5b536SDennis Zhou extern struct percpu_stats pcpu_stats; 12530a5b536SDennis Zhou extern struct pcpu_alloc_info pcpu_stats_ai; 12630a5b536SDennis Zhou 12730a5b536SDennis Zhou /* 12830a5b536SDennis Zhou * For debug purposes. We don't care about the flexible array. 12930a5b536SDennis Zhou */ 13030a5b536SDennis Zhou static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 13130a5b536SDennis Zhou { 13230a5b536SDennis Zhou memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 13330a5b536SDennis Zhou 13430a5b536SDennis Zhou /* initialize min_alloc_size to unit_size */ 13530a5b536SDennis Zhou pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 13630a5b536SDennis Zhou } 13730a5b536SDennis Zhou 13830a5b536SDennis Zhou /* 13930a5b536SDennis Zhou * pcpu_stats_area_alloc - increment area allocation stats 14030a5b536SDennis Zhou * @chunk: the location of the area being allocated 14130a5b536SDennis Zhou * @size: size of area to allocate in bytes 14230a5b536SDennis Zhou * 14330a5b536SDennis Zhou * CONTEXT: 14430a5b536SDennis Zhou * pcpu_lock. 14530a5b536SDennis Zhou */ 14630a5b536SDennis Zhou static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 14730a5b536SDennis Zhou { 14830a5b536SDennis Zhou lockdep_assert_held(&pcpu_lock); 14930a5b536SDennis Zhou 15030a5b536SDennis Zhou pcpu_stats.nr_alloc++; 15130a5b536SDennis Zhou pcpu_stats.nr_cur_alloc++; 15230a5b536SDennis Zhou pcpu_stats.nr_max_alloc = 15330a5b536SDennis Zhou max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 15430a5b536SDennis Zhou pcpu_stats.min_alloc_size = 15530a5b536SDennis Zhou min(pcpu_stats.min_alloc_size, size); 15630a5b536SDennis Zhou pcpu_stats.max_alloc_size = 15730a5b536SDennis Zhou max(pcpu_stats.max_alloc_size, size); 15830a5b536SDennis Zhou 15930a5b536SDennis Zhou chunk->nr_alloc++; 16030a5b536SDennis Zhou chunk->max_alloc_size = max(chunk->max_alloc_size, size); 16130a5b536SDennis Zhou } 16230a5b536SDennis Zhou 16330a5b536SDennis Zhou /* 16430a5b536SDennis Zhou * pcpu_stats_area_dealloc - decrement allocation stats 16530a5b536SDennis Zhou * @chunk: the location of the area being deallocated 16630a5b536SDennis Zhou * 16730a5b536SDennis Zhou * CONTEXT: 16830a5b536SDennis Zhou * pcpu_lock. 16930a5b536SDennis Zhou */ 17030a5b536SDennis Zhou static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 17130a5b536SDennis Zhou { 17230a5b536SDennis Zhou lockdep_assert_held(&pcpu_lock); 17330a5b536SDennis Zhou 17430a5b536SDennis Zhou pcpu_stats.nr_dealloc++; 17530a5b536SDennis Zhou pcpu_stats.nr_cur_alloc--; 17630a5b536SDennis Zhou 17730a5b536SDennis Zhou chunk->nr_alloc--; 17830a5b536SDennis Zhou } 17930a5b536SDennis Zhou 18030a5b536SDennis Zhou /* 18130a5b536SDennis Zhou * pcpu_stats_chunk_alloc - increment chunk stats 18230a5b536SDennis Zhou */ 18330a5b536SDennis Zhou static inline void pcpu_stats_chunk_alloc(void) 18430a5b536SDennis Zhou { 185303abfdfSDennis Zhou unsigned long flags; 186303abfdfSDennis Zhou spin_lock_irqsave(&pcpu_lock, flags); 18730a5b536SDennis Zhou 18830a5b536SDennis Zhou pcpu_stats.nr_chunks++; 18930a5b536SDennis Zhou pcpu_stats.nr_max_chunks = 19030a5b536SDennis Zhou max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 19130a5b536SDennis Zhou 192303abfdfSDennis Zhou spin_unlock_irqrestore(&pcpu_lock, flags); 19330a5b536SDennis Zhou } 19430a5b536SDennis Zhou 19530a5b536SDennis Zhou /* 19630a5b536SDennis Zhou * pcpu_stats_chunk_dealloc - decrement chunk stats 19730a5b536SDennis Zhou */ 19830a5b536SDennis Zhou static inline void pcpu_stats_chunk_dealloc(void) 19930a5b536SDennis Zhou { 200303abfdfSDennis Zhou unsigned long flags; 201303abfdfSDennis Zhou spin_lock_irqsave(&pcpu_lock, flags); 20230a5b536SDennis Zhou 20330a5b536SDennis Zhou pcpu_stats.nr_chunks--; 20430a5b536SDennis Zhou 205303abfdfSDennis Zhou spin_unlock_irqrestore(&pcpu_lock, flags); 20630a5b536SDennis Zhou } 20730a5b536SDennis Zhou 20830a5b536SDennis Zhou #else 20930a5b536SDennis Zhou 21030a5b536SDennis Zhou static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 21130a5b536SDennis Zhou { 21230a5b536SDennis Zhou } 21330a5b536SDennis Zhou 21430a5b536SDennis Zhou static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 21530a5b536SDennis Zhou { 21630a5b536SDennis Zhou } 21730a5b536SDennis Zhou 21830a5b536SDennis Zhou static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 21930a5b536SDennis Zhou { 22030a5b536SDennis Zhou } 22130a5b536SDennis Zhou 22230a5b536SDennis Zhou static inline void pcpu_stats_chunk_alloc(void) 22330a5b536SDennis Zhou { 22430a5b536SDennis Zhou } 22530a5b536SDennis Zhou 22630a5b536SDennis Zhou static inline void pcpu_stats_chunk_dealloc(void) 22730a5b536SDennis Zhou { 22830a5b536SDennis Zhou } 22930a5b536SDennis Zhou 23030a5b536SDennis Zhou #endif /* !CONFIG_PERCPU_STATS */ 23130a5b536SDennis Zhou 2328fa3ed80SDennis Zhou #endif 233