1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 #include "jemalloc/internal/sz.h" 4 5 JEMALLOC_ALIGNED(CACHELINE) 6 size_t sz_pind2sz_tab[SC_NPSIZES+1]; 7 size_t sz_large_pad; 8 9 size_t 10 sz_psz_quantize_floor(size_t size) { 11 size_t ret; 12 pszind_t pind; 13 14 assert(size > 0); 15 assert((size & PAGE_MASK) == 0); 16 17 pind = sz_psz2ind(size - sz_large_pad + 1); 18 if (pind == 0) { 19 /* 20 * Avoid underflow. This short-circuit would also do the right 21 * thing for all sizes in the range for which there are 22 * PAGE-spaced size classes, but it's simplest to just handle 23 * the one case that would cause erroneous results. 24 */ 25 return size; 26 } 27 ret = sz_pind2sz(pind - 1) + sz_large_pad; 28 assert(ret <= size); 29 return ret; 30 } 31 32 size_t 33 sz_psz_quantize_ceil(size_t size) { 34 size_t ret; 35 36 assert(size > 0); 37 assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); 38 assert((size & PAGE_MASK) == 0); 39 40 ret = sz_psz_quantize_floor(size); 41 if (ret < size) { 42 /* 43 * Skip a quantization that may have an adequately large extent, 44 * because under-sized extents may be mixed in. This only 45 * happens when an unusual size is requested, i.e. for aligned 46 * allocation, and is just one of several places where linear 47 * search would potentially find sufficiently aligned available 48 * memory somewhere lower. 49 */ 50 ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + 51 sz_large_pad; 52 } 53 return ret; 54 } 55 56 static void 57 sz_boot_pind2sz_tab(const sc_data_t *sc_data) { 58 int pind = 0; 59 for (unsigned i = 0; i < SC_NSIZES; i++) { 60 const sc_t *sc = &sc_data->sc[i]; 61 if (sc->psz) { 62 sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base) 63 + (ZU(sc->ndelta) << sc->lg_delta); 64 pind++; 65 } 66 } 67 for (int i = pind; i <= (int)SC_NPSIZES; i++) { 68 sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; 69 } 70 } 71 72 JEMALLOC_ALIGNED(CACHELINE) 73 size_t sz_index2size_tab[SC_NSIZES]; 74 75 static void 76 sz_boot_index2size_tab(const sc_data_t *sc_data) { 77 for (unsigned i = 0; i < SC_NSIZES; i++) { 78 const sc_t *sc = &sc_data->sc[i]; 79 sz_index2size_tab[i] = (ZU(1) << sc->lg_base) 80 + (ZU(sc->ndelta) << (sc->lg_delta)); 81 } 82 } 83 84 /* 85 * To keep this table small, we divide sizes by the tiny min size, which gives 86 * the smallest interval for which the result can change. 87 */ 88 JEMALLOC_ALIGNED(CACHELINE) 89 uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1]; 90 91 static void 92 sz_boot_size2index_tab(const sc_data_t *sc_data) { 93 size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1; 94 size_t dst_ind = 0; 95 for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; 96 sc_ind++) { 97 const sc_t *sc = &sc_data->sc[sc_ind]; 98 size_t sz = (ZU(1) << sc->lg_base) 99 + (ZU(sc->ndelta) << sc->lg_delta); 100 size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1) 101 >> SC_LG_TINY_MIN); 102 for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) { 103 sz_size2index_tab[dst_ind] = sc_ind; 104 } 105 } 106 } 107 108 void 109 sz_boot(const sc_data_t *sc_data, bool cache_oblivious) { 110 sz_large_pad = cache_oblivious ? PAGE : 0; 111 sz_boot_pind2sz_tab(sc_data); 112 sz_boot_index2size_tab(sc_data); 113 sz_boot_size2index_tab(sc_data); 114 } 115