Lines Matching refs:cp

931 	umem_cache_t *cp;  in umem_cache_applyall()  local
934 for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; in umem_cache_applyall()
935 cp = cp->cache_next) in umem_cache_applyall()
936 func(cp); in umem_cache_applyall()
941 umem_add_update_unlocked(umem_cache_t *cp, int flags) in umem_add_update_unlocked() argument
950 if (cp->cache_uflags & UMU_ACTIVE) { in umem_add_update_unlocked()
951 cp->cache_uflags |= flags; in umem_add_update_unlocked()
953 if (cp->cache_unext != NULL) { in umem_add_update_unlocked()
954 ASSERT(cp->cache_uflags != 0); in umem_add_update_unlocked()
955 cp->cache_uflags |= flags; in umem_add_update_unlocked()
957 ASSERT(cp->cache_uflags == 0); in umem_add_update_unlocked()
958 cp->cache_uflags = flags; in umem_add_update_unlocked()
959 cp->cache_unext = cnext = &umem_null_cache; in umem_add_update_unlocked()
960 cp->cache_uprev = cprev = umem_null_cache.cache_uprev; in umem_add_update_unlocked()
961 cnext->cache_uprev = cp; in umem_add_update_unlocked()
962 cprev->cache_unext = cp; in umem_add_update_unlocked()
968 umem_add_update(umem_cache_t *cp, int flags) in umem_add_update() argument
972 umem_add_update_unlocked(cp, flags); in umem_add_update()
985 umem_remove_updates(umem_cache_t *cp) in umem_remove_updates() argument
992 while (cp->cache_uflags & UMU_ACTIVE) { in umem_remove_updates()
995 ASSERT(cp->cache_unext == NULL); in umem_remove_updates()
997 cp->cache_uflags |= UMU_NOTIFY; in umem_remove_updates()
1014 if (cp->cache_unext != NULL) { in umem_remove_updates()
1015 cp->cache_uprev->cache_unext = cp->cache_unext; in umem_remove_updates()
1016 cp->cache_unext->cache_uprev = cp->cache_uprev; in umem_remove_updates()
1017 cp->cache_uprev = cp->cache_unext = NULL; in umem_remove_updates()
1018 cp->cache_uflags = 0; in umem_remove_updates()
1023 ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0); in umem_remove_updates()
1030 umem_cache_t *cp; in umem_updateall() local
1040 for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; in umem_updateall()
1041 cp = cp->cache_next) in umem_updateall()
1042 umem_add_update_unlocked(cp, flags); in umem_updateall()
1055 umem_findslab(umem_cache_t *cp, void *buf) in umem_findslab() argument
1059 (void) mutex_lock(&cp->cache_lock); in umem_findslab()
1060 for (sp = cp->cache_nullslab.slab_next; in umem_findslab()
1061 sp != &cp->cache_nullslab; sp = sp->slab_next) { in umem_findslab()
1063 (void) mutex_unlock(&cp->cache_lock); in umem_findslab()
1067 (void) mutex_unlock(&cp->cache_lock); in umem_findslab()
1077 umem_cache_t *cp = cparg; in umem_error() local
1088 sp = umem_findslab(cp, buf); in umem_error()
1090 for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache; in umem_error()
1091 cp = cp->cache_prev) { in umem_error()
1092 if ((sp = umem_findslab(cp, buf)) != NULL) in umem_error()
1098 cp = NULL; in umem_error()
1101 if (cp != cparg) in umem_error()
1105 (uintptr_t)sp->slab_base) % cp->cache_chunksize; in umem_error()
1108 if (cp->cache_flags & UMF_BUFTAG) in umem_error()
1109 btp = UMEM_BUFTAG(cp, buf); in umem_error()
1110 if (cp->cache_flags & UMF_HASH) { in umem_error()
1111 (void) mutex_lock(&cp->cache_lock); in umem_error()
1112 for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) in umem_error()
1115 (void) mutex_unlock(&cp->cache_lock); in umem_error()
1118 if (umem_findslab(cp->cache_bufctl_cache, bcp) == in umem_error()
1131 umem_abort_info.ump_realcache = cp; in umem_error()
1141 off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); in umem_error()
1175 umem_printf("buffer was allocated from %s,\n", cp->cache_name); in umem_error()
1195 if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) && in umem_error()
1209 (void *)sp, cp->cache_name); in umem_error()
1228 umem_alloc_retry(umem_cache_t *cp, int umflag) in umem_alloc_retry() argument
1230 if (cp == &umem_null_cache) { in umem_alloc_retry()
1367 #define UMEM_AUDIT(lp, cp, bcp) \ argument
1373 (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL)); \
1379 umem_log_event(umem_log_header_t *lp, umem_cache_t *cp, in umem_log_event() argument
1388 bcp->bc_cache = cp; in umem_log_event()
1389 UMEM_AUDIT(lp, cp, bcp); in umem_log_event()
1396 umem_slab_create(umem_cache_t *cp, int umflag) in umem_slab_create() argument
1398 size_t slabsize = cp->cache_slabsize; in umem_slab_create()
1399 size_t chunksize = cp->cache_chunksize; in umem_slab_create()
1400 int cache_flags = cp->cache_flags; in umem_slab_create()
1405 vmem_t *vmp = cp->cache_arena; in umem_slab_create()
1407 color = cp->cache_color + cp->cache_align; in umem_slab_create()
1408 if (color > cp->cache_maxcolor) in umem_slab_create()
1409 color = cp->cache_mincolor; in umem_slab_create()
1410 cp->cache_color = color; in umem_slab_create()
1419 if (!(cp->cache_cflags & UMC_NOTOUCH) && in umem_slab_create()
1420 (cp->cache_flags & UMF_DEADBEEF)) in umem_slab_create()
1428 sp = UMEM_SLAB(cp, slab); in umem_slab_create()
1432 sp->slab_cache = cp; in umem_slab_create()
1441 bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag); in umem_slab_create()
1448 bcap->bc_cache = cp; in umem_slab_create()
1453 bcp = UMEM_BUFCTL(cp, buf); in umem_slab_create()
1456 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in umem_slab_create()
1462 cp->cache_verify); in umem_slab_create()
1470 umem_log_event(umem_slab_log, cp, sp, slab); in umem_slab_create()
1478 _umem_cache_free(cp->cache_bufctl_cache, bcp); in umem_slab_create()
1488 umem_log_event(umem_failure_log, cp, NULL, NULL); in umem_slab_create()
1489 atomic_add_64(&cp->cache_alloc_fail, 1); in umem_slab_create()
1498 umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp) in umem_slab_destroy() argument
1500 vmem_t *vmp = cp->cache_arena; in umem_slab_destroy()
1503 if (cp->cache_flags & UMF_HASH) { in umem_slab_destroy()
1507 _umem_cache_free(cp->cache_bufctl_cache, bcp); in umem_slab_destroy()
1511 vmem_free(vmp, slab, cp->cache_slabsize); in umem_slab_destroy()
1518 umem_slab_alloc(umem_cache_t *cp, int umflag) in umem_slab_alloc() argument
1524 (void) mutex_lock(&cp->cache_lock); in umem_slab_alloc()
1525 cp->cache_slab_alloc++; in umem_slab_alloc()
1526 sp = cp->cache_freelist; in umem_slab_alloc()
1527 ASSERT(sp->slab_cache == cp); in umem_slab_alloc()
1532 (void) mutex_unlock(&cp->cache_lock); in umem_slab_alloc()
1533 if (cp == &umem_null_cache) in umem_slab_alloc()
1535 if ((sp = umem_slab_create(cp, umflag)) == NULL) in umem_slab_alloc()
1537 (void) mutex_lock(&cp->cache_lock); in umem_slab_alloc()
1538 cp->cache_slab_create++; in umem_slab_alloc()
1539 if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) in umem_slab_alloc()
1540 cp->cache_bufmax = cp->cache_buftotal; in umem_slab_alloc()
1541 sp->slab_next = cp->cache_freelist; in umem_slab_alloc()
1542 sp->slab_prev = cp->cache_freelist->slab_prev; in umem_slab_alloc()
1545 cp->cache_freelist = sp; in umem_slab_alloc()
1557 cp->cache_freelist = sp->slab_next; in umem_slab_alloc()
1561 if (cp->cache_flags & UMF_HASH) { in umem_slab_alloc()
1566 hash_bucket = UMEM_HASH(cp, buf); in umem_slab_alloc()
1569 if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { in umem_slab_alloc()
1570 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_slab_alloc()
1573 buf = UMEM_BUF(cp, bcp); in umem_slab_alloc()
1578 (void) mutex_unlock(&cp->cache_lock); in umem_slab_alloc()
1587 umem_slab_free(umem_cache_t *cp, void *buf) in umem_slab_free() argument
1594 (void) mutex_lock(&cp->cache_lock); in umem_slab_free()
1595 cp->cache_slab_free++; in umem_slab_free()
1597 if (cp->cache_flags & UMF_HASH) { in umem_slab_free()
1601 prev_bcpp = UMEM_HASH(cp, buf); in umem_slab_free()
1608 cp->cache_lookup_depth++; in umem_slab_free()
1612 bcp = UMEM_BUFCTL(cp, buf); in umem_slab_free()
1613 sp = UMEM_SLAB(cp, buf); in umem_slab_free()
1616 if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) { in umem_slab_free()
1617 (void) mutex_unlock(&cp->cache_lock); in umem_slab_free()
1618 umem_error(UMERR_BADADDR, cp, buf); in umem_slab_free()
1622 if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { in umem_slab_free()
1623 if (cp->cache_flags & UMF_CONTENTS) in umem_slab_free()
1626 cp->cache_contents); in umem_slab_free()
1627 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_slab_free()
1635 ASSERT(cp->cache_freelist != sp); in umem_slab_free()
1638 sp->slab_next = cp->cache_freelist; in umem_slab_free()
1639 sp->slab_prev = cp->cache_freelist->slab_prev; in umem_slab_free()
1642 cp->cache_freelist = sp; in umem_slab_free()
1656 if (sp == cp->cache_freelist) in umem_slab_free()
1657 cp->cache_freelist = sp->slab_next; in umem_slab_free()
1658 cp->cache_slab_destroy++; in umem_slab_free()
1659 cp->cache_buftotal -= sp->slab_chunks; in umem_slab_free()
1660 (void) mutex_unlock(&cp->cache_lock); in umem_slab_free()
1661 umem_slab_destroy(cp, sp); in umem_slab_free()
1664 (void) mutex_unlock(&cp->cache_lock); in umem_slab_free()
1668 umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag) in umem_cache_alloc_debug() argument
1670 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in umem_cache_alloc_debug()
1676 umem_error(UMERR_BADBUFTAG, cp, buf); in umem_cache_alloc_debug()
1682 if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { in umem_cache_alloc_debug()
1683 umem_error(UMERR_BADBUFCTL, cp, buf); in umem_cache_alloc_debug()
1689 if (cp->cache_flags & UMF_DEADBEEF) { in umem_cache_alloc_debug()
1691 UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) { in umem_cache_alloc_debug()
1692 umem_error(UMERR_MODIFIED, cp, buf); in umem_cache_alloc_debug()
1697 if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 && in umem_cache_alloc_debug()
1700 umem_log_event(umem_failure_log, cp, NULL, NULL); in umem_cache_alloc_debug()
1710 if (mtbf || (cp->cache_constructor != NULL && in umem_cache_alloc_debug()
1711 cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) { in umem_cache_alloc_debug()
1712 atomic_add_64(&cp->cache_alloc_fail, 1); in umem_cache_alloc_debug()
1714 copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); in umem_cache_alloc_debug()
1715 umem_slab_free(cp, buf); in umem_cache_alloc_debug()
1719 if (cp->cache_flags & UMF_AUDIT) { in umem_cache_alloc_debug()
1720 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_cache_alloc_debug()
1727 umem_cache_free_debug(umem_cache_t *cp, void *buf) in umem_cache_free_debug() argument
1729 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in umem_cache_free_debug()
1735 umem_error(UMERR_DUPFREE, cp, buf); in umem_cache_free_debug()
1738 sp = umem_findslab(cp, buf); in umem_cache_free_debug()
1739 if (sp == NULL || sp->slab_cache != cp) in umem_cache_free_debug()
1740 umem_error(UMERR_BADADDR, cp, buf); in umem_cache_free_debug()
1742 umem_error(UMERR_REDZONE, cp, buf); in umem_cache_free_debug()
1748 if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { in umem_cache_free_debug()
1749 umem_error(UMERR_BADBUFCTL, cp, buf); in umem_cache_free_debug()
1754 umem_error(UMERR_REDZONE, cp, buf); in umem_cache_free_debug()
1758 if (cp->cache_flags & UMF_AUDIT) { in umem_cache_free_debug()
1759 if (cp->cache_flags & UMF_CONTENTS) in umem_cache_free_debug()
1761 buf, cp->cache_contents); in umem_cache_free_debug()
1762 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_cache_free_debug()
1765 if (cp->cache_destructor != NULL) in umem_cache_free_debug()
1766 cp->cache_destructor(buf, cp->cache_private); in umem_cache_free_debug()
1768 if (cp->cache_flags & UMF_DEADBEEF) in umem_cache_free_debug()
1769 copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); in umem_cache_free_debug()
1778 umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds) in umem_magazine_destroy() argument
1782 ASSERT(cp->cache_next == NULL || IN_UPDATE()); in umem_magazine_destroy()
1787 if ((cp->cache_flags & UMF_DEADBEEF) && in umem_magazine_destroy()
1789 cp->cache_verify) != NULL) { in umem_magazine_destroy()
1790 umem_error(UMERR_MODIFIED, cp, buf); in umem_magazine_destroy()
1794 if (!(cp->cache_flags & UMF_BUFTAG) && in umem_magazine_destroy()
1795 cp->cache_destructor != NULL) in umem_magazine_destroy()
1796 cp->cache_destructor(buf, cp->cache_private); in umem_magazine_destroy()
1798 umem_slab_free(cp, buf); in umem_magazine_destroy()
1800 ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); in umem_magazine_destroy()
1801 _umem_cache_free(cp->cache_magtype->mt_cache, mp); in umem_magazine_destroy()
1808 umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp) in umem_depot_alloc() argument
1818 if (mutex_trylock(&cp->cache_depot_lock) != 0) { in umem_depot_alloc()
1819 (void) mutex_lock(&cp->cache_depot_lock); in umem_depot_alloc()
1820 cp->cache_depot_contention++; in umem_depot_alloc()
1824 ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); in umem_depot_alloc()
1831 (void) mutex_unlock(&cp->cache_depot_lock); in umem_depot_alloc()
1840 umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp) in umem_depot_free() argument
1842 (void) mutex_lock(&cp->cache_depot_lock); in umem_depot_free()
1843 ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); in umem_depot_free()
1847 (void) mutex_unlock(&cp->cache_depot_lock); in umem_depot_free()
1854 umem_depot_ws_update(umem_cache_t *cp) in umem_depot_ws_update() argument
1856 (void) mutex_lock(&cp->cache_depot_lock); in umem_depot_ws_update()
1857 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; in umem_depot_ws_update()
1858 cp->cache_full.ml_min = cp->cache_full.ml_total; in umem_depot_ws_update()
1859 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; in umem_depot_ws_update()
1860 cp->cache_empty.ml_min = cp->cache_empty.ml_total; in umem_depot_ws_update()
1861 (void) mutex_unlock(&cp->cache_depot_lock); in umem_depot_ws_update()
1868 umem_depot_ws_reap(umem_cache_t *cp) in umem_depot_ws_reap() argument
1873 ASSERT(cp->cache_next == NULL || IN_REAP()); in umem_depot_ws_reap()
1875 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); in umem_depot_ws_reap()
1876 while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL) in umem_depot_ws_reap()
1877 umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); in umem_depot_ws_reap()
1879 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); in umem_depot_ws_reap()
1880 while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL) in umem_depot_ws_reap()
1881 umem_magazine_destroy(cp, mp, 0); in umem_depot_ws_reap()
1902 _umem_cache_alloc(umem_cache_t *cp, int umflag) in _umem_cache_alloc() argument
1910 ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); in _umem_cache_alloc()
1922 umem_cache_alloc_debug(cp, buf, umflag) == -1) { in _umem_cache_alloc()
1923 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
1950 fmp = umem_depot_alloc(cp, &cp->cache_full); in _umem_cache_alloc()
1953 umem_depot_free(cp, &cp->cache_empty, in _umem_cache_alloc()
1971 buf = umem_slab_alloc(cp, umflag); in _umem_cache_alloc()
1974 if (cp == &umem_null_cache) in _umem_cache_alloc()
1976 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
1983 if (cp->cache_flags & UMF_BUFTAG) { in _umem_cache_alloc()
1987 if (umem_cache_alloc_debug(cp, buf, umflag) == -1) { in _umem_cache_alloc()
1988 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
2001 if (cp->cache_constructor != NULL && in _umem_cache_alloc()
2002 cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) { in _umem_cache_alloc()
2003 atomic_add_64(&cp->cache_alloc_fail, 1); in _umem_cache_alloc()
2004 umem_slab_free(cp, buf); in _umem_cache_alloc()
2006 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
2020 _umem_cache_free(umem_cache_t *cp, void *buf) in _umem_cache_free() argument
2022 umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); in _umem_cache_free()
2027 if (umem_cache_free_debug(cp, buf) == -1) in _umem_cache_free()
2061 emp = umem_depot_alloc(cp, &cp->cache_empty); in _umem_cache_free()
2064 umem_depot_free(cp, &cp->cache_full, in _umem_cache_free()
2076 mtp = cp->cache_magtype; in _umem_cache_free()
2099 umem_depot_free(cp, &cp->cache_empty, emp); in _umem_cache_free()
2117 if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL) in _umem_cache_free()
2118 cp->cache_destructor(buf, cp->cache_private); in _umem_cache_free()
2120 umem_slab_free(cp, buf); in _umem_cache_free()
2132 umem_cache_t *cp = umem_alloc_table[index]; in _umem_zalloc() local
2133 buf = _umem_cache_alloc(cp, umflag); in _umem_zalloc()
2135 if (cp->cache_flags & UMF_BUFTAG) { in _umem_zalloc()
2136 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in _umem_zalloc()
2141 } else if (umem_alloc_retry(cp, umflag)) in _umem_zalloc()
2159 umem_cache_t *cp = umem_alloc_table[index]; in _umem_alloc() local
2160 buf = _umem_cache_alloc(cp, umflag); in _umem_alloc()
2161 if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) { in _umem_alloc()
2162 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in _umem_alloc()
2166 if (buf == NULL && umem_alloc_retry(cp, umflag)) in _umem_alloc()
2224 umem_cache_t *cp = umem_alloc_table[index]; in _umem_free() local
2225 if (cp->cache_flags & UMF_BUFTAG) { in _umem_free()
2226 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in _umem_free()
2230 umem_error(UMERR_DUPFREE, cp, buf); in _umem_free()
2235 umem_error(UMERR_BADSIZE, cp, buf); in _umem_free()
2237 umem_error(UMERR_REDZONE, cp, buf); in _umem_free()
2242 umem_error(UMERR_REDZONE, cp, buf); in _umem_free()
2247 _umem_cache_free(cp, buf); in _umem_free()
2294 umem_cache_reap(umem_cache_t *cp) in umem_cache_reap() argument
2303 if (cp->cache_reclaim != NULL) in umem_cache_reap()
2304 cp->cache_reclaim(cp->cache_private); in umem_cache_reap()
2306 umem_depot_ws_reap(cp); in umem_cache_reap()
2315 umem_cache_magazine_purge(umem_cache_t *cp) in umem_cache_magazine_purge() argument
2321 ASSERT(cp->cache_next == NULL || IN_UPDATE()); in umem_cache_magazine_purge()
2324 ccp = &cp->cache_cpu[cpu_seqid]; in umem_cache_magazine_purge()
2339 umem_magazine_destroy(cp, mp, rounds); in umem_cache_magazine_purge()
2341 umem_magazine_destroy(cp, pmp, prounds); in umem_cache_magazine_purge()
2349 umem_depot_ws_update(cp); in umem_cache_magazine_purge()
2350 umem_depot_ws_update(cp); in umem_cache_magazine_purge()
2352 umem_depot_ws_reap(cp); in umem_cache_magazine_purge()
2359 umem_cache_magazine_enable(umem_cache_t *cp) in umem_cache_magazine_enable() argument
2363 if (cp->cache_flags & UMF_NOMAGAZINE) in umem_cache_magazine_enable()
2367 umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; in umem_cache_magazine_enable()
2369 ccp->cc_magsize = cp->cache_magtype->mt_magsize; in umem_cache_magazine_enable()
2388 umem_cache_magazine_resize(umem_cache_t *cp) in umem_cache_magazine_resize() argument
2390 umem_magtype_t *mtp = cp->cache_magtype; in umem_cache_magazine_resize()
2394 if (cp->cache_chunksize < mtp->mt_maxbuf) { in umem_cache_magazine_resize()
2395 umem_cache_magazine_purge(cp); in umem_cache_magazine_resize()
2396 (void) mutex_lock(&cp->cache_depot_lock); in umem_cache_magazine_resize()
2397 cp->cache_magtype = ++mtp; in umem_cache_magazine_resize()
2398 cp->cache_depot_contention_prev = in umem_cache_magazine_resize()
2399 cp->cache_depot_contention + INT_MAX; in umem_cache_magazine_resize()
2400 (void) mutex_unlock(&cp->cache_depot_lock); in umem_cache_magazine_resize()
2401 umem_cache_magazine_enable(cp); in umem_cache_magazine_resize()
2410 umem_hash_rescale(umem_cache_t *cp) in umem_hash_rescale() argument
2418 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); in umem_hash_rescale()
2419 old_size = cp->cache_hash_mask + 1; in umem_hash_rescale()
2430 (void) mutex_lock(&cp->cache_lock); in umem_hash_rescale()
2432 old_size = cp->cache_hash_mask + 1; in umem_hash_rescale()
2433 old_table = cp->cache_hash_table; in umem_hash_rescale()
2435 cp->cache_hash_mask = new_size - 1; in umem_hash_rescale()
2436 cp->cache_hash_table = new_table; in umem_hash_rescale()
2437 cp->cache_rescale++; in umem_hash_rescale()
2444 umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr); in umem_hash_rescale()
2451 (void) mutex_unlock(&cp->cache_lock); in umem_hash_rescale()
2461 umem_cache_update(umem_cache_t *cp) in umem_cache_update() argument
2471 (void) mutex_lock(&cp->cache_lock); in umem_cache_update()
2473 if ((cp->cache_flags & UMF_HASH) && in umem_cache_update()
2474 (cp->cache_buftotal > (cp->cache_hash_mask << 1) || in umem_cache_update()
2475 (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && in umem_cache_update()
2476 cp->cache_hash_mask > UMEM_HASH_INITIAL))) in umem_cache_update()
2479 (void) mutex_unlock(&cp->cache_lock); in umem_cache_update()
2484 umem_depot_ws_update(cp); in umem_cache_update()
2490 (void) mutex_lock(&cp->cache_depot_lock); in umem_cache_update()
2492 if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && in umem_cache_update()
2493 (int)(cp->cache_depot_contention - in umem_cache_update()
2494 cp->cache_depot_contention_prev) > umem_depot_contention) in umem_cache_update()
2497 cp->cache_depot_contention_prev = cp->cache_depot_contention; in umem_cache_update()
2499 (void) mutex_unlock(&cp->cache_depot_lock); in umem_cache_update()
2502 umem_add_update(cp, update_flags); in umem_cache_update()
2517 umem_cache_t *cp = umem_null_cache.cache_unext; in umem_process_updates() local
2519 cp->cache_uprev->cache_unext = cp->cache_unext; in umem_process_updates()
2520 cp->cache_unext->cache_uprev = cp->cache_uprev; in umem_process_updates()
2521 cp->cache_uprev = cp->cache_unext = NULL; in umem_process_updates()
2523 ASSERT(!(cp->cache_uflags & UMU_ACTIVE)); in umem_process_updates()
2525 while (cp->cache_uflags) { in umem_process_updates()
2526 int uflags = (cp->cache_uflags |= UMU_ACTIVE); in umem_process_updates()
2535 umem_hash_rescale(cp); in umem_process_updates()
2538 umem_cache_magazine_resize(cp); in umem_process_updates()
2541 umem_cache_reap(cp); in umem_process_updates()
2548 if (cp->cache_uflags & UMU_NOTIFY) { in umem_process_updates()
2552 cp->cache_uflags &= ~uflags; in umem_process_updates()
2653 umem_cache_t *cp, *cnext, *cprev; in umem_cache_create() local
2711 cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase, in umem_cache_create()
2714 if (cp == NULL) { in umem_cache_create()
2719 bzero(cp, csize); in umem_cache_create()
2725 cp->cache_flags = umem_flags | (cflags & UMF_DEBUG); in umem_cache_create()
2731 if (cp->cache_flags & UMF_LITE) { in umem_cache_create()
2735 cp->cache_flags |= UMF_BUFTAG; in umem_cache_create()
2736 cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); in umem_cache_create()
2738 cp->cache_flags &= ~UMF_DEBUG; in umem_cache_create()
2742 if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT)) in umem_cache_create()
2743 cp->cache_flags |= UMF_NOMAGAZINE; in umem_cache_create()
2746 cp->cache_flags &= ~UMF_DEBUG; in umem_cache_create()
2749 cp->cache_flags &= ~UMF_TOUCH; in umem_cache_create()
2752 cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); in umem_cache_create()
2755 cp->cache_flags |= UMF_NOMAGAZINE; in umem_cache_create()
2757 if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH)) in umem_cache_create()
2758 cp->cache_flags |= UMF_REDZONE; in umem_cache_create()
2760 if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall && in umem_cache_create()
2761 !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH)) in umem_cache_create()
2762 cp->cache_flags |= UMF_FIREWALL; in umem_cache_create()
2765 cp->cache_flags &= ~UMF_FIREWALL; in umem_cache_create()
2767 if (cp->cache_flags & UMF_FIREWALL) { in umem_cache_create()
2768 cp->cache_flags &= ~UMF_BUFTAG; in umem_cache_create()
2769 cp->cache_flags |= UMF_NOMAGAZINE; in umem_cache_create()
2777 (void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1); in umem_cache_create()
2778 cp->cache_bufsize = bufsize; in umem_cache_create()
2779 cp->cache_align = align; in umem_cache_create()
2780 cp->cache_constructor = constructor; in umem_cache_create()
2781 cp->cache_destructor = destructor; in umem_cache_create()
2782 cp->cache_reclaim = reclaim; in umem_cache_create()
2783 cp->cache_private = private; in umem_cache_create()
2784 cp->cache_arena = vmp; in umem_cache_create()
2785 cp->cache_cflags = cflags; in umem_cache_create()
2786 cp->cache_cpu_mask = umem_cpu_mask; in umem_cache_create()
2795 cp->cache_bufctl = chunksize - UMEM_ALIGN; in umem_cache_create()
2798 if (cp->cache_flags & UMF_BUFTAG) { in umem_cache_create()
2799 cp->cache_bufctl = chunksize; in umem_cache_create()
2800 cp->cache_buftag = chunksize; in umem_cache_create()
2804 if (cp->cache_flags & UMF_DEADBEEF) { in umem_cache_create()
2805 cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify); in umem_cache_create()
2806 if (cp->cache_flags & UMF_LITE) in umem_cache_create()
2807 cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN); in umem_cache_create()
2810 cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave); in umem_cache_create()
2812 cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); in umem_cache_create()
2823 cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); in umem_cache_create()
2824 cp->cache_mincolor = cp->cache_slabsize - chunksize; in umem_cache_create()
2825 cp->cache_maxcolor = cp->cache_mincolor; in umem_cache_create()
2826 cp->cache_flags |= UMF_HASH; in umem_cache_create()
2827 ASSERT(!(cp->cache_flags & UMF_BUFTAG)); in umem_cache_create()
2829 !(cp->cache_flags & UMF_AUDIT) && in umem_cache_create()
2831 cp->cache_slabsize = vmp->vm_quantum; in umem_cache_create()
2832 cp->cache_mincolor = 0; in umem_cache_create()
2833 cp->cache_maxcolor = in umem_cache_create()
2834 (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize; in umem_cache_create()
2836 if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) { in umem_cache_create()
2840 ASSERT(!(cp->cache_flags & UMF_AUDIT)); in umem_cache_create()
2864 cp->cache_slabsize = bestfit; in umem_cache_create()
2865 cp->cache_mincolor = 0; in umem_cache_create()
2866 cp->cache_maxcolor = bestfit % chunksize; in umem_cache_create()
2867 cp->cache_flags |= UMF_HASH; in umem_cache_create()
2870 if (cp->cache_flags & UMF_HASH) { in umem_cache_create()
2872 cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ? in umem_cache_create()
2876 if (cp->cache_maxcolor >= vmp->vm_quantum) in umem_cache_create()
2877 cp->cache_maxcolor = vmp->vm_quantum - 1; in umem_cache_create()
2879 cp->cache_color = cp->cache_mincolor; in umem_cache_create()
2884 (void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL); in umem_cache_create()
2886 cp->cache_freelist = &cp->cache_nullslab; in umem_cache_create()
2887 cp->cache_nullslab.slab_cache = cp; in umem_cache_create()
2888 cp->cache_nullslab.slab_refcnt = -1; in umem_cache_create()
2889 cp->cache_nullslab.slab_next = &cp->cache_nullslab; in umem_cache_create()
2890 cp->cache_nullslab.slab_prev = &cp->cache_nullslab; in umem_cache_create()
2892 if (cp->cache_flags & UMF_HASH) { in umem_cache_create()
2893 cp->cache_hash_table = vmem_alloc(umem_hash_arena, in umem_cache_create()
2895 if (cp->cache_hash_table == NULL) { in umem_cache_create()
2899 bzero(cp->cache_hash_table, in umem_cache_create()
2901 cp->cache_hash_mask = UMEM_HASH_INITIAL - 1; in umem_cache_create()
2902 cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; in umem_cache_create()
2908 (void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL); in umem_cache_create()
2913 cp->cache_magtype = mtp; in umem_cache_create()
2919 umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; in umem_cache_create()
2921 ccp->cc_flags = cp->cache_flags; in umem_cache_create()
2931 cp->cache_next = cnext = &umem_null_cache; in umem_cache_create()
2932 cp->cache_prev = cprev = umem_null_cache.cache_prev; in umem_cache_create()
2933 cnext->cache_prev = cp; in umem_cache_create()
2934 cprev->cache_next = cp; in umem_cache_create()
2938 umem_cache_magazine_enable(cp); in umem_cache_create()
2940 return (cp); in umem_cache_create()
2943 (void) mutex_destroy(&cp->cache_lock); in umem_cache_create()
2945 vmem_xfree(umem_cache_arena, cp, csize); in umem_cache_create()
2950 umem_cache_destroy(umem_cache_t *cp) in umem_cache_destroy() argument
2960 cp->cache_prev->cache_next = cp->cache_next; in umem_cache_destroy()
2961 cp->cache_next->cache_prev = cp->cache_prev; in umem_cache_destroy()
2962 cp->cache_prev = cp->cache_next = NULL; in umem_cache_destroy()
2965 umem_remove_updates(cp); in umem_cache_destroy()
2967 umem_cache_magazine_purge(cp); in umem_cache_destroy()
2969 (void) mutex_lock(&cp->cache_lock); in umem_cache_destroy()
2970 if (cp->cache_buftotal != 0) in umem_cache_destroy()
2972 cp->cache_name, (void *)cp); in umem_cache_destroy()
2973 cp->cache_reclaim = NULL; in umem_cache_destroy()
2979 cp->cache_constructor = (umem_constructor_t *)1; in umem_cache_destroy()
2980 cp->cache_destructor = (umem_destructor_t *)2; in umem_cache_destroy()
2981 (void) mutex_unlock(&cp->cache_lock); in umem_cache_destroy()
2983 if (cp->cache_hash_table != NULL) in umem_cache_destroy()
2984 vmem_free(umem_hash_arena, cp->cache_hash_table, in umem_cache_destroy()
2985 (cp->cache_hash_mask + 1) * sizeof (void *)); in umem_cache_destroy()
2988 (void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); in umem_cache_destroy()
2990 (void) mutex_destroy(&cp->cache_depot_lock); in umem_cache_destroy()
2991 (void) mutex_destroy(&cp->cache_lock); in umem_cache_destroy()
2993 vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus)); in umem_cache_destroy()
3089 umem_cache_t *cp; in umem_cache_tmem_cleanup() local
3092 cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT]; in umem_cache_tmem_cleanup()
3093 _umem_cache_free(cp, buf); in umem_cache_tmem_cleanup()
3101 umem_cache_t *cp; in umem_cache_init() local
3203 cp = umem_cache_create(name, cache_size, align, in umem_cache_init()
3205 if (cp == NULL) in umem_cache_init()
3208 umem_alloc_caches[i] = cp; in umem_cache_init()
3232 cp = umem_alloc_caches[i]; in umem_cache_init()
3235 umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp; in umem_cache_init()