1*7c478bd9Sstevel@tonic-gate /* 2*7c478bd9Sstevel@tonic-gate * CDDL HEADER START 3*7c478bd9Sstevel@tonic-gate * 4*7c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*7c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*7c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*7c478bd9Sstevel@tonic-gate * with the License. 8*7c478bd9Sstevel@tonic-gate * 9*7c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*7c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*7c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 12*7c478bd9Sstevel@tonic-gate * and limitations under the License. 13*7c478bd9Sstevel@tonic-gate * 14*7c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*7c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*7c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*7c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*7c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*7c478bd9Sstevel@tonic-gate * 20*7c478bd9Sstevel@tonic-gate * CDDL HEADER END 21*7c478bd9Sstevel@tonic-gate */ 22*7c478bd9Sstevel@tonic-gate /* 23*7c478bd9Sstevel@tonic-gate * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24*7c478bd9Sstevel@tonic-gate * Use is subject to license terms. 25*7c478bd9Sstevel@tonic-gate */ 26*7c478bd9Sstevel@tonic-gate 27*7c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*7c478bd9Sstevel@tonic-gate 29*7c478bd9Sstevel@tonic-gate /* 30*7c478bd9Sstevel@tonic-gate * The following routines implement the hat layer's 31*7c478bd9Sstevel@tonic-gate * recording of the referenced and modified bits. 32*7c478bd9Sstevel@tonic-gate */ 33*7c478bd9Sstevel@tonic-gate 34*7c478bd9Sstevel@tonic-gate #include <sys/types.h> 35*7c478bd9Sstevel@tonic-gate #include <sys/param.h> 36*7c478bd9Sstevel@tonic-gate #include <sys/systm.h> 37*7c478bd9Sstevel@tonic-gate #include <sys/debug.h> 38*7c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 39*7c478bd9Sstevel@tonic-gate 40*7c478bd9Sstevel@tonic-gate /* 41*7c478bd9Sstevel@tonic-gate * Note, usage of cmn_err requires you not hold any hat layer locks. 42*7c478bd9Sstevel@tonic-gate */ 43*7c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 44*7c478bd9Sstevel@tonic-gate 45*7c478bd9Sstevel@tonic-gate #include <vm/as.h> 46*7c478bd9Sstevel@tonic-gate #include <vm/hat.h> 47*7c478bd9Sstevel@tonic-gate 48*7c478bd9Sstevel@tonic-gate kmutex_t hat_statlock; /* protects all hat statistics data */ 49*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm_memlist; /* tracks memory alloced for hrm_blist blocks */ 50*7c478bd9Sstevel@tonic-gate struct hrmstat **hrm_hashtab; /* hash table for finding blocks quickly */ 51*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm_blist; 52*7c478bd9Sstevel@tonic-gate int hrm_blist_incr = HRM_BLIST_INCR; 53*7c478bd9Sstevel@tonic-gate int hrm_blist_lowater = HRM_BLIST_INCR/2; 54*7c478bd9Sstevel@tonic-gate int hrm_blist_num = 0; 55*7c478bd9Sstevel@tonic-gate int hrm_blist_total = 0; 56*7c478bd9Sstevel@tonic-gate int hrm_mlockinited = 0; 57*7c478bd9Sstevel@tonic-gate int hrm_allocfailmsg = 0; /* print a message when allocations fail */ 58*7c478bd9Sstevel@tonic-gate int hrm_allocfail = 0; 59*7c478bd9Sstevel@tonic-gate 60*7c478bd9Sstevel@tonic-gate static struct hrmstat *hrm_balloc(void); 61*7c478bd9Sstevel@tonic-gate static int hrm_init(void); 62*7c478bd9Sstevel@tonic-gate static void hrm_link(struct hrmstat *); 63*7c478bd9Sstevel@tonic-gate static void hrm_setbits(struct hrmstat *, caddr_t, uint_t); 64*7c478bd9Sstevel@tonic-gate static void hrm_hashout(struct hrmstat *); 65*7c478bd9Sstevel@tonic-gate static void hrm_getblk(int); 66*7c478bd9Sstevel@tonic-gate 67*7c478bd9Sstevel@tonic-gate #define hrm_hash(as, addr) \ 68*7c478bd9Sstevel@tonic-gate (HRM_HASHMASK & \ 69*7c478bd9Sstevel@tonic-gate (((uintptr_t)(addr) >> HRM_BASESHIFT) ^ ((uintptr_t)(as) >> 2))) 70*7c478bd9Sstevel@tonic-gate 71*7c478bd9Sstevel@tonic-gate #define hrm_match(hrm, as, addr) \ 72*7c478bd9Sstevel@tonic-gate (((hrm)->hrm_as == (as) && \ 73*7c478bd9Sstevel@tonic-gate ((hrm)->hrm_base == ((uintptr_t)(addr) & HRM_BASEMASK))) ? 1 : 0) 74*7c478bd9Sstevel@tonic-gate 75*7c478bd9Sstevel@tonic-gate /* 76*7c478bd9Sstevel@tonic-gate * reserve enough statistic blocks for 77*7c478bd9Sstevel@tonic-gate * chunk of bytes (pages) in a given as. 78*7c478bd9Sstevel@tonic-gate */ 79*7c478bd9Sstevel@tonic-gate /* ARGSUSED */ 80*7c478bd9Sstevel@tonic-gate void 81*7c478bd9Sstevel@tonic-gate hat_resvstat(size_t chunk, struct as *as, caddr_t addr) 82*7c478bd9Sstevel@tonic-gate { 83*7c478bd9Sstevel@tonic-gate int nhrm = btop(chunk)/HRM_PAGES; 84*7c478bd9Sstevel@tonic-gate 85*7c478bd9Sstevel@tonic-gate if (nhrm < HRM_BLIST_INCR) 86*7c478bd9Sstevel@tonic-gate nhrm = 0; /* preallocate at least HRM_BLIST_INCR */ 87*7c478bd9Sstevel@tonic-gate hrm_getblk(nhrm); 88*7c478bd9Sstevel@tonic-gate } 89*7c478bd9Sstevel@tonic-gate 90*7c478bd9Sstevel@tonic-gate /* 91*7c478bd9Sstevel@tonic-gate * Start the statistics gathering for an address space. 92*7c478bd9Sstevel@tonic-gate * Return -1 if we can't do it, otherwise return an opaque 93*7c478bd9Sstevel@tonic-gate * identifier to be used when querying for the gathered statistics. 94*7c478bd9Sstevel@tonic-gate * The identifier is an unused bit in a_vbits. 95*7c478bd9Sstevel@tonic-gate * Bit 0 is reserved for swsmon. 96*7c478bd9Sstevel@tonic-gate */ 97*7c478bd9Sstevel@tonic-gate int 98*7c478bd9Sstevel@tonic-gate hat_startstat(struct as *as) 99*7c478bd9Sstevel@tonic-gate { 100*7c478bd9Sstevel@tonic-gate uint_t nbits; /* number of bits */ 101*7c478bd9Sstevel@tonic-gate uint_t bn; /* bit number */ 102*7c478bd9Sstevel@tonic-gate uint_t id; /* new vbit, identifier */ 103*7c478bd9Sstevel@tonic-gate uint_t vbits; /* used vbits of address space */ 104*7c478bd9Sstevel@tonic-gate size_t chunk; /* mapped size for stats */ 105*7c478bd9Sstevel@tonic-gate /* 106*7c478bd9Sstevel@tonic-gate * Initialize global data, if needed. 107*7c478bd9Sstevel@tonic-gate */ 108*7c478bd9Sstevel@tonic-gate if (hrm_init() == -1) 109*7c478bd9Sstevel@tonic-gate return (-1); 110*7c478bd9Sstevel@tonic-gate 111*7c478bd9Sstevel@tonic-gate /* 112*7c478bd9Sstevel@tonic-gate * If the refmod saving memory allocator runs out, print 113*7c478bd9Sstevel@tonic-gate * a warning message about how to fix it, see comment at 114*7c478bd9Sstevel@tonic-gate * the beginning of hat_setstat. 115*7c478bd9Sstevel@tonic-gate */ 116*7c478bd9Sstevel@tonic-gate if (hrm_allocfailmsg) { 117*7c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 118*7c478bd9Sstevel@tonic-gate "hrm_balloc failures occured, increase hrm_blist_incr"); 119*7c478bd9Sstevel@tonic-gate hrm_allocfailmsg = 0; 120*7c478bd9Sstevel@tonic-gate } 121*7c478bd9Sstevel@tonic-gate 122*7c478bd9Sstevel@tonic-gate /* 123*7c478bd9Sstevel@tonic-gate * Verify that a buffer of statistics blocks exists 124*7c478bd9Sstevel@tonic-gate * and allocate more, if needed. 125*7c478bd9Sstevel@tonic-gate */ 126*7c478bd9Sstevel@tonic-gate 127*7c478bd9Sstevel@tonic-gate chunk = hat_get_mapped_size(as->a_hat); 128*7c478bd9Sstevel@tonic-gate chunk = (btop(chunk)/HRM_PAGES); 129*7c478bd9Sstevel@tonic-gate if (chunk < HRM_BLIST_INCR) 130*7c478bd9Sstevel@tonic-gate chunk = 0; 131*7c478bd9Sstevel@tonic-gate 132*7c478bd9Sstevel@tonic-gate hrm_getblk((int)chunk); 133*7c478bd9Sstevel@tonic-gate 134*7c478bd9Sstevel@tonic-gate /* 135*7c478bd9Sstevel@tonic-gate * Find a unused id in the given address space. 136*7c478bd9Sstevel@tonic-gate */ 137*7c478bd9Sstevel@tonic-gate hat_enter(as->a_hat); 138*7c478bd9Sstevel@tonic-gate vbits = as->a_vbits; 139*7c478bd9Sstevel@tonic-gate nbits = sizeof (as->a_vbits) * NBBY; 140*7c478bd9Sstevel@tonic-gate for (bn = 1, id = 2; bn < (nbits - 1); bn++, id <<= 1) 141*7c478bd9Sstevel@tonic-gate if ((id & vbits) == 0) 142*7c478bd9Sstevel@tonic-gate break; 143*7c478bd9Sstevel@tonic-gate if (bn >= (nbits - 1)) { 144*7c478bd9Sstevel@tonic-gate hat_exit(as->a_hat); 145*7c478bd9Sstevel@tonic-gate return (-1); 146*7c478bd9Sstevel@tonic-gate } 147*7c478bd9Sstevel@tonic-gate as->a_vbits |= id; 148*7c478bd9Sstevel@tonic-gate hat_exit(as->a_hat); 149*7c478bd9Sstevel@tonic-gate (void) hat_stats_enable(as->a_hat); 150*7c478bd9Sstevel@tonic-gate return (id); 151*7c478bd9Sstevel@tonic-gate } 152*7c478bd9Sstevel@tonic-gate 153*7c478bd9Sstevel@tonic-gate /* 154*7c478bd9Sstevel@tonic-gate * Record referenced and modified information for an address space. 155*7c478bd9Sstevel@tonic-gate * Rmbits is a word containing the referenced bit in bit position 1 156*7c478bd9Sstevel@tonic-gate * and the modified bit in bit position 0. 157*7c478bd9Sstevel@tonic-gate * 158*7c478bd9Sstevel@tonic-gate * For current informational uses, one can rerun any program using 159*7c478bd9Sstevel@tonic-gate * this facility after modifying the hrm_blist_incr to be a larger 160*7c478bd9Sstevel@tonic-gate * amount so that a larger buffer of blocks will be maintained. 161*7c478bd9Sstevel@tonic-gate */ 162*7c478bd9Sstevel@tonic-gate void 163*7c478bd9Sstevel@tonic-gate hat_setstat(struct as *as, caddr_t addr, size_t len, uint_t rmbits) 164*7c478bd9Sstevel@tonic-gate { 165*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm; 166*7c478bd9Sstevel@tonic-gate uint_t vbits, newbits, nb; 167*7c478bd9Sstevel@tonic-gate int h; 168*7c478bd9Sstevel@tonic-gate 169*7c478bd9Sstevel@tonic-gate ASSERT(len == PAGESIZE); 170*7c478bd9Sstevel@tonic-gate ASSERT((rmbits & ~(P_MOD|P_REF)) == 0); 171*7c478bd9Sstevel@tonic-gate 172*7c478bd9Sstevel@tonic-gate if (rmbits == 0) 173*7c478bd9Sstevel@tonic-gate return; 174*7c478bd9Sstevel@tonic-gate 175*7c478bd9Sstevel@tonic-gate /* 176*7c478bd9Sstevel@tonic-gate * Initialize global data, if needed. 177*7c478bd9Sstevel@tonic-gate */ 178*7c478bd9Sstevel@tonic-gate if (hrm_init() == -1) 179*7c478bd9Sstevel@tonic-gate return; 180*7c478bd9Sstevel@tonic-gate 181*7c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock); 182*7c478bd9Sstevel@tonic-gate 183*7c478bd9Sstevel@tonic-gate /* 184*7c478bd9Sstevel@tonic-gate * Search the hash list for the as and addr we are looking for 185*7c478bd9Sstevel@tonic-gate * and set the ref and mod bits in every block that matches. 186*7c478bd9Sstevel@tonic-gate */ 187*7c478bd9Sstevel@tonic-gate vbits = 0; 188*7c478bd9Sstevel@tonic-gate h = hrm_hash(as, addr); 189*7c478bd9Sstevel@tonic-gate for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) { 190*7c478bd9Sstevel@tonic-gate if (hrm_match(hrm, as, addr)) { 191*7c478bd9Sstevel@tonic-gate hrm_setbits(hrm, addr, rmbits); 192*7c478bd9Sstevel@tonic-gate vbits |= hrm->hrm_id; 193*7c478bd9Sstevel@tonic-gate } 194*7c478bd9Sstevel@tonic-gate } 195*7c478bd9Sstevel@tonic-gate 196*7c478bd9Sstevel@tonic-gate /* 197*7c478bd9Sstevel@tonic-gate * If we didn't find a block for all of the enabled 198*7c478bd9Sstevel@tonic-gate * vpages bits, then allocate and initialize a block 199*7c478bd9Sstevel@tonic-gate * for each bit that was not found. 200*7c478bd9Sstevel@tonic-gate */ 201*7c478bd9Sstevel@tonic-gate if (vbits != as->a_vbits) { 202*7c478bd9Sstevel@tonic-gate newbits = vbits ^ as->a_vbits; 203*7c478bd9Sstevel@tonic-gate while (newbits) { 204*7c478bd9Sstevel@tonic-gate if (ffs(newbits)) 205*7c478bd9Sstevel@tonic-gate nb = 1 << (ffs(newbits)-1); 206*7c478bd9Sstevel@tonic-gate hrm = (struct hrmstat *)hrm_balloc(); 207*7c478bd9Sstevel@tonic-gate if (hrm == NULL) { 208*7c478bd9Sstevel@tonic-gate hrm_allocfailmsg = 1; 209*7c478bd9Sstevel@tonic-gate hrm_allocfail++; 210*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 211*7c478bd9Sstevel@tonic-gate return; 212*7c478bd9Sstevel@tonic-gate } 213*7c478bd9Sstevel@tonic-gate hrm->hrm_as = as; 214*7c478bd9Sstevel@tonic-gate hrm->hrm_base = (uintptr_t)addr & HRM_BASEMASK; 215*7c478bd9Sstevel@tonic-gate hrm->hrm_id = nb; 216*7c478bd9Sstevel@tonic-gate hrm_link(hrm); 217*7c478bd9Sstevel@tonic-gate hrm_setbits(hrm, addr, rmbits); 218*7c478bd9Sstevel@tonic-gate newbits &= ~nb; 219*7c478bd9Sstevel@tonic-gate } 220*7c478bd9Sstevel@tonic-gate } 221*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 222*7c478bd9Sstevel@tonic-gate } 223*7c478bd9Sstevel@tonic-gate 224*7c478bd9Sstevel@tonic-gate /* 225*7c478bd9Sstevel@tonic-gate * Free the resources used to maintain the referenced and modified 226*7c478bd9Sstevel@tonic-gate * statistics for the virtual page view of an address space 227*7c478bd9Sstevel@tonic-gate * identified by id. 228*7c478bd9Sstevel@tonic-gate */ 229*7c478bd9Sstevel@tonic-gate void 230*7c478bd9Sstevel@tonic-gate hat_freestat(struct as *as, int id) 231*7c478bd9Sstevel@tonic-gate { 232*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm, *prev_ahrm; 233*7c478bd9Sstevel@tonic-gate 234*7c478bd9Sstevel@tonic-gate hat_stats_disable(as->a_hat); /* tell the hat layer to stop */ 235*7c478bd9Sstevel@tonic-gate hat_enter(as->a_hat); 236*7c478bd9Sstevel@tonic-gate if (id == 0) 237*7c478bd9Sstevel@tonic-gate as->a_vbits = 0; 238*7c478bd9Sstevel@tonic-gate else 239*7c478bd9Sstevel@tonic-gate as->a_vbits &= ~id; 240*7c478bd9Sstevel@tonic-gate 241*7c478bd9Sstevel@tonic-gate if ((hrm = as->a_hrm) == NULL) { 242*7c478bd9Sstevel@tonic-gate hat_exit(as->a_hat); 243*7c478bd9Sstevel@tonic-gate return; 244*7c478bd9Sstevel@tonic-gate } 245*7c478bd9Sstevel@tonic-gate hat_exit(as->a_hat); 246*7c478bd9Sstevel@tonic-gate 247*7c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock); 248*7c478bd9Sstevel@tonic-gate if (hrm_hashtab == NULL) { 249*7c478bd9Sstevel@tonic-gate /* can't happen? */ 250*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 251*7c478bd9Sstevel@tonic-gate return; 252*7c478bd9Sstevel@tonic-gate } 253*7c478bd9Sstevel@tonic-gate for (prev_ahrm = NULL; hrm; hrm = hrm->hrm_anext) { 254*7c478bd9Sstevel@tonic-gate if ((id == hrm->hrm_id) || (id == NULL)) { 255*7c478bd9Sstevel@tonic-gate 256*7c478bd9Sstevel@tonic-gate hrm_hashout(hrm); 257*7c478bd9Sstevel@tonic-gate hrm->hrm_hnext = hrm_blist; 258*7c478bd9Sstevel@tonic-gate hrm_blist = hrm; 259*7c478bd9Sstevel@tonic-gate hrm_blist_num++; 260*7c478bd9Sstevel@tonic-gate 261*7c478bd9Sstevel@tonic-gate if (prev_ahrm == NULL) 262*7c478bd9Sstevel@tonic-gate as->a_hrm = hrm->hrm_anext; 263*7c478bd9Sstevel@tonic-gate else 264*7c478bd9Sstevel@tonic-gate prev_ahrm->hrm_anext = hrm->hrm_anext; 265*7c478bd9Sstevel@tonic-gate 266*7c478bd9Sstevel@tonic-gate } else 267*7c478bd9Sstevel@tonic-gate prev_ahrm = hrm; 268*7c478bd9Sstevel@tonic-gate } 269*7c478bd9Sstevel@tonic-gate 270*7c478bd9Sstevel@tonic-gate /* 271*7c478bd9Sstevel@tonic-gate * If all statistics blocks are free, 272*7c478bd9Sstevel@tonic-gate * return the memory to the system. 273*7c478bd9Sstevel@tonic-gate */ 274*7c478bd9Sstevel@tonic-gate if (hrm_blist_num == hrm_blist_total) { 275*7c478bd9Sstevel@tonic-gate /* zero the block list since we are giving back its memory */ 276*7c478bd9Sstevel@tonic-gate hrm_blist = NULL; 277*7c478bd9Sstevel@tonic-gate hrm_blist_num = 0; 278*7c478bd9Sstevel@tonic-gate hrm_blist_total = 0; 279*7c478bd9Sstevel@tonic-gate while (hrm_memlist) { 280*7c478bd9Sstevel@tonic-gate hrm = hrm_memlist; 281*7c478bd9Sstevel@tonic-gate hrm_memlist = hrm->hrm_hnext; 282*7c478bd9Sstevel@tonic-gate kmem_free(hrm, hrm->hrm_base); 283*7c478bd9Sstevel@tonic-gate } 284*7c478bd9Sstevel@tonic-gate ASSERT(hrm_memlist == NULL); 285*7c478bd9Sstevel@tonic-gate kmem_free(hrm_hashtab, HRM_HASHSIZE * sizeof (char *)); 286*7c478bd9Sstevel@tonic-gate hrm_hashtab = NULL; 287*7c478bd9Sstevel@tonic-gate } 288*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 289*7c478bd9Sstevel@tonic-gate } 290*7c478bd9Sstevel@tonic-gate 291*7c478bd9Sstevel@tonic-gate /* 292*7c478bd9Sstevel@tonic-gate * Initialize any global state for the statistics handling. 293*7c478bd9Sstevel@tonic-gate * Hrm_lock protects the globally allocted memory: 294*7c478bd9Sstevel@tonic-gate * hrm_memlist and hrm_hashtab. 295*7c478bd9Sstevel@tonic-gate */ 296*7c478bd9Sstevel@tonic-gate static int 297*7c478bd9Sstevel@tonic-gate hrm_init(void) 298*7c478bd9Sstevel@tonic-gate { 299*7c478bd9Sstevel@tonic-gate /* 300*7c478bd9Sstevel@tonic-gate * Alloacte the hashtable if it doesn't exist yet. 301*7c478bd9Sstevel@tonic-gate */ 302*7c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock); 303*7c478bd9Sstevel@tonic-gate if (hrm_hashtab == NULL) 304*7c478bd9Sstevel@tonic-gate hrm_hashtab = 305*7c478bd9Sstevel@tonic-gate kmem_zalloc(HRM_HASHSIZE * sizeof (char *), KM_SLEEP); 306*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 307*7c478bd9Sstevel@tonic-gate return (0); 308*7c478bd9Sstevel@tonic-gate } 309*7c478bd9Sstevel@tonic-gate 310*7c478bd9Sstevel@tonic-gate /* 311*7c478bd9Sstevel@tonic-gate * Grab memory for statistics gathering of the hat layer. 312*7c478bd9Sstevel@tonic-gate */ 313*7c478bd9Sstevel@tonic-gate static void 314*7c478bd9Sstevel@tonic-gate hrm_getblk(int chunk) 315*7c478bd9Sstevel@tonic-gate { 316*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm, *l; 317*7c478bd9Sstevel@tonic-gate int i; 318*7c478bd9Sstevel@tonic-gate int hrm_incr; 319*7c478bd9Sstevel@tonic-gate 320*7c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock); 321*7c478bd9Sstevel@tonic-gate if ((hrm_blist == NULL) || 322*7c478bd9Sstevel@tonic-gate (hrm_blist_num <= hrm_blist_lowater) || 323*7c478bd9Sstevel@tonic-gate chunk) { 324*7c478bd9Sstevel@tonic-gate 325*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 326*7c478bd9Sstevel@tonic-gate 327*7c478bd9Sstevel@tonic-gate hrm_incr = chunk? chunk : hrm_blist_incr; 328*7c478bd9Sstevel@tonic-gate hrm = kmem_zalloc(sizeof (struct hrmstat) * hrm_incr, KM_SLEEP); 329*7c478bd9Sstevel@tonic-gate hrm->hrm_base = sizeof (struct hrmstat) * hrm_incr; 330*7c478bd9Sstevel@tonic-gate 331*7c478bd9Sstevel@tonic-gate /* 332*7c478bd9Sstevel@tonic-gate * thread the allocated blocks onto a freelist 333*7c478bd9Sstevel@tonic-gate * using the first block to hold information for 334*7c478bd9Sstevel@tonic-gate * freeing them all later 335*7c478bd9Sstevel@tonic-gate */ 336*7c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock); 337*7c478bd9Sstevel@tonic-gate hrm->hrm_hnext = hrm_memlist; 338*7c478bd9Sstevel@tonic-gate hrm_memlist = hrm; 339*7c478bd9Sstevel@tonic-gate 340*7c478bd9Sstevel@tonic-gate hrm_blist_total += (hrm_incr - 1); 341*7c478bd9Sstevel@tonic-gate for (i = 1; i < hrm_incr; i++) { 342*7c478bd9Sstevel@tonic-gate l = &hrm[i]; 343*7c478bd9Sstevel@tonic-gate l->hrm_hnext = hrm_blist; 344*7c478bd9Sstevel@tonic-gate hrm_blist = l; 345*7c478bd9Sstevel@tonic-gate hrm_blist_num++; 346*7c478bd9Sstevel@tonic-gate } 347*7c478bd9Sstevel@tonic-gate } 348*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 349*7c478bd9Sstevel@tonic-gate } 350*7c478bd9Sstevel@tonic-gate 351*7c478bd9Sstevel@tonic-gate static void 352*7c478bd9Sstevel@tonic-gate hrm_hashin(struct hrmstat *hrm) 353*7c478bd9Sstevel@tonic-gate { 354*7c478bd9Sstevel@tonic-gate int h; 355*7c478bd9Sstevel@tonic-gate 356*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 357*7c478bd9Sstevel@tonic-gate h = hrm_hash(hrm->hrm_as, hrm->hrm_base); 358*7c478bd9Sstevel@tonic-gate 359*7c478bd9Sstevel@tonic-gate hrm->hrm_hnext = hrm_hashtab[h]; 360*7c478bd9Sstevel@tonic-gate hrm_hashtab[h] = hrm; 361*7c478bd9Sstevel@tonic-gate } 362*7c478bd9Sstevel@tonic-gate 363*7c478bd9Sstevel@tonic-gate static void 364*7c478bd9Sstevel@tonic-gate hrm_hashout(struct hrmstat *hrm) 365*7c478bd9Sstevel@tonic-gate { 366*7c478bd9Sstevel@tonic-gate struct hrmstat *list, **prev_hrm; 367*7c478bd9Sstevel@tonic-gate int h; 368*7c478bd9Sstevel@tonic-gate 369*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 370*7c478bd9Sstevel@tonic-gate h = hrm_hash(hrm->hrm_as, hrm->hrm_base); 371*7c478bd9Sstevel@tonic-gate list = hrm_hashtab[h]; 372*7c478bd9Sstevel@tonic-gate prev_hrm = &hrm_hashtab[h]; 373*7c478bd9Sstevel@tonic-gate 374*7c478bd9Sstevel@tonic-gate while (list) { 375*7c478bd9Sstevel@tonic-gate if (list == hrm) { 376*7c478bd9Sstevel@tonic-gate *prev_hrm = list->hrm_hnext; 377*7c478bd9Sstevel@tonic-gate return; 378*7c478bd9Sstevel@tonic-gate } 379*7c478bd9Sstevel@tonic-gate prev_hrm = &list->hrm_hnext; 380*7c478bd9Sstevel@tonic-gate list = list->hrm_hnext; 381*7c478bd9Sstevel@tonic-gate } 382*7c478bd9Sstevel@tonic-gate } 383*7c478bd9Sstevel@tonic-gate 384*7c478bd9Sstevel@tonic-gate 385*7c478bd9Sstevel@tonic-gate /* 386*7c478bd9Sstevel@tonic-gate * Link a statistic block into an address space and also put it 387*7c478bd9Sstevel@tonic-gate * on the hash list for future references. 388*7c478bd9Sstevel@tonic-gate */ 389*7c478bd9Sstevel@tonic-gate static void 390*7c478bd9Sstevel@tonic-gate hrm_link(struct hrmstat *hrm) 391*7c478bd9Sstevel@tonic-gate { 392*7c478bd9Sstevel@tonic-gate struct as *as = hrm->hrm_as; 393*7c478bd9Sstevel@tonic-gate 394*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 395*7c478bd9Sstevel@tonic-gate hrm->hrm_anext = as->a_hrm; 396*7c478bd9Sstevel@tonic-gate as->a_hrm = hrm; 397*7c478bd9Sstevel@tonic-gate hrm_hashin(hrm); 398*7c478bd9Sstevel@tonic-gate } 399*7c478bd9Sstevel@tonic-gate 400*7c478bd9Sstevel@tonic-gate /* 401*7c478bd9Sstevel@tonic-gate * Allocate a block for statistics keeping. 402*7c478bd9Sstevel@tonic-gate * Returns NULL if blocks are unavailable. 403*7c478bd9Sstevel@tonic-gate */ 404*7c478bd9Sstevel@tonic-gate static struct hrmstat * 405*7c478bd9Sstevel@tonic-gate hrm_balloc(void) 406*7c478bd9Sstevel@tonic-gate { 407*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm; 408*7c478bd9Sstevel@tonic-gate 409*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock)); 410*7c478bd9Sstevel@tonic-gate 411*7c478bd9Sstevel@tonic-gate hrm = hrm_blist; 412*7c478bd9Sstevel@tonic-gate if (hrm != NULL) { 413*7c478bd9Sstevel@tonic-gate hrm_blist = hrm->hrm_hnext; 414*7c478bd9Sstevel@tonic-gate hrm_blist_num--; 415*7c478bd9Sstevel@tonic-gate hrm->hrm_hnext = NULL; 416*7c478bd9Sstevel@tonic-gate } 417*7c478bd9Sstevel@tonic-gate return (hrm); 418*7c478bd9Sstevel@tonic-gate } 419*7c478bd9Sstevel@tonic-gate 420*7c478bd9Sstevel@tonic-gate /* 421*7c478bd9Sstevel@tonic-gate * Set the ref and mod bits for addr within statistics block hrm. 422*7c478bd9Sstevel@tonic-gate */ 423*7c478bd9Sstevel@tonic-gate static void 424*7c478bd9Sstevel@tonic-gate hrm_setbits(struct hrmstat *hrm, caddr_t addr, uint_t bits) 425*7c478bd9Sstevel@tonic-gate { 426*7c478bd9Sstevel@tonic-gate uint_t po, bo, spb; 427*7c478bd9Sstevel@tonic-gate uint_t nbits; 428*7c478bd9Sstevel@tonic-gate 429*7c478bd9Sstevel@tonic-gate po = ((uintptr_t)addr & HRM_BASEOFFSET) >> MMU_PAGESHIFT; /* pg off */ 430*7c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2); /* which byte in bit array */ 431*7c478bd9Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; /* shift position within byte */ 432*7c478bd9Sstevel@tonic-gate nbits = bits << spb; /* bit mask */ 433*7c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] |= nbits; 434*7c478bd9Sstevel@tonic-gate } 435*7c478bd9Sstevel@tonic-gate 436*7c478bd9Sstevel@tonic-gate /* 437*7c478bd9Sstevel@tonic-gate * Return collected statistics about an address space. 438*7c478bd9Sstevel@tonic-gate * If clearflag is set, atomically read and zero the bits. 439*7c478bd9Sstevel@tonic-gate * 440*7c478bd9Sstevel@tonic-gate * Fill in the data array supplied with the referenced and 441*7c478bd9Sstevel@tonic-gate * modified bits collected for address range [addr ... addr + len] 442*7c478bd9Sstevel@tonic-gate * in address space, as, uniquely identified by id. 443*7c478bd9Sstevel@tonic-gate * The destination is a byte array. We fill in three bits per byte: 444*7c478bd9Sstevel@tonic-gate * referenced, modified, and hwmapped bits. 445*7c478bd9Sstevel@tonic-gate * Kernel only interface, can't fault on destination data array. 446*7c478bd9Sstevel@tonic-gate * 447*7c478bd9Sstevel@tonic-gate */ 448*7c478bd9Sstevel@tonic-gate void 449*7c478bd9Sstevel@tonic-gate hat_getstat(struct as *as, caddr_t addr, size_t len, uint_t id, 450*7c478bd9Sstevel@tonic-gate caddr_t datap, int clearflag) 451*7c478bd9Sstevel@tonic-gate { 452*7c478bd9Sstevel@tonic-gate size_t np; /* number of pages */ 453*7c478bd9Sstevel@tonic-gate caddr_t a; 454*7c478bd9Sstevel@tonic-gate char *dp; 455*7c478bd9Sstevel@tonic-gate 456*7c478bd9Sstevel@tonic-gate np = btop(len); 457*7c478bd9Sstevel@tonic-gate bzero(datap, np); 458*7c478bd9Sstevel@tonic-gate 459*7c478bd9Sstevel@tonic-gate hat_sync(as->a_hat, addr, len, clearflag); 460*7c478bd9Sstevel@tonic-gate 461*7c478bd9Sstevel@tonic-gate /* allocate more statistics blocks if needed */ 462*7c478bd9Sstevel@tonic-gate hrm_getblk(0); 463*7c478bd9Sstevel@tonic-gate 464*7c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock); 465*7c478bd9Sstevel@tonic-gate if (hrm_hashtab == NULL) { 466*7c478bd9Sstevel@tonic-gate /* can happen when victim process exits */ 467*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 468*7c478bd9Sstevel@tonic-gate return; 469*7c478bd9Sstevel@tonic-gate } 470*7c478bd9Sstevel@tonic-gate dp = datap; 471*7c478bd9Sstevel@tonic-gate a = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 472*7c478bd9Sstevel@tonic-gate while (a < addr + len) { 473*7c478bd9Sstevel@tonic-gate struct hrmstat *hrm; 474*7c478bd9Sstevel@tonic-gate size_t n; /* number of pages, temp */ 475*7c478bd9Sstevel@tonic-gate int h; /* hash index */ 476*7c478bd9Sstevel@tonic-gate uint_t po; 477*7c478bd9Sstevel@tonic-gate 478*7c478bd9Sstevel@tonic-gate h = hrm_hash(as, a); 479*7c478bd9Sstevel@tonic-gate n = (HRM_PAGES - 480*7c478bd9Sstevel@tonic-gate (((uintptr_t)a & HRM_PAGEMASK) >> MMU_PAGESHIFT)); 481*7c478bd9Sstevel@tonic-gate if (n > np) 482*7c478bd9Sstevel@tonic-gate n = np; 483*7c478bd9Sstevel@tonic-gate po = ((uintptr_t)a & HRM_BASEOFFSET) >> MMU_PAGESHIFT; 484*7c478bd9Sstevel@tonic-gate 485*7c478bd9Sstevel@tonic-gate for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) { 486*7c478bd9Sstevel@tonic-gate if (hrm->hrm_as == as && 487*7c478bd9Sstevel@tonic-gate hrm->hrm_base == ((uintptr_t)a & HRM_BASEMASK) && 488*7c478bd9Sstevel@tonic-gate id == hrm->hrm_id) { 489*7c478bd9Sstevel@tonic-gate int i, nr; 490*7c478bd9Sstevel@tonic-gate uint_t bo, spb; 491*7c478bd9Sstevel@tonic-gate 492*7c478bd9Sstevel@tonic-gate /* 493*7c478bd9Sstevel@tonic-gate * Extract leading unaligned bits. 494*7c478bd9Sstevel@tonic-gate */ 495*7c478bd9Sstevel@tonic-gate i = 0; 496*7c478bd9Sstevel@tonic-gate while (i < n && (po & 3)) { 497*7c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2); 498*7c478bd9Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; 499*7c478bd9Sstevel@tonic-gate *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3; 500*7c478bd9Sstevel@tonic-gate if (clearflag) 501*7c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] &= ~(3<<spb); 502*7c478bd9Sstevel@tonic-gate po++; 503*7c478bd9Sstevel@tonic-gate i++; 504*7c478bd9Sstevel@tonic-gate } 505*7c478bd9Sstevel@tonic-gate /* 506*7c478bd9Sstevel@tonic-gate * Extract aligned bits. 507*7c478bd9Sstevel@tonic-gate */ 508*7c478bd9Sstevel@tonic-gate nr = n/4*4; 509*7c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2); 510*7c478bd9Sstevel@tonic-gate while (i < nr) { 511*7c478bd9Sstevel@tonic-gate int bits = hrm->hrm_bits[bo]; 512*7c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 6) & 3; 513*7c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 4) & 3; 514*7c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 2) & 3; 515*7c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 0) & 3; 516*7c478bd9Sstevel@tonic-gate if (clearflag) 517*7c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] = 0; 518*7c478bd9Sstevel@tonic-gate bo++; 519*7c478bd9Sstevel@tonic-gate po += 4; 520*7c478bd9Sstevel@tonic-gate i += 4; 521*7c478bd9Sstevel@tonic-gate } 522*7c478bd9Sstevel@tonic-gate /* 523*7c478bd9Sstevel@tonic-gate * Extract trailing unaligned bits. 524*7c478bd9Sstevel@tonic-gate */ 525*7c478bd9Sstevel@tonic-gate while (i < n) { 526*7c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2); 527*7c478bd9Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; 528*7c478bd9Sstevel@tonic-gate *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3; 529*7c478bd9Sstevel@tonic-gate if (clearflag) 530*7c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] &= ~(3<<spb); 531*7c478bd9Sstevel@tonic-gate po++; 532*7c478bd9Sstevel@tonic-gate i++; 533*7c478bd9Sstevel@tonic-gate } 534*7c478bd9Sstevel@tonic-gate 535*7c478bd9Sstevel@tonic-gate break; 536*7c478bd9Sstevel@tonic-gate } 537*7c478bd9Sstevel@tonic-gate } 538*7c478bd9Sstevel@tonic-gate if (hrm == NULL) 539*7c478bd9Sstevel@tonic-gate dp += n; 540*7c478bd9Sstevel@tonic-gate np -= n; 541*7c478bd9Sstevel@tonic-gate a += n * MMU_PAGESIZE; 542*7c478bd9Sstevel@tonic-gate } 543*7c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock); 544*7c478bd9Sstevel@tonic-gate } 545