1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2005 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include <sys/param.h> 31 #include <sys/malloc.h> 32 #include <sys/sysctl.h> 33 34 #include <err.h> 35 #include <errno.h> 36 #include <kvm.h> 37 #include <nlist.h> 38 #include <stdio.h> 39 #include <stdlib.h> 40 #include <string.h> 41 42 #include "memstat.h" 43 #include "memstat_internal.h" 44 45 static int memstat_malloc_zone_count; 46 static int memstat_malloc_zone_sizes[32]; 47 48 static int memstat_malloc_zone_init(void); 49 static int memstat_malloc_zone_init_kvm(kvm_t *kvm); 50 51 static struct nlist namelist[] = { 52 #define X_KMEMSTATISTICS 0 53 { .n_name = "_kmemstatistics" }, 54 #define X_KMEMZONES 1 55 { .n_name = "_kmemzones" }, 56 #define X_NUMZONES 2 57 { .n_name = "_numzones" }, 58 #define X_VM_MALLOC_ZONE_COUNT 3 59 { .n_name = "_vm_malloc_zone_count" }, 60 #define X_MP_MAXCPUS 4 61 { .n_name = "_mp_maxcpus" }, 62 { .n_name = "" }, 63 }; 64 65 /* 66 * Extract malloc(9) statistics from the running kernel, and store all memory 67 * type information in the passed list. For each type, check the list for an 68 * existing entry with the right name/allocator -- if present, update that 69 * entry. Otherwise, add a new entry. On error, the entire list will be 70 * cleared, as entries will be in an inconsistent state. 71 * 72 * To reduce the level of work for a list that starts empty, we keep around a 73 * hint as to whether it was empty when we began, so we can avoid searching 74 * the list for entries to update. Updates are O(n^2) due to searching for 75 * each entry before adding it. 76 */ 77 int 78 memstat_sysctl_malloc(struct memory_type_list *list, int flags) 79 { 80 struct malloc_type_stream_header *mtshp; 81 struct malloc_type_header *mthp; 82 struct malloc_type_stats *mtsp; 83 struct memory_type *mtp; 84 int count, hint_dontsearch, i, j, maxcpus; 85 char *buffer, *p; 86 size_t size; 87 88 hint_dontsearch = LIST_EMPTY(&list->mtl_list); 89 90 /* 91 * Query the number of CPUs, number of malloc types so that we can 92 * guess an initial buffer size. We loop until we succeed or really 93 * fail. Note that the value of maxcpus we query using sysctl is not 94 * the version we use when processing the real data -- that is read 95 * from the header. 96 */ 97 retry: 98 size = sizeof(maxcpus); 99 if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) { 100 if (errno == EACCES || errno == EPERM) 101 list->mtl_error = MEMSTAT_ERROR_PERMISSION; 102 else 103 list->mtl_error = MEMSTAT_ERROR_DATAERROR; 104 return (-1); 105 } 106 if (size != sizeof(maxcpus)) { 107 list->mtl_error = MEMSTAT_ERROR_DATAERROR; 108 return (-1); 109 } 110 111 size = sizeof(count); 112 if (sysctlbyname("kern.malloc_count", &count, &size, NULL, 0) < 0) { 113 if (errno == EACCES || errno == EPERM) 114 list->mtl_error = MEMSTAT_ERROR_PERMISSION; 115 else 116 list->mtl_error = MEMSTAT_ERROR_VERSION; 117 return (-1); 118 } 119 if (size != sizeof(count)) { 120 list->mtl_error = MEMSTAT_ERROR_DATAERROR; 121 return (-1); 122 } 123 124 if (memstat_malloc_zone_init() == -1) { 125 list->mtl_error = MEMSTAT_ERROR_VERSION; 126 return (-1); 127 } 128 129 size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) * 130 maxcpus); 131 132 buffer = malloc(size); 133 if (buffer == NULL) { 134 list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 135 return (-1); 136 } 137 138 if (sysctlbyname("kern.malloc_stats", buffer, &size, NULL, 0) < 0) { 139 /* 140 * XXXRW: ENOMEM is an ambiguous return, we should bound the 141 * number of loops, perhaps. 142 */ 143 if (errno == ENOMEM) { 144 free(buffer); 145 goto retry; 146 } 147 if (errno == EACCES || errno == EPERM) 148 list->mtl_error = MEMSTAT_ERROR_PERMISSION; 149 else 150 list->mtl_error = MEMSTAT_ERROR_VERSION; 151 free(buffer); 152 return (-1); 153 } 154 155 if (size == 0) { 156 free(buffer); 157 return (0); 158 } 159 160 if (size < sizeof(*mtshp)) { 161 list->mtl_error = MEMSTAT_ERROR_VERSION; 162 free(buffer); 163 return (-1); 164 } 165 p = buffer; 166 mtshp = (struct malloc_type_stream_header *)p; 167 p += sizeof(*mtshp); 168 169 if (mtshp->mtsh_version != MALLOC_TYPE_STREAM_VERSION) { 170 list->mtl_error = MEMSTAT_ERROR_VERSION; 171 free(buffer); 172 return (-1); 173 } 174 175 /* 176 * For the remainder of this function, we are quite trusting about 177 * the layout of structures and sizes, since we've determined we have 178 * a matching version and acceptable CPU count. 179 */ 180 maxcpus = mtshp->mtsh_maxcpus; 181 count = mtshp->mtsh_count; 182 for (i = 0; i < count; i++) { 183 mthp = (struct malloc_type_header *)p; 184 p += sizeof(*mthp); 185 186 if (hint_dontsearch == 0) { 187 mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, 188 mthp->mth_name); 189 } else 190 mtp = NULL; 191 if (mtp == NULL) 192 mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC, 193 mthp->mth_name, maxcpus); 194 if (mtp == NULL) { 195 _memstat_mtl_empty(list); 196 free(buffer); 197 list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 198 return (-1); 199 } 200 201 /* 202 * Reset the statistics on a current node. 203 */ 204 _memstat_mt_reset_stats(mtp, maxcpus); 205 206 for (j = 0; j < maxcpus; j++) { 207 mtsp = (struct malloc_type_stats *)p; 208 p += sizeof(*mtsp); 209 210 /* 211 * Sumarize raw statistics across CPUs into coalesced 212 * statistics. 213 */ 214 mtp->mt_memalloced += mtsp->mts_memalloced; 215 mtp->mt_memfreed += mtsp->mts_memfreed; 216 mtp->mt_numallocs += mtsp->mts_numallocs; 217 mtp->mt_numfrees += mtsp->mts_numfrees; 218 mtp->mt_sizemask |= mtsp->mts_size; 219 220 /* 221 * Copies of per-CPU statistics. 222 */ 223 mtp->mt_percpu_alloc[j].mtp_memalloced = 224 mtsp->mts_memalloced; 225 mtp->mt_percpu_alloc[j].mtp_memfreed = 226 mtsp->mts_memfreed; 227 mtp->mt_percpu_alloc[j].mtp_numallocs = 228 mtsp->mts_numallocs; 229 mtp->mt_percpu_alloc[j].mtp_numfrees = 230 mtsp->mts_numfrees; 231 mtp->mt_percpu_alloc[j].mtp_sizemask = 232 mtsp->mts_size; 233 } 234 235 /* 236 * Derived cross-CPU statistics. 237 */ 238 mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed; 239 mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees; 240 } 241 242 free(buffer); 243 244 return (0); 245 } 246 247 static int 248 kread(kvm_t *kvm, void *kvm_pointer, void *address, size_t size, 249 size_t offset) 250 { 251 ssize_t ret; 252 253 ret = kvm_read(kvm, (unsigned long)kvm_pointer + offset, address, 254 size); 255 if (ret < 0) 256 return (MEMSTAT_ERROR_KVM); 257 if ((size_t)ret != size) 258 return (MEMSTAT_ERROR_KVM_SHORTREAD); 259 return (0); 260 } 261 262 static int 263 kread_string(kvm_t *kvm, const void *kvm_pointer, char *buffer, int buflen) 264 { 265 ssize_t ret; 266 int i; 267 268 for (i = 0; i < buflen; i++) { 269 ret = kvm_read(kvm, __DECONST(unsigned long, kvm_pointer) + 270 i, &(buffer[i]), sizeof(char)); 271 if (ret < 0) 272 return (MEMSTAT_ERROR_KVM); 273 if ((size_t)ret != sizeof(char)) 274 return (MEMSTAT_ERROR_KVM_SHORTREAD); 275 if (buffer[i] == '\0') 276 return (0); 277 } 278 /* Truncate. */ 279 buffer[i-1] = '\0'; 280 return (0); 281 } 282 283 static int 284 kread_symbol(kvm_t *kvm, int index, void *address, size_t size, 285 size_t offset) 286 { 287 ssize_t ret; 288 289 ret = kvm_read(kvm, namelist[index].n_value + offset, address, size); 290 if (ret < 0) 291 return (MEMSTAT_ERROR_KVM); 292 if ((size_t)ret != size) 293 return (MEMSTAT_ERROR_KVM_SHORTREAD); 294 return (0); 295 } 296 297 static int 298 kread_zpcpu(kvm_t *kvm, u_long base, void *buf, size_t size, int cpu) 299 { 300 ssize_t ret; 301 302 ret = kvm_read_zpcpu(kvm, base, buf, size, cpu); 303 if (ret < 0) 304 return (MEMSTAT_ERROR_KVM); 305 if ((size_t)ret != size) 306 return (MEMSTAT_ERROR_KVM_SHORTREAD); 307 return (0); 308 } 309 310 int 311 memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle) 312 { 313 struct memory_type *mtp; 314 void *kmemstatistics; 315 int hint_dontsearch, j, mp_maxcpus, mp_ncpus, ret; 316 char name[MEMTYPE_MAXNAME]; 317 struct malloc_type_stats mts; 318 struct malloc_type_internal *mtip; 319 struct malloc_type type, *typep; 320 kvm_t *kvm; 321 322 kvm = (kvm_t *)kvm_handle; 323 324 hint_dontsearch = LIST_EMPTY(&list->mtl_list); 325 326 if (kvm_nlist(kvm, namelist) != 0) { 327 list->mtl_error = MEMSTAT_ERROR_KVM; 328 return (-1); 329 } 330 331 if (namelist[X_KMEMSTATISTICS].n_type == 0 || 332 namelist[X_KMEMSTATISTICS].n_value == 0) { 333 list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL; 334 return (-1); 335 } 336 337 ret = kread_symbol(kvm, X_MP_MAXCPUS, &mp_maxcpus, 338 sizeof(mp_maxcpus), 0); 339 if (ret != 0) { 340 list->mtl_error = ret; 341 return (-1); 342 } 343 344 ret = kread_symbol(kvm, X_KMEMSTATISTICS, &kmemstatistics, 345 sizeof(kmemstatistics), 0); 346 if (ret != 0) { 347 list->mtl_error = ret; 348 return (-1); 349 } 350 351 ret = memstat_malloc_zone_init_kvm(kvm); 352 if (ret != 0) { 353 list->mtl_error = ret; 354 return (-1); 355 } 356 357 mp_ncpus = kvm_getncpus(kvm); 358 359 for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) { 360 ret = kread(kvm, typep, &type, sizeof(type), 0); 361 if (ret != 0) { 362 _memstat_mtl_empty(list); 363 list->mtl_error = ret; 364 return (-1); 365 } 366 ret = kread_string(kvm, (void *)type.ks_shortdesc, name, 367 MEMTYPE_MAXNAME); 368 if (ret != 0) { 369 _memstat_mtl_empty(list); 370 list->mtl_error = ret; 371 return (-1); 372 } 373 if (type.ks_version != M_VERSION) { 374 warnx("type %s with unsupported version %lu; skipped", 375 name, type.ks_version); 376 continue; 377 } 378 379 /* 380 * Since our compile-time value for MAXCPU may differ from the 381 * kernel's, we populate our own array. 382 */ 383 mtip = &type.ks_mti; 384 385 if (hint_dontsearch == 0) { 386 mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, name); 387 } else 388 mtp = NULL; 389 if (mtp == NULL) 390 mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC, 391 name, mp_maxcpus); 392 if (mtp == NULL) { 393 _memstat_mtl_empty(list); 394 list->mtl_error = MEMSTAT_ERROR_NOMEMORY; 395 return (-1); 396 } 397 398 /* 399 * This logic is replicated from kern_malloc.c, and should 400 * be kept in sync. 401 */ 402 _memstat_mt_reset_stats(mtp, mp_maxcpus); 403 for (j = 0; j < mp_ncpus; j++) { 404 ret = kread_zpcpu(kvm, (u_long)mtip->mti_stats, &mts, 405 sizeof(mts), j); 406 if (ret != 0) { 407 _memstat_mtl_empty(list); 408 list->mtl_error = ret; 409 return (-1); 410 } 411 mtp->mt_memalloced += mts.mts_memalloced; 412 mtp->mt_memfreed += mts.mts_memfreed; 413 mtp->mt_numallocs += mts.mts_numallocs; 414 mtp->mt_numfrees += mts.mts_numfrees; 415 mtp->mt_sizemask |= mts.mts_size; 416 417 mtp->mt_percpu_alloc[j].mtp_memalloced = 418 mts.mts_memalloced; 419 mtp->mt_percpu_alloc[j].mtp_memfreed = 420 mts.mts_memfreed; 421 mtp->mt_percpu_alloc[j].mtp_numallocs = 422 mts.mts_numallocs; 423 mtp->mt_percpu_alloc[j].mtp_numfrees = 424 mts.mts_numfrees; 425 mtp->mt_percpu_alloc[j].mtp_sizemask = 426 mts.mts_size; 427 } 428 for (; j < mp_maxcpus; j++) { 429 bzero(&mtp->mt_percpu_alloc[j], 430 sizeof(mtp->mt_percpu_alloc[0])); 431 } 432 433 mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed; 434 mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees; 435 } 436 437 return (0); 438 } 439 440 static int 441 memstat_malloc_zone_init(void) 442 { 443 size_t size; 444 445 size = sizeof(memstat_malloc_zone_count); 446 if (sysctlbyname("vm.malloc.zone_count", &memstat_malloc_zone_count, 447 &size, NULL, 0) < 0) { 448 return (-1); 449 } 450 451 if (memstat_malloc_zone_count > (int)nitems(memstat_malloc_zone_sizes)) { 452 return (-1); 453 } 454 455 size = sizeof(memstat_malloc_zone_sizes); 456 if (sysctlbyname("vm.malloc.zone_sizes", &memstat_malloc_zone_sizes, 457 &size, NULL, 0) < 0) { 458 return (-1); 459 } 460 461 return (0); 462 } 463 464 /* 465 * Copied from kern_malloc.c 466 * 467 * kz_zone is an array sized at compilation time, the size is exported in 468 * "numzones". Below we need to iterate kz_size. 469 */ 470 struct memstat_kmemzone { 471 int kz_size; 472 const char *kz_name; 473 void *kz_zone[1]; 474 }; 475 476 static int 477 memstat_malloc_zone_init_kvm(kvm_t *kvm) 478 { 479 struct memstat_kmemzone *kmemzones, *kz; 480 int numzones, objsize, allocsize, ret; 481 int i; 482 483 ret = kread_symbol(kvm, X_VM_MALLOC_ZONE_COUNT, 484 &memstat_malloc_zone_count, sizeof(memstat_malloc_zone_count), 0); 485 if (ret != 0) { 486 return (ret); 487 } 488 489 ret = kread_symbol(kvm, X_NUMZONES, &numzones, sizeof(numzones), 0); 490 if (ret != 0) { 491 return (ret); 492 } 493 494 objsize = __offsetof(struct memstat_kmemzone, kz_zone) + 495 sizeof(void *) * numzones; 496 497 allocsize = objsize * memstat_malloc_zone_count; 498 kmemzones = malloc(allocsize); 499 if (kmemzones == NULL) { 500 return (MEMSTAT_ERROR_NOMEMORY); 501 } 502 ret = kread_symbol(kvm, X_KMEMZONES, kmemzones, allocsize, 0); 503 if (ret != 0) { 504 free(kmemzones); 505 return (ret); 506 } 507 508 kz = kmemzones; 509 for (i = 0; i < (int)nitems(memstat_malloc_zone_sizes); i++) { 510 memstat_malloc_zone_sizes[i] = kz->kz_size; 511 kz = (struct memstat_kmemzone *)((char *)kz + objsize); 512 } 513 514 free(kmemzones); 515 return (0); 516 } 517 518 size_t 519 memstat_malloc_zone_get_count(void) 520 { 521 522 return (memstat_malloc_zone_count); 523 } 524 525 size_t 526 memstat_malloc_zone_get_size(size_t n) 527 { 528 529 if (n >= nitems(memstat_malloc_zone_sizes)) { 530 return (-1); 531 } 532 533 return (memstat_malloc_zone_sizes[n]); 534 } 535 536 int 537 memstat_malloc_zone_used(const struct memory_type *mtp, size_t n) 538 { 539 540 if (memstat_get_sizemask(mtp) & (1 << n)) 541 return (1); 542 543 return (0); 544 } 545