xref: /freebsd/lib/libmemstat/memstat_malloc.c (revision 36639c3942f587e652d2aba6a71ad45b64c2ce47) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2005 Robert N. M. Watson
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/malloc.h>
34 #include <sys/sysctl.h>
35 
36 #include <err.h>
37 #include <errno.h>
38 #include <kvm.h>
39 #include <nlist.h>
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43 
44 #include "memstat.h"
45 #include "memstat_internal.h"
46 
47 static int memstat_malloc_zone_count;
48 static int memstat_malloc_zone_sizes[32];
49 
50 static int	memstat_malloc_zone_init(void);
51 static int	memstat_malloc_zone_init_kvm(kvm_t *kvm);
52 
53 static struct nlist namelist[] = {
54 #define	X_KMEMSTATISTICS	0
55 	{ .n_name = "_kmemstatistics" },
56 #define	X_KMEMZONES		1
57 	{ .n_name = "_kmemzones" },
58 #define	X_NUMZONES		2
59 	{ .n_name = "_numzones" },
60 #define	X_VM_MALLOC_ZONE_COUNT	3
61 	{ .n_name = "_vm_malloc_zone_count" },
62 #define	X_MP_MAXCPUS		4
63 	{ .n_name = "_mp_maxcpus" },
64 	{ .n_name = "" },
65 };
66 
67 /*
68  * Extract malloc(9) statistics from the running kernel, and store all memory
69  * type information in the passed list.  For each type, check the list for an
70  * existing entry with the right name/allocator -- if present, update that
71  * entry.  Otherwise, add a new entry.  On error, the entire list will be
72  * cleared, as entries will be in an inconsistent state.
73  *
74  * To reduce the level of work for a list that starts empty, we keep around a
75  * hint as to whether it was empty when we began, so we can avoid searching
76  * the list for entries to update.  Updates are O(n^2) due to searching for
77  * each entry before adding it.
78  */
79 int
80 memstat_sysctl_malloc(struct memory_type_list *list, int flags)
81 {
82 	struct malloc_type_stream_header *mtshp;
83 	struct malloc_type_header *mthp;
84 	struct malloc_type_stats *mtsp;
85 	struct memory_type *mtp;
86 	int count, hint_dontsearch, i, j, maxcpus;
87 	char *buffer, *p;
88 	size_t size;
89 
90 	hint_dontsearch = LIST_EMPTY(&list->mtl_list);
91 
92 	/*
93 	 * Query the number of CPUs, number of malloc types so that we can
94 	 * guess an initial buffer size.  We loop until we succeed or really
95 	 * fail.  Note that the value of maxcpus we query using sysctl is not
96 	 * the version we use when processing the real data -- that is read
97 	 * from the header.
98 	 */
99 retry:
100 	size = sizeof(maxcpus);
101 	if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) {
102 		if (errno == EACCES || errno == EPERM)
103 			list->mtl_error = MEMSTAT_ERROR_PERMISSION;
104 		else
105 			list->mtl_error = MEMSTAT_ERROR_DATAERROR;
106 		return (-1);
107 	}
108 	if (size != sizeof(maxcpus)) {
109 		list->mtl_error = MEMSTAT_ERROR_DATAERROR;
110 		return (-1);
111 	}
112 
113 	size = sizeof(count);
114 	if (sysctlbyname("kern.malloc_count", &count, &size, NULL, 0) < 0) {
115 		if (errno == EACCES || errno == EPERM)
116 			list->mtl_error = MEMSTAT_ERROR_PERMISSION;
117 		else
118 			list->mtl_error = MEMSTAT_ERROR_VERSION;
119 		return (-1);
120 	}
121 	if (size != sizeof(count)) {
122 		list->mtl_error = MEMSTAT_ERROR_DATAERROR;
123 		return (-1);
124 	}
125 
126 	if (memstat_malloc_zone_init() == -1) {
127 		list->mtl_error = MEMSTAT_ERROR_VERSION;
128 		return (-1);
129 	}
130 
131 	size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) *
132 	    maxcpus);
133 
134 	buffer = malloc(size);
135 	if (buffer == NULL) {
136 		list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
137 		return (-1);
138 	}
139 
140 	if (sysctlbyname("kern.malloc_stats", buffer, &size, NULL, 0) < 0) {
141 		/*
142 		 * XXXRW: ENOMEM is an ambiguous return, we should bound the
143 		 * number of loops, perhaps.
144 		 */
145 		if (errno == ENOMEM) {
146 			free(buffer);
147 			goto retry;
148 		}
149 		if (errno == EACCES || errno == EPERM)
150 			list->mtl_error = MEMSTAT_ERROR_PERMISSION;
151 		else
152 			list->mtl_error = MEMSTAT_ERROR_VERSION;
153 		free(buffer);
154 		return (-1);
155 	}
156 
157 	if (size == 0) {
158 		free(buffer);
159 		return (0);
160 	}
161 
162 	if (size < sizeof(*mtshp)) {
163 		list->mtl_error = MEMSTAT_ERROR_VERSION;
164 		free(buffer);
165 		return (-1);
166 	}
167 	p = buffer;
168 	mtshp = (struct malloc_type_stream_header *)p;
169 	p += sizeof(*mtshp);
170 
171 	if (mtshp->mtsh_version != MALLOC_TYPE_STREAM_VERSION) {
172 		list->mtl_error = MEMSTAT_ERROR_VERSION;
173 		free(buffer);
174 		return (-1);
175 	}
176 
177 	/*
178 	 * For the remainder of this function, we are quite trusting about
179 	 * the layout of structures and sizes, since we've determined we have
180 	 * a matching version and acceptable CPU count.
181 	 */
182 	maxcpus = mtshp->mtsh_maxcpus;
183 	count = mtshp->mtsh_count;
184 	for (i = 0; i < count; i++) {
185 		mthp = (struct malloc_type_header *)p;
186 		p += sizeof(*mthp);
187 
188 		if (hint_dontsearch == 0) {
189 			mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC,
190 			    mthp->mth_name);
191 		} else
192 			mtp = NULL;
193 		if (mtp == NULL)
194 			mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
195 			    mthp->mth_name, maxcpus);
196 		if (mtp == NULL) {
197 			_memstat_mtl_empty(list);
198 			free(buffer);
199 			list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
200 			return (-1);
201 		}
202 
203 		/*
204 		 * Reset the statistics on a current node.
205 		 */
206 		_memstat_mt_reset_stats(mtp, maxcpus);
207 
208 		for (j = 0; j < maxcpus; j++) {
209 			mtsp = (struct malloc_type_stats *)p;
210 			p += sizeof(*mtsp);
211 
212 			/*
213 			 * Sumarize raw statistics across CPUs into coalesced
214 			 * statistics.
215 			 */
216 			mtp->mt_memalloced += mtsp->mts_memalloced;
217 			mtp->mt_memfreed += mtsp->mts_memfreed;
218 			mtp->mt_numallocs += mtsp->mts_numallocs;
219 			mtp->mt_numfrees += mtsp->mts_numfrees;
220 			mtp->mt_sizemask |= mtsp->mts_size;
221 
222 			/*
223 			 * Copies of per-CPU statistics.
224 			 */
225 			mtp->mt_percpu_alloc[j].mtp_memalloced =
226 			    mtsp->mts_memalloced;
227 			mtp->mt_percpu_alloc[j].mtp_memfreed =
228 			    mtsp->mts_memfreed;
229 			mtp->mt_percpu_alloc[j].mtp_numallocs =
230 			    mtsp->mts_numallocs;
231 			mtp->mt_percpu_alloc[j].mtp_numfrees =
232 			    mtsp->mts_numfrees;
233 			mtp->mt_percpu_alloc[j].mtp_sizemask =
234 			    mtsp->mts_size;
235 		}
236 
237 		/*
238 		 * Derived cross-CPU statistics.
239 		 */
240 		mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
241 		mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
242 	}
243 
244 	free(buffer);
245 
246 	return (0);
247 }
248 
249 static int
250 kread(kvm_t *kvm, void *kvm_pointer, void *address, size_t size,
251     size_t offset)
252 {
253 	ssize_t ret;
254 
255 	ret = kvm_read(kvm, (unsigned long)kvm_pointer + offset, address,
256 	    size);
257 	if (ret < 0)
258 		return (MEMSTAT_ERROR_KVM);
259 	if ((size_t)ret != size)
260 		return (MEMSTAT_ERROR_KVM_SHORTREAD);
261 	return (0);
262 }
263 
264 static int
265 kread_string(kvm_t *kvm, const void *kvm_pointer, char *buffer, int buflen)
266 {
267 	ssize_t ret;
268 	int i;
269 
270 	for (i = 0; i < buflen; i++) {
271 		ret = kvm_read(kvm, __DECONST(unsigned long, kvm_pointer) +
272 		    i, &(buffer[i]), sizeof(char));
273 		if (ret < 0)
274 			return (MEMSTAT_ERROR_KVM);
275 		if ((size_t)ret != sizeof(char))
276 			return (MEMSTAT_ERROR_KVM_SHORTREAD);
277 		if (buffer[i] == '\0')
278 			return (0);
279 	}
280 	/* Truncate. */
281 	buffer[i-1] = '\0';
282 	return (0);
283 }
284 
285 static int
286 kread_symbol(kvm_t *kvm, int index, void *address, size_t size,
287     size_t offset)
288 {
289 	ssize_t ret;
290 
291 	ret = kvm_read(kvm, namelist[index].n_value + offset, address, size);
292 	if (ret < 0)
293 		return (MEMSTAT_ERROR_KVM);
294 	if ((size_t)ret != size)
295 		return (MEMSTAT_ERROR_KVM_SHORTREAD);
296 	return (0);
297 }
298 
299 static int
300 kread_zpcpu(kvm_t *kvm, u_long base, void *buf, size_t size, int cpu)
301 {
302 	ssize_t ret;
303 
304 	ret = kvm_read_zpcpu(kvm, base, buf, size, cpu);
305 	if (ret < 0)
306 		return (MEMSTAT_ERROR_KVM);
307 	if ((size_t)ret != size)
308 		return (MEMSTAT_ERROR_KVM_SHORTREAD);
309 	return (0);
310 }
311 
312 int
313 memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle)
314 {
315 	struct memory_type *mtp;
316 	void *kmemstatistics;
317 	int hint_dontsearch, j, mp_maxcpus, mp_ncpus, ret;
318 	char name[MEMTYPE_MAXNAME];
319 	struct malloc_type_stats mts;
320 	struct malloc_type_internal mti, *mtip;
321 	struct malloc_type type, *typep;
322 	kvm_t *kvm;
323 
324 	kvm = (kvm_t *)kvm_handle;
325 
326 	hint_dontsearch = LIST_EMPTY(&list->mtl_list);
327 
328 	if (kvm_nlist(kvm, namelist) != 0) {
329 		list->mtl_error = MEMSTAT_ERROR_KVM;
330 		return (-1);
331 	}
332 
333 	if (namelist[X_KMEMSTATISTICS].n_type == 0 ||
334 	    namelist[X_KMEMSTATISTICS].n_value == 0) {
335 		list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL;
336 		return (-1);
337 	}
338 
339 	ret = kread_symbol(kvm, X_MP_MAXCPUS, &mp_maxcpus,
340 	    sizeof(mp_maxcpus), 0);
341 	if (ret != 0) {
342 		list->mtl_error = ret;
343 		return (-1);
344 	}
345 
346 	ret = kread_symbol(kvm, X_KMEMSTATISTICS, &kmemstatistics,
347 	    sizeof(kmemstatistics), 0);
348 	if (ret != 0) {
349 		list->mtl_error = ret;
350 		return (-1);
351 	}
352 
353 	ret = memstat_malloc_zone_init_kvm(kvm);
354 	if (ret != 0) {
355 		list->mtl_error = ret;
356 		return (-1);
357 	}
358 
359 	mp_ncpus = kvm_getncpus(kvm);
360 
361 	for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) {
362 		ret = kread(kvm, typep, &type, sizeof(type), 0);
363 		if (ret != 0) {
364 			_memstat_mtl_empty(list);
365 			list->mtl_error = ret;
366 			return (-1);
367 		}
368 		ret = kread_string(kvm, (void *)type.ks_shortdesc, name,
369 		    MEMTYPE_MAXNAME);
370 		if (ret != 0) {
371 			_memstat_mtl_empty(list);
372 			list->mtl_error = ret;
373 			return (-1);
374 		}
375 
376 		/*
377 		 * Since our compile-time value for MAXCPU may differ from the
378 		 * kernel's, we populate our own array.
379 		 */
380 		mtip = type.ks_handle;
381 		ret = kread(kvm, mtip, &mti, sizeof(mti), 0);
382 		if (ret != 0) {
383 			_memstat_mtl_empty(list);
384 			list->mtl_error = ret;
385 			return (-1);
386 		}
387 
388 		if (hint_dontsearch == 0) {
389 			mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, name);
390 		} else
391 			mtp = NULL;
392 		if (mtp == NULL)
393 			mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
394 			    name, mp_maxcpus);
395 		if (mtp == NULL) {
396 			_memstat_mtl_empty(list);
397 			list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
398 			return (-1);
399 		}
400 
401 		/*
402 		 * This logic is replicated from kern_malloc.c, and should
403 		 * be kept in sync.
404 		 */
405 		_memstat_mt_reset_stats(mtp, mp_maxcpus);
406 		for (j = 0; j < mp_ncpus; j++) {
407 			ret = kread_zpcpu(kvm, (u_long)mti.mti_stats, &mts,
408 			    sizeof(mts), j);
409 			if (ret != 0) {
410 				_memstat_mtl_empty(list);
411 				list->mtl_error = ret;
412 				return (-1);
413 			}
414 			mtp->mt_memalloced += mts.mts_memalloced;
415 			mtp->mt_memfreed += mts.mts_memfreed;
416 			mtp->mt_numallocs += mts.mts_numallocs;
417 			mtp->mt_numfrees += mts.mts_numfrees;
418 			mtp->mt_sizemask |= mts.mts_size;
419 
420 			mtp->mt_percpu_alloc[j].mtp_memalloced =
421 			    mts.mts_memalloced;
422 			mtp->mt_percpu_alloc[j].mtp_memfreed =
423 			    mts.mts_memfreed;
424 			mtp->mt_percpu_alloc[j].mtp_numallocs =
425 			    mts.mts_numallocs;
426 			mtp->mt_percpu_alloc[j].mtp_numfrees =
427 			    mts.mts_numfrees;
428 			mtp->mt_percpu_alloc[j].mtp_sizemask =
429 			    mts.mts_size;
430 		}
431 		for (; j < mp_maxcpus; j++) {
432 			bzero(&mtp->mt_percpu_alloc[j],
433 			    sizeof(mtp->mt_percpu_alloc[0]));
434 		}
435 
436 		mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
437 		mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
438 	}
439 
440 	return (0);
441 }
442 
443 static int
444 memstat_malloc_zone_init(void)
445 {
446 	size_t size;
447 
448 	size = sizeof(memstat_malloc_zone_count);
449 	if (sysctlbyname("vm.malloc.zone_count", &memstat_malloc_zone_count,
450 	    &size, NULL, 0) < 0) {
451 		return (-1);
452 	}
453 
454 	if (memstat_malloc_zone_count > (int)nitems(memstat_malloc_zone_sizes)) {
455 		return (-1);
456 	}
457 
458 	size = sizeof(memstat_malloc_zone_sizes);
459 	if (sysctlbyname("vm.malloc.zone_sizes", &memstat_malloc_zone_sizes,
460 	    &size, NULL, 0) < 0) {
461 		return (-1);
462 	}
463 
464 	return (0);
465 }
466 
467 /*
468  * Copied from kern_malloc.c
469  *
470  * kz_zone is an array sized at compilation time, the size is exported in
471  * "numzones". Below we need to iterate kz_size.
472  */
473 struct memstat_kmemzone {
474 	int kz_size;
475 	const char *kz_name;
476 	void *kz_zone[1];
477 };
478 
479 static int
480 memstat_malloc_zone_init_kvm(kvm_t *kvm)
481 {
482 	struct memstat_kmemzone *kmemzones, *kz;
483 	int numzones, objsize, allocsize, ret;
484 	int i;
485 
486 	ret = kread_symbol(kvm, X_VM_MALLOC_ZONE_COUNT,
487 	    &memstat_malloc_zone_count, sizeof(memstat_malloc_zone_count), 0);
488 	if (ret != 0) {
489 		return (ret);
490 	}
491 
492 	ret = kread_symbol(kvm, X_NUMZONES, &numzones, sizeof(numzones), 0);
493 	if (ret != 0) {
494 		return (ret);
495 	}
496 
497 	objsize = __offsetof(struct memstat_kmemzone, kz_zone) +
498 	    sizeof(void *) * numzones;
499 
500 	allocsize = objsize * memstat_malloc_zone_count;
501 	kmemzones = malloc(allocsize);
502 	if (kmemzones == NULL) {
503 		return (MEMSTAT_ERROR_NOMEMORY);
504 	}
505 	ret = kread_symbol(kvm, X_KMEMZONES, kmemzones, allocsize, 0);
506 	if (ret != 0) {
507 		free(kmemzones);
508 		return (ret);
509 	}
510 
511 	kz = kmemzones;
512 	for (i = 0; i < (int)nitems(memstat_malloc_zone_sizes); i++) {
513 		memstat_malloc_zone_sizes[i] = kz->kz_size;
514 		kz = (struct memstat_kmemzone *)((char *)kz + objsize);
515 	}
516 
517 	free(kmemzones);
518 	return (0);
519 }
520 
521 size_t
522 memstat_malloc_zone_get_count(void)
523 {
524 
525 	return (memstat_malloc_zone_count);
526 }
527 
528 size_t
529 memstat_malloc_zone_get_size(size_t n)
530 {
531 
532 	if (n >= nitems(memstat_malloc_zone_sizes)) {
533 		return (-1);
534 	}
535 
536 	return (memstat_malloc_zone_sizes[n]);
537 }
538 
539 int
540 memstat_malloc_zone_used(const struct memory_type *mtp, size_t n)
541 {
542 
543 	if (memstat_get_sizemask(mtp) & (1 << n))
544 		return (1);
545 
546 	return (0);
547 }
548