1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org> 5 * Copyright (c) 2010 Juniper Networks, Inc. 6 * Copyright (c) 2009 Robert N. M. Watson 7 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org> 8 * Copyright (c) 2008 Yahoo!, Inc. 9 * All rights reserved. 10 * 11 * Written by: John Baldwin <jhb@FreeBSD.org> 12 * 13 * This software was developed by Robert N. M. Watson under contract 14 * to Juniper Networks, Inc. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. Neither the name of the author nor the names of any co-contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/pcpu.h> 43 #include <sys/sysctl.h> 44 #include <kvm.h> 45 #include <limits.h> 46 #include <stdlib.h> 47 48 #include "kvm_private.h" 49 50 #ifdef __amd64__ 51 #define __OFFSET_BY_PCPU 52 #endif 53 54 static struct nlist kvm_pcpu_nl[] = { 55 { .n_name = "_cpuid_to_pcpu" }, 56 { .n_name = "_mp_maxcpus" }, 57 { .n_name = "_mp_ncpus" }, 58 #ifdef __OFFSET_BY_PCPU 59 { .n_name = "___pcpu" }, 60 #endif 61 { .n_name = NULL }, 62 }; 63 #define NL_CPUID_TO_PCPU 0 64 #define NL_MP_MAXCPUS 1 65 #define NL_MP_NCPUS 2 66 #define NL___PCPU 3 67 68 /* 69 * Kernel per-CPU data state. We cache this stuff on the first 70 * access. 71 * 72 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the 73 * consumer has multiple handles in flight to differently configured 74 * kernels/crashdumps. 75 */ 76 static void **pcpu_data; 77 static int maxcpu; 78 static int mp_ncpus; 79 #ifdef __OFFSET_BY_PCPU 80 static unsigned long __pcpu; 81 #endif 82 83 static int 84 _kvm_pcpu_init(kvm_t *kd) 85 { 86 size_t len; 87 int max; 88 void *data; 89 90 if (kvm_nlist(kd, kvm_pcpu_nl) < 0) 91 return (-1); 92 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) { 93 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu"); 94 return (-1); 95 } 96 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) { 97 _kvm_err(kd, kd->program, "unable to find mp_maxcpus"); 98 return (-1); 99 } 100 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max, 101 sizeof(max)) != sizeof(max)) { 102 _kvm_err(kd, kd->program, "cannot read mp_maxcpus"); 103 return (-1); 104 } 105 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) { 106 _kvm_err(kd, kd->program, "unable to find mp_ncpus"); 107 return (-1); 108 } 109 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus, 110 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) { 111 _kvm_err(kd, kd->program, "cannot read mp_ncpus"); 112 return (-1); 113 } 114 #ifdef __OFFSET_BY_PCPU 115 if (kvm_pcpu_nl[NL___PCPU].n_value == 0) { 116 _kvm_err(kd, kd->program, "unable to find __pcpu"); 117 return (-1); 118 } 119 if (kvm_read(kd, kvm_pcpu_nl[NL___PCPU].n_value, &__pcpu, 120 sizeof(__pcpu)) != sizeof(__pcpu)) { 121 _kvm_err(kd, kd->program, "cannot read __pcpu"); 122 return (-1); 123 } 124 #endif 125 len = max * sizeof(void *); 126 data = malloc(len); 127 if (data == NULL) { 128 _kvm_err(kd, kd->program, "out of memory"); 129 return (-1); 130 } 131 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) != 132 (ssize_t)len) { 133 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array"); 134 free(data); 135 return (-1); 136 } 137 pcpu_data = data; 138 maxcpu = max; 139 return (0); 140 } 141 142 static void 143 _kvm_pcpu_clear(void) 144 { 145 146 maxcpu = 0; 147 free(pcpu_data); 148 pcpu_data = NULL; 149 } 150 151 void * 152 kvm_getpcpu(kvm_t *kd, int cpu) 153 { 154 char *buf; 155 156 if (kd == NULL) { 157 _kvm_pcpu_clear(); 158 return (NULL); 159 } 160 161 if (maxcpu == 0) 162 if (_kvm_pcpu_init(kd) < 0) 163 return ((void *)-1); 164 165 if (cpu >= maxcpu || pcpu_data[cpu] == NULL) 166 return (NULL); 167 168 buf = malloc(sizeof(struct pcpu)); 169 if (buf == NULL) { 170 _kvm_err(kd, kd->program, "out of memory"); 171 return ((void *)-1); 172 } 173 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf, 174 sizeof(struct pcpu)) != sizeof(struct pcpu)) { 175 _kvm_err(kd, kd->program, "unable to read per-CPU data"); 176 free(buf); 177 return ((void *)-1); 178 } 179 return (buf); 180 } 181 182 int 183 kvm_getmaxcpu(kvm_t *kd) 184 { 185 186 if (kd == NULL) { 187 _kvm_pcpu_clear(); 188 return (0); 189 } 190 191 if (maxcpu == 0) 192 if (_kvm_pcpu_init(kd) < 0) 193 return (-1); 194 return (maxcpu); 195 } 196 197 int 198 kvm_getncpus(kvm_t *kd) 199 { 200 201 if (mp_ncpus == 0) 202 if (_kvm_pcpu_init(kd) < 0) 203 return (-1); 204 return (mp_ncpus); 205 } 206 207 static int 208 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error) 209 { 210 211 if (!kd->dpcpu_initialized) { 212 if (report_error) 213 _kvm_err(kd, kd->program, "%s: not initialized", 214 __func__); 215 return (-1); 216 } 217 if (cpu >= kd->dpcpu_maxcpus) { 218 if (report_error) 219 _kvm_err(kd, kd->program, "%s: CPU %u too big", 220 __func__, cpu); 221 return (-1); 222 } 223 if (kd->dpcpu_off[cpu] == 0) { 224 if (report_error) 225 _kvm_err(kd, kd->program, "%s: CPU %u not found", 226 __func__, cpu); 227 return (-1); 228 } 229 kd->dpcpu_curcpu = cpu; 230 kd->dpcpu_curoff = kd->dpcpu_off[cpu]; 231 return (0); 232 } 233 234 /* 235 * Set up libkvm to handle dynamic per-CPU memory. 236 */ 237 static int 238 _kvm_dpcpu_init(kvm_t *kd) 239 { 240 struct kvm_nlist nl[] = { 241 #define NLIST_START_SET_PCPU 0 242 { .n_name = "___start_" DPCPU_SETNAME }, 243 #define NLIST_STOP_SET_PCPU 1 244 { .n_name = "___stop_" DPCPU_SETNAME }, 245 #define NLIST_DPCPU_OFF 2 246 { .n_name = "_dpcpu_off" }, 247 #define NLIST_MP_MAXCPUS 3 248 { .n_name = "_mp_maxcpus" }, 249 { .n_name = NULL }, 250 }; 251 uintptr_t *dpcpu_off_buf; 252 size_t len; 253 u_int dpcpu_maxcpus; 254 255 /* 256 * XXX: This only works for native kernels for now. 257 */ 258 if (!kvm_native(kd)) 259 return (-1); 260 261 /* 262 * Locate and cache locations of important symbols using the internal 263 * version of _kvm_nlist, turning off initialization to avoid 264 * recursion in case of unresolveable symbols. 265 */ 266 if (_kvm_nlist(kd, nl, 0) != 0) 267 return (-1); 268 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus, 269 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus)) 270 return (-1); 271 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf); 272 dpcpu_off_buf = malloc(len); 273 if (dpcpu_off_buf == NULL) 274 return (-1); 275 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) != 276 (ssize_t)len) { 277 free(dpcpu_off_buf); 278 return (-1); 279 } 280 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value; 281 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value; 282 kd->dpcpu_maxcpus = dpcpu_maxcpus; 283 kd->dpcpu_off = dpcpu_off_buf; 284 kd->dpcpu_initialized = 1; 285 (void)_kvm_dpcpu_setcpu(kd, 0, 0); 286 return (0); 287 } 288 289 /* 290 * Check whether the dpcpu module has been initialized successfully or not, 291 * initialize it if permitted. 292 */ 293 int 294 _kvm_dpcpu_initialized(kvm_t *kd, int intialize) 295 { 296 297 if (kd->dpcpu_initialized || !intialize) 298 return (kd->dpcpu_initialized); 299 300 (void)_kvm_dpcpu_init(kd); 301 302 return (kd->dpcpu_initialized); 303 } 304 305 /* 306 * Check whether the value is within the dpcpu symbol range and only if so 307 * adjust the offset relative to the current offset. 308 */ 309 kvaddr_t 310 _kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value) 311 { 312 313 if (value == 0) 314 return (value); 315 316 if (!kd->dpcpu_initialized) 317 return (value); 318 319 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop) 320 return (value); 321 322 return (kd->dpcpu_curoff + value); 323 } 324 325 int 326 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu) 327 { 328 int ret; 329 330 if (!kd->dpcpu_initialized) { 331 ret = _kvm_dpcpu_init(kd); 332 if (ret != 0) { 333 _kvm_err(kd, kd->program, "%s: init failed", 334 __func__); 335 return (ret); 336 } 337 } 338 339 return (_kvm_dpcpu_setcpu(kd, cpu, 1)); 340 } 341 342 /* 343 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation. 344 */ 345 ssize_t 346 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu) 347 { 348 349 if (!kvm_native(kd)) 350 return (-1); 351 if (mp_ncpus == 0) 352 if (_kvm_pcpu_init(kd) < 0) 353 return (0); 354 355 #ifdef __OFFSET_BY_PCPU 356 base += __pcpu; 357 #endif 358 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu), 359 buf, size)); 360 } 361 362 /* 363 * Fetch value of a counter(9). 364 */ 365 uint64_t 366 kvm_counter_u64_fetch(kvm_t *kd, u_long base) 367 { 368 uint64_t r, c; 369 370 if (mp_ncpus == 0) 371 if (_kvm_pcpu_init(kd) < 0) 372 return (0); 373 374 r = 0; 375 for (int i = 0; i < mp_ncpus; i++) { 376 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c)) 377 return (0); 378 r += c; 379 } 380 381 return (r); 382 } 383