1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org> 5 * Copyright (c) 2010 Juniper Networks, Inc. 6 * Copyright (c) 2009 Robert N. M. Watson 7 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org> 8 * Copyright (c) 2008 Yahoo!, Inc. 9 * All rights reserved. 10 * 11 * Written by: John Baldwin <jhb@FreeBSD.org> 12 * 13 * This software was developed by Robert N. M. Watson under contract 14 * to Juniper Networks, Inc. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. Neither the name of the author nor the names of any co-contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/pcpu.h> 46 #include <sys/sysctl.h> 47 #include <kvm.h> 48 #include <limits.h> 49 #include <stdlib.h> 50 51 #include "kvm_private.h" 52 53 static struct nlist kvm_pcpu_nl[] = { 54 { .n_name = "_cpuid_to_pcpu" }, 55 { .n_name = "_mp_maxcpus" }, 56 { .n_name = "_mp_ncpus" }, 57 { .n_name = NULL }, 58 }; 59 #define NL_CPUID_TO_PCPU 0 60 #define NL_MP_MAXCPUS 1 61 #define NL_MP_NCPUS 2 62 63 /* 64 * Kernel per-CPU data state. We cache this stuff on the first 65 * access. 66 * 67 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the 68 * consumer has multiple handles in flight to differently configured 69 * kernels/crashdumps. 70 */ 71 static void **pcpu_data; 72 static int maxcpu; 73 static int mp_ncpus; 74 75 static int 76 _kvm_pcpu_init(kvm_t *kd) 77 { 78 size_t len; 79 int max; 80 void *data; 81 82 if (kvm_nlist(kd, kvm_pcpu_nl) < 0) 83 return (-1); 84 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) { 85 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu"); 86 return (-1); 87 } 88 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) { 89 _kvm_err(kd, kd->program, "unable to find mp_maxcpus"); 90 return (-1); 91 } 92 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max, 93 sizeof(max)) != sizeof(max)) { 94 _kvm_err(kd, kd->program, "cannot read mp_maxcpus"); 95 return (-1); 96 } 97 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) { 98 _kvm_err(kd, kd->program, "unable to find mp_ncpus"); 99 return (-1); 100 } 101 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus, 102 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) { 103 _kvm_err(kd, kd->program, "cannot read mp_ncpus"); 104 return (-1); 105 } 106 len = max * sizeof(void *); 107 data = malloc(len); 108 if (data == NULL) { 109 _kvm_err(kd, kd->program, "out of memory"); 110 return (-1); 111 } 112 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) != 113 (ssize_t)len) { 114 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array"); 115 free(data); 116 return (-1); 117 } 118 pcpu_data = data; 119 maxcpu = max; 120 return (0); 121 } 122 123 static void 124 _kvm_pcpu_clear(void) 125 { 126 127 maxcpu = 0; 128 free(pcpu_data); 129 pcpu_data = NULL; 130 } 131 132 void * 133 kvm_getpcpu(kvm_t *kd, int cpu) 134 { 135 char *buf; 136 137 if (kd == NULL) { 138 _kvm_pcpu_clear(); 139 return (NULL); 140 } 141 142 if (maxcpu == 0) 143 if (_kvm_pcpu_init(kd) < 0) 144 return ((void *)-1); 145 146 if (cpu >= maxcpu || pcpu_data[cpu] == NULL) 147 return (NULL); 148 149 buf = malloc(sizeof(struct pcpu)); 150 if (buf == NULL) { 151 _kvm_err(kd, kd->program, "out of memory"); 152 return ((void *)-1); 153 } 154 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf, 155 sizeof(struct pcpu)) != sizeof(struct pcpu)) { 156 _kvm_err(kd, kd->program, "unable to read per-CPU data"); 157 free(buf); 158 return ((void *)-1); 159 } 160 return (buf); 161 } 162 163 int 164 kvm_getmaxcpu(kvm_t *kd) 165 { 166 167 if (kd == NULL) { 168 _kvm_pcpu_clear(); 169 return (0); 170 } 171 172 if (maxcpu == 0) 173 if (_kvm_pcpu_init(kd) < 0) 174 return (-1); 175 return (maxcpu); 176 } 177 178 int 179 kvm_getncpus(kvm_t *kd) 180 { 181 182 if (mp_ncpus == 0) 183 if (_kvm_pcpu_init(kd) < 0) 184 return (-1); 185 return (mp_ncpus); 186 } 187 188 static int 189 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error) 190 { 191 192 if (!kd->dpcpu_initialized) { 193 if (report_error) 194 _kvm_err(kd, kd->program, "%s: not initialized", 195 __func__); 196 return (-1); 197 } 198 if (cpu >= kd->dpcpu_maxcpus) { 199 if (report_error) 200 _kvm_err(kd, kd->program, "%s: CPU %u too big", 201 __func__, cpu); 202 return (-1); 203 } 204 if (kd->dpcpu_off[cpu] == 0) { 205 if (report_error) 206 _kvm_err(kd, kd->program, "%s: CPU %u not found", 207 __func__, cpu); 208 return (-1); 209 } 210 kd->dpcpu_curcpu = cpu; 211 kd->dpcpu_curoff = kd->dpcpu_off[cpu]; 212 return (0); 213 } 214 215 /* 216 * Set up libkvm to handle dynamic per-CPU memory. 217 */ 218 static int 219 _kvm_dpcpu_init(kvm_t *kd) 220 { 221 struct kvm_nlist nl[] = { 222 #define NLIST_START_SET_PCPU 0 223 { .n_name = "___start_" DPCPU_SETNAME }, 224 #define NLIST_STOP_SET_PCPU 1 225 { .n_name = "___stop_" DPCPU_SETNAME }, 226 #define NLIST_DPCPU_OFF 2 227 { .n_name = "_dpcpu_off" }, 228 #define NLIST_MP_MAXCPUS 3 229 { .n_name = "_mp_maxcpus" }, 230 { .n_name = NULL }, 231 }; 232 uintptr_t *dpcpu_off_buf; 233 size_t len; 234 u_int dpcpu_maxcpus; 235 236 /* 237 * XXX: This only works for native kernels for now. 238 */ 239 if (!kvm_native(kd)) 240 return (-1); 241 242 /* 243 * Locate and cache locations of important symbols using the internal 244 * version of _kvm_nlist, turning off initialization to avoid 245 * recursion in case of unresolveable symbols. 246 */ 247 if (_kvm_nlist(kd, nl, 0) != 0) 248 return (-1); 249 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus, 250 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus)) 251 return (-1); 252 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf); 253 dpcpu_off_buf = malloc(len); 254 if (dpcpu_off_buf == NULL) 255 return (-1); 256 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) != 257 (ssize_t)len) { 258 free(dpcpu_off_buf); 259 return (-1); 260 } 261 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value; 262 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value; 263 kd->dpcpu_maxcpus = dpcpu_maxcpus; 264 kd->dpcpu_off = dpcpu_off_buf; 265 kd->dpcpu_initialized = 1; 266 (void)_kvm_dpcpu_setcpu(kd, 0, 0); 267 return (0); 268 } 269 270 /* 271 * Check whether the dpcpu module has been initialized successfully or not, 272 * initialize it if permitted. 273 */ 274 int 275 _kvm_dpcpu_initialized(kvm_t *kd, int intialize) 276 { 277 278 if (kd->dpcpu_initialized || !intialize) 279 return (kd->dpcpu_initialized); 280 281 (void)_kvm_dpcpu_init(kd); 282 283 return (kd->dpcpu_initialized); 284 } 285 286 /* 287 * Check whether the value is within the dpcpu symbol range and only if so 288 * adjust the offset relative to the current offset. 289 */ 290 kvaddr_t 291 _kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value) 292 { 293 294 if (value == 0) 295 return (value); 296 297 if (!kd->dpcpu_initialized) 298 return (value); 299 300 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop) 301 return (value); 302 303 return (kd->dpcpu_curoff + value); 304 } 305 306 int 307 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu) 308 { 309 int ret; 310 311 if (!kd->dpcpu_initialized) { 312 ret = _kvm_dpcpu_init(kd); 313 if (ret != 0) { 314 _kvm_err(kd, kd->program, "%s: init failed", 315 __func__); 316 return (ret); 317 } 318 } 319 320 return (_kvm_dpcpu_setcpu(kd, cpu, 1)); 321 } 322 323 /* 324 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation. 325 */ 326 ssize_t 327 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu) 328 { 329 330 if (!kvm_native(kd)) 331 return (-1); 332 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu), 333 buf, size)); 334 } 335 336 /* 337 * Fetch value of a counter(9). 338 */ 339 uint64_t 340 kvm_counter_u64_fetch(kvm_t *kd, u_long base) 341 { 342 uint64_t r, c; 343 344 if (mp_ncpus == 0) 345 if (_kvm_pcpu_init(kd) < 0) 346 return (0); 347 348 r = 0; 349 for (int i = 0; i < mp_ncpus; i++) { 350 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c)) 351 return (0); 352 r += c; 353 } 354 355 return (r); 356 } 357