1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2001 Wind River Systems, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. Neither the name of the author nor the names of any co-contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 */ 35 36 /* 37 * This module provides MI support for per-cpu data. 38 * 39 * Each architecture determines the mapping of logical CPU IDs to physical 40 * CPUs. The requirements of this mapping are as follows: 41 * - Logical CPU IDs must reside in the range 0 ... MAXCPU - 1. 42 * - The mapping is not required to be dense. That is, there may be 43 * gaps in the mappings. 44 * - The platform sets the value of MAXCPU in <machine/param.h>. 45 * - It is suggested, but not required, that in the non-SMP case, the 46 * platform define MAXCPU to be 1 and define the logical ID of the 47 * sole CPU as 0. 48 */ 49 50 #include <sys/cdefs.h> 51 __FBSDID("$FreeBSD$"); 52 53 #include "opt_ddb.h" 54 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/sysctl.h> 58 #include <sys/lock.h> 59 #include <sys/malloc.h> 60 #include <sys/pcpu.h> 61 #include <sys/proc.h> 62 #include <sys/smp.h> 63 #include <sys/sx.h> 64 #include <vm/uma.h> 65 #include <ddb/ddb.h> 66 67 static MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); 68 69 struct dpcpu_free { 70 uintptr_t df_start; 71 int df_len; 72 TAILQ_ENTRY(dpcpu_free) df_link; 73 }; 74 75 DPCPU_DEFINE_STATIC(char, modspace[DPCPU_MODMIN] __aligned(__alignof(void *))); 76 static TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); 77 static struct sx dpcpu_lock; 78 uintptr_t dpcpu_off[MAXCPU]; 79 struct pcpu *cpuid_to_pcpu[MAXCPU]; 80 struct cpuhead cpuhead = STAILQ_HEAD_INITIALIZER(cpuhead); 81 82 /* 83 * Initialize the MI portions of a struct pcpu. 84 */ 85 void 86 pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 87 { 88 89 bzero(pcpu, size); 90 KASSERT(cpuid >= 0 && cpuid < MAXCPU, 91 ("pcpu_init: invalid cpuid %d", cpuid)); 92 pcpu->pc_cpuid = cpuid; 93 cpuid_to_pcpu[cpuid] = pcpu; 94 STAILQ_INSERT_TAIL(&cpuhead, pcpu, pc_allcpu); 95 cpu_pcpu_init(pcpu, cpuid, size); 96 pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; 97 pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; 98 } 99 100 void 101 dpcpu_init(void *dpcpu, int cpuid) 102 { 103 struct pcpu *pcpu; 104 105 pcpu = pcpu_find(cpuid); 106 pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START; 107 108 /* 109 * Initialize defaults from our linker section. 110 */ 111 memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES); 112 113 /* 114 * Place it in the global pcpu offset array. 115 */ 116 dpcpu_off[cpuid] = pcpu->pc_dynamic; 117 } 118 119 static void 120 dpcpu_startup(void *dummy __unused) 121 { 122 struct dpcpu_free *df; 123 124 df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 125 df->df_start = (uintptr_t)&DPCPU_NAME(modspace); 126 df->df_len = DPCPU_MODMIN; 127 TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); 128 sx_init(&dpcpu_lock, "dpcpu alloc lock"); 129 } 130 SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, NULL); 131 132 /* 133 * UMA_PCPU_ZONE zones, that are available for all kernel 134 * consumers. Right now 64 bit zone is used for counter(9). 135 */ 136 137 uma_zone_t pcpu_zone_64; 138 139 static void 140 pcpu_zones_startup(void) 141 { 142 143 pcpu_zone_64 = uma_zcreate("64 pcpu", sizeof(uint64_t), 144 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); 145 } 146 SYSINIT(pcpu_zones, SI_SUB_VM, SI_ORDER_ANY, pcpu_zones_startup, NULL); 147 148 /* 149 * First-fit extent based allocator for allocating space in the per-cpu 150 * region reserved for modules. This is only intended for use by the 151 * kernel linkers to place module linker sets. 152 */ 153 void * 154 dpcpu_alloc(int size) 155 { 156 struct dpcpu_free *df; 157 void *s; 158 159 s = NULL; 160 size = roundup2(size, sizeof(void *)); 161 sx_xlock(&dpcpu_lock); 162 TAILQ_FOREACH(df, &dpcpu_head, df_link) { 163 if (df->df_len < size) 164 continue; 165 if (df->df_len == size) { 166 s = (void *)df->df_start; 167 TAILQ_REMOVE(&dpcpu_head, df, df_link); 168 free(df, M_PCPU); 169 break; 170 } 171 s = (void *)df->df_start; 172 df->df_len -= size; 173 df->df_start = df->df_start + size; 174 break; 175 } 176 sx_xunlock(&dpcpu_lock); 177 178 return (s); 179 } 180 181 /* 182 * Free dynamic per-cpu space at module unload time. 183 */ 184 void 185 dpcpu_free(void *s, int size) 186 { 187 struct dpcpu_free *df; 188 struct dpcpu_free *dn; 189 uintptr_t start; 190 uintptr_t end; 191 192 size = roundup2(size, sizeof(void *)); 193 start = (uintptr_t)s; 194 end = start + size; 195 /* 196 * Free a region of space and merge it with as many neighbors as 197 * possible. Keeping the list sorted simplifies this operation. 198 */ 199 sx_xlock(&dpcpu_lock); 200 TAILQ_FOREACH(df, &dpcpu_head, df_link) { 201 if (df->df_start > end) 202 break; 203 /* 204 * If we expand at the end of an entry we may have to 205 * merge it with the one following it as well. 206 */ 207 if (df->df_start + df->df_len == start) { 208 df->df_len += size; 209 dn = TAILQ_NEXT(df, df_link); 210 if (df->df_start + df->df_len == dn->df_start) { 211 df->df_len += dn->df_len; 212 TAILQ_REMOVE(&dpcpu_head, dn, df_link); 213 free(dn, M_PCPU); 214 } 215 sx_xunlock(&dpcpu_lock); 216 return; 217 } 218 if (df->df_start == end) { 219 df->df_start = start; 220 df->df_len += size; 221 sx_xunlock(&dpcpu_lock); 222 return; 223 } 224 } 225 dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 226 dn->df_start = start; 227 dn->df_len = size; 228 if (df) 229 TAILQ_INSERT_BEFORE(df, dn, df_link); 230 else 231 TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link); 232 sx_xunlock(&dpcpu_lock); 233 } 234 235 /* 236 * Initialize the per-cpu storage from an updated linker-set region. 237 */ 238 void 239 dpcpu_copy(void *s, int size) 240 { 241 #ifdef SMP 242 uintptr_t dpcpu; 243 int i; 244 245 CPU_FOREACH(i) { 246 dpcpu = dpcpu_off[i]; 247 if (dpcpu == 0) 248 continue; 249 memcpy((void *)(dpcpu + (uintptr_t)s), s, size); 250 } 251 #else 252 memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size); 253 #endif 254 } 255 256 /* 257 * Destroy a struct pcpu. 258 */ 259 void 260 pcpu_destroy(struct pcpu *pcpu) 261 { 262 263 STAILQ_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu); 264 cpuid_to_pcpu[pcpu->pc_cpuid] = NULL; 265 dpcpu_off[pcpu->pc_cpuid] = 0; 266 } 267 268 /* 269 * Locate a struct pcpu by cpu id. 270 */ 271 struct pcpu * 272 pcpu_find(u_int cpuid) 273 { 274 275 return (cpuid_to_pcpu[cpuid]); 276 } 277 278 int 279 sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) 280 { 281 uintptr_t dpcpu; 282 int64_t count; 283 int i; 284 285 count = 0; 286 CPU_FOREACH(i) { 287 dpcpu = dpcpu_off[i]; 288 if (dpcpu == 0) 289 continue; 290 count += *(int64_t *)(dpcpu + (uintptr_t)arg1); 291 } 292 return (SYSCTL_OUT(req, &count, sizeof(count))); 293 } 294 295 int 296 sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS) 297 { 298 uintptr_t dpcpu; 299 long count; 300 int i; 301 302 count = 0; 303 CPU_FOREACH(i) { 304 dpcpu = dpcpu_off[i]; 305 if (dpcpu == 0) 306 continue; 307 count += *(long *)(dpcpu + (uintptr_t)arg1); 308 } 309 return (SYSCTL_OUT(req, &count, sizeof(count))); 310 } 311 312 int 313 sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) 314 { 315 uintptr_t dpcpu; 316 int count; 317 int i; 318 319 count = 0; 320 CPU_FOREACH(i) { 321 dpcpu = dpcpu_off[i]; 322 if (dpcpu == 0) 323 continue; 324 count += *(int *)(dpcpu + (uintptr_t)arg1); 325 } 326 return (SYSCTL_OUT(req, &count, sizeof(count))); 327 } 328 329 #ifdef DDB 330 DB_SHOW_COMMAND(dpcpu_off, db_show_dpcpu_off) 331 { 332 int id; 333 334 CPU_FOREACH(id) { 335 db_printf("dpcpu_off[%2d] = 0x%jx (+ DPCPU_START = %p)\n", 336 id, (uintmax_t)dpcpu_off[id], 337 (void *)(uintptr_t)(dpcpu_off[id] + DPCPU_START)); 338 } 339 } 340 341 static void 342 show_pcpu(struct pcpu *pc) 343 { 344 struct thread *td; 345 346 db_printf("cpuid = %d\n", pc->pc_cpuid); 347 db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic); 348 db_printf("curthread = "); 349 td = pc->pc_curthread; 350 if (td != NULL) 351 db_printf("%p: pid %d tid %d \"%s\"\n", td, td->td_proc->p_pid, 352 td->td_tid, td->td_name); 353 else 354 db_printf("none\n"); 355 db_printf("curpcb = %p\n", pc->pc_curpcb); 356 db_printf("fpcurthread = "); 357 td = pc->pc_fpcurthread; 358 if (td != NULL) 359 db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 360 td->td_name); 361 else 362 db_printf("none\n"); 363 db_printf("idlethread = "); 364 td = pc->pc_idlethread; 365 if (td != NULL) 366 db_printf("%p: tid %d \"%s\"\n", td, td->td_tid, td->td_name); 367 else 368 db_printf("none\n"); 369 db_show_mdpcpu(pc); 370 371 #ifdef VIMAGE 372 db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet); 373 #endif 374 375 #ifdef WITNESS 376 db_printf("spin locks held:\n"); 377 witness_list_locks(&pc->pc_spinlocks, db_printf); 378 #endif 379 } 380 381 DB_SHOW_COMMAND(pcpu, db_show_pcpu) 382 { 383 struct pcpu *pc; 384 int id; 385 386 if (have_addr) 387 id = ((addr >> 4) % 16) * 10 + (addr % 16); 388 else 389 id = PCPU_GET(cpuid); 390 pc = pcpu_find(id); 391 if (pc == NULL) { 392 db_printf("CPU %d not found\n", id); 393 return; 394 } 395 show_pcpu(pc); 396 } 397 398 DB_SHOW_ALL_COMMAND(pcpu, db_show_cpu_all) 399 { 400 struct pcpu *pc; 401 int id; 402 403 db_printf("Current CPU: %d\n\n", PCPU_GET(cpuid)); 404 CPU_FOREACH(id) { 405 pc = pcpu_find(id); 406 if (pc != NULL) { 407 show_pcpu(pc); 408 db_printf("\n"); 409 } 410 } 411 } 412 DB_SHOW_ALIAS(allpcpu, db_show_cpu_all); 413 #endif 414