1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/param.h> 30 #include <sys/t_lock.h> 31 #include <sys/systm.h> 32 #include <sys/spl.h> 33 #include <sys/cmn_err.h> 34 #include <sys/debug.h> 35 #include <sys/kdi_impl.h> 36 #include <sys/cpuvar.h> 37 #include <sys/cpuvar.h> 38 #include <sys/archsystm.h> 39 40 /* 41 * Handle software interrupts through 'softcall' mechanism 42 * 43 * At present softcall mechanism uses a global list headed by softhead. 44 * Entries are added to tail and removed from head so as to preserve FIFO 45 * nature of entries in the softcall list. softcall() takes care of adding 46 * entries to the softtail. 47 * 48 * softint must take care of executing the entries in the FIFO 49 * order. It could be called simultaneously from multiple cpus, however only 50 * one instance of softint should process the softcall list with the exception 51 * when CPU is stuck due to high interrupt load and can't execute callbacks. 52 * State diagram is as follows :- 53 * 54 * - Upper half which is same as old state machine 55 * (IDLE->PEND->DRAIN->IDLE) 56 * 57 * - Lower half which steals the entries from softcall queue and execute 58 * in the context of softint interrupt handler. The interrupt handler 59 * is fired on a different CPU by sending a cross-call. 60 * 61 * Starting state is IDLE. 62 * 63 * softint() 64 * 65 * 66 * (c) 67 * ____________________________________________________ 68 * | ^ ^ 69 * v (a) | (b) | 70 * IDLE--------------------->PEND--------------------->DRAIN 71 * ^ | | 72 * | | | 73 * | | | 74 * | | | 75 * | | | 76 * | d d 77 * | | | 78 * | v v 79 * | PEND DRAIN 80 * | (e) & & 81 * |<-----------------------STEAL STEAL 82 * ^ | 83 * | | 84 * | (e) v 85 * |_________________________<__________________________| 86 * 87 * 88 * 89 * Edge (a)->(b)->(c) are same as old state machine and these 90 * are mutually exclusive state. 91 * 92 * a - When an entry is being enqueued to softcall queue then the state 93 * moves from IDLE to PEND. 94 * 95 * b - When interrupt handler has started processing softcall queue. 96 * 97 * c - When interrupt handler finished processing softcall queue, the 98 * state of machines goes back to IDLE. 99 * 100 * d - softcall() generates another softlevel1 iff interrupt handler 101 * hasn't run recently. 102 * 103 * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1 104 * handler exit because we have processed all the entries. 105 * 106 * When CPU is being pinned by higher level interrupts for more than 107 * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1 108 * handler on the other CPU can drain the queue. 109 * 110 * These states are needed for softcall mechanism since Solaris has only 111 * one interface (ie. siron ) as of now for : 112 * 113 * - raising a soft interrupt architecture independently (ie not through 114 * setsoftint(..) ) 115 * - to process the softcall queue. 116 */ 117 118 #define NSOFTCALLS 200 119 120 /* 121 * Defined states for softcall processing. 122 */ 123 #define SOFT_IDLE 0x01 /* no processing is needed */ 124 #define SOFT_PEND 0x02 /* softcall list needs processing */ 125 #define SOFT_DRAIN 0x04 /* list is being processed */ 126 #define SOFT_STEAL 0x08 /* list is being stolen for draining */ 127 128 typedef struct softcall { 129 void (*sc_func)(void *); /* function to call */ 130 void *sc_arg; /* arg to pass to func */ 131 struct softcall *sc_next; /* next in list */ 132 } softcall_t; 133 134 /* 135 * softcall list and state variables. 136 */ 137 static softcall_t *softcalls; 138 static softcall_t *softhead, *softtail, *softfree; 139 static uint_t softcall_state; 140 static clock_t softcall_tick; 141 static clock_t softcall_countstart, softcall_lastpoke; 142 static uint_t softcall_pokecount; 143 144 /* 145 * Max number of pokes per second before increasing softcall_delay 146 */ 147 uint_t softcall_pokemax = 10; 148 149 /* 150 * This ensures that softcall entries don't get stuck for long. It's expressed 151 * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency 152 * is used, softcall_init() ensures that it's still expressed as 1 = 10 milli 153 * seconds. 154 */ 155 unsigned int softcall_delay = 1; 156 157 /* 158 * The last CPU which will drain softcall queue. 159 */ 160 static int softcall_latest_cpuid = -1; 161 162 /* 163 * CPUSET to hold the CPU which is processing softcall queue 164 * currently. There can be more than one CPU having bit set 165 * but it will happen only when they are stuck. 166 */ 167 static cpuset_t *softcall_cpuset = NULL; 168 169 /* 170 * protects softcall lists and control variable softcall_state. 171 */ 172 static kmutex_t softcall_lock; 173 174 static void (*kdi_softcall_func)(void); 175 extern void siron_poke_cpu(cpuset_t); 176 177 extern void siron(void); 178 extern void kdi_siron(void); 179 180 181 void 182 softcall_init(void) 183 { 184 softcall_t *sc; 185 186 softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP); 187 softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP); 188 for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) { 189 sc->sc_next = softfree; 190 softfree = sc; 191 } 192 mutex_init(&softcall_lock, NULL, MUTEX_SPIN, 193 (void *)ipltospl(SPL8)); 194 softcall_state = SOFT_IDLE; 195 softcall_tick = lbolt; 196 197 /* 198 * Since softcall_delay is expressed as 1 = 10 milliseconds. 199 */ 200 softcall_delay = softcall_delay * (hz/100); 201 CPUSET_ZERO(*softcall_cpuset); 202 } 203 204 /* 205 * Gets called when softcall queue is not moving forward. We choose 206 * a CPU and poke except the ones which are already poked. 207 */ 208 static int 209 softcall_choose_cpu() 210 { 211 cpu_t *cplist = CPU; 212 cpu_t *cp; 213 int intr_load = INT_MAX; 214 int cpuid = -1; 215 cpuset_t poke; 216 int s; 217 218 ASSERT(getpil() >= DISP_LEVEL); 219 ASSERT(ncpus > 1); 220 ASSERT(MUTEX_HELD(&softcall_lock)); 221 222 CPUSET_ZERO(poke); 223 224 /* 225 * The hint is to start from current CPU. 226 */ 227 cp = cplist; 228 do { 229 /* 230 * Don't select this CPU if : 231 * - in cpuset already 232 * - CPU is not accepting interrupts 233 * - CPU is being offlined 234 */ 235 if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) || 236 (cp->cpu_flags & CPU_ENABLE) == 0 || 237 (cp == cpu_inmotion)) 238 continue; 239 #if defined(__x86) 240 /* 241 * Don't select this CPU if a hypervisor indicates it 242 * isn't currently scheduled onto a physical cpu. We are 243 * looking for a cpu that can respond quickly and the time 244 * to get the virtual cpu scheduled and switched to running 245 * state is likely to be relatively lengthy. 246 */ 247 if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU) 248 continue; 249 #endif /* __x86 */ 250 251 /* if CPU is not busy */ 252 if (cp->cpu_intrload == 0) { 253 cpuid = cp->cpu_id; 254 break; 255 } 256 257 if (cp->cpu_intrload < intr_load) { 258 cpuid = cp->cpu_id; 259 intr_load = cp->cpu_intrload; 260 } else if (cp->cpu_intrload == intr_load) { 261 /* 262 * We want to poke CPUs having similar 263 * load because we don't know which CPU is 264 * can acknowledge level1 interrupt. The 265 * list of such CPUs should not be large. 266 */ 267 if (cpuid != -1) { 268 /* 269 * Put the last CPU chosen because 270 * it also has same interrupt load. 271 */ 272 CPUSET_ADD(poke, cpuid); 273 cpuid = -1; 274 } 275 276 CPUSET_ADD(poke, cp->cpu_id); 277 } 278 } while ((cp = cp->cpu_next_onln) != cplist); 279 280 /* if we found a CPU which suits best to poke */ 281 if (cpuid != -1) { 282 CPUSET_ZERO(poke); 283 CPUSET_ADD(poke, cpuid); 284 } 285 286 if (CPUSET_ISNULL(poke)) { 287 mutex_exit(&softcall_lock); 288 return (0); 289 } 290 291 /* 292 * We first set the bit in cpuset and then poke. 293 */ 294 CPUSET_XOR(*softcall_cpuset, poke); 295 mutex_exit(&softcall_lock); 296 297 /* 298 * If softcall() was called at low pil then we may 299 * get preempted before we raise PIL. It should be okay 300 * because we are just going to poke CPUs now or at most 301 * another thread may start choosing CPUs in this routine. 302 */ 303 s = splhigh(); 304 siron_poke_cpu(poke); 305 splx(s); 306 return (1); 307 } 308 309 310 /* 311 * Call function func with argument arg 312 * at some later time at software interrupt priority 313 */ 314 void 315 softcall(void (*func)(void *), void *arg) 316 { 317 softcall_t *sc; 318 clock_t w, now; 319 320 /* 321 * protect against cross-calls 322 */ 323 mutex_enter(&softcall_lock); 324 /* coalesce identical softcalls */ 325 for (sc = softhead; sc != 0; sc = sc->sc_next) { 326 if (sc->sc_func == func && sc->sc_arg == arg) { 327 goto intr; 328 } 329 } 330 331 if ((sc = softfree) == 0) 332 panic("too many softcalls"); 333 334 softfree = sc->sc_next; 335 sc->sc_func = func; 336 sc->sc_arg = arg; 337 sc->sc_next = 0; 338 339 if (softhead) { 340 softtail->sc_next = sc; 341 softtail = sc; 342 } else 343 softhead = softtail = sc; 344 345 intr: 346 if (softcall_state & SOFT_IDLE) { 347 softcall_state = SOFT_PEND; 348 softcall_tick = lbolt; 349 mutex_exit(&softcall_lock); 350 siron(); 351 } else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) { 352 now = lbolt; 353 w = now - softcall_tick; 354 if (w <= softcall_delay || ncpus == 1) { 355 mutex_exit(&softcall_lock); 356 return; 357 } 358 /* 359 * Did we poke less than a second ago? 360 */ 361 if (now - softcall_lastpoke < hz) { 362 /* 363 * We did, increment the poke count and 364 * see if we are poking too often 365 */ 366 if (softcall_pokecount++ == 0) 367 softcall_countstart = now; 368 if (softcall_pokecount > softcall_pokemax) { 369 /* 370 * If poking too much increase the delay 371 */ 372 if (now - softcall_countstart <= hz) 373 softcall_delay++; 374 softcall_pokecount = 0; 375 } 376 } else { 377 /* 378 * poke rate has dropped off, reset the poke monitor 379 */ 380 softcall_pokecount = 0; 381 } 382 softcall_lastpoke = lbolt; 383 if (!(softcall_state & SOFT_STEAL)) { 384 softcall_state |= SOFT_STEAL; 385 386 /* 387 * We want to give some more chance before 388 * fishing around again. 389 */ 390 softcall_tick = lbolt; 391 } 392 393 /* softcall_lock will be released by this routine */ 394 (void) softcall_choose_cpu(); 395 } 396 } 397 398 void 399 kdi_softcall(void (*func)(void)) 400 { 401 kdi_softcall_func = func; 402 403 if (softhead == NULL) 404 kdi_siron(); 405 } 406 407 /* 408 * Called to process software interrupts take one off queue, call it, 409 * repeat. 410 * 411 * Note queue may change during call; softcall_lock, state variables 412 * softcall_state and softcall_latest_cpuid ensures that - 413 * - we don't have multiple cpus pulling from the list (thus causing 414 * a violation of FIFO order with an exception when we are stuck). 415 * - we don't miss a new entry having been added to the head. 416 * - we don't miss a wakeup. 417 */ 418 419 void 420 softint(void) 421 { 422 softcall_t *sc = NULL; 423 void (*func)(); 424 caddr_t arg; 425 int cpu_id = CPU->cpu_id; 426 427 /* 428 * Don't process softcall queue if current CPU is quiesced or 429 * offlined. This can happen when a CPU is running pause 430 * thread but softcall already sent a xcall. 431 */ 432 if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) { 433 if (softcall_cpuset != NULL && 434 CPU_IN_SET(*softcall_cpuset, cpu_id)) { 435 CPUSET_DEL(*softcall_cpuset, cpu_id); 436 goto out; 437 } 438 } 439 440 mutex_enter(&softcall_lock); 441 442 if (softcall_state & (SOFT_STEAL|SOFT_PEND)) { 443 softcall_state = SOFT_DRAIN; 444 } else { 445 /* 446 * The check for softcall_cpuset being 447 * NULL is required because it may get 448 * called very early during boot. 449 */ 450 if (softcall_cpuset != NULL && 451 CPU_IN_SET(*softcall_cpuset, cpu_id)) 452 CPUSET_DEL(*softcall_cpuset, cpu_id); 453 mutex_exit(&softcall_lock); 454 goto out; 455 } 456 457 /* 458 * Setting softcall_latest_cpuid to current CPU ensures 459 * that there is only one active softlevel1 handler to 460 * process softcall queues. 461 * 462 * Since softcall_lock lock is dropped before calling 463 * func (callback), we need softcall_latest_cpuid 464 * to prevent two softlevel1 hanlders working on the 465 * queue when the first softlevel1 handler gets 466 * stuck due to high interrupt load. 467 */ 468 softcall_latest_cpuid = cpu_id; 469 470 /* add ourself to the cpuset */ 471 if (!CPU_IN_SET(*softcall_cpuset, cpu_id)) 472 CPUSET_ADD(*softcall_cpuset, cpu_id); 473 474 for (;;) { 475 softcall_tick = lbolt; 476 if ((sc = softhead) != NULL) { 477 func = sc->sc_func; 478 arg = sc->sc_arg; 479 softhead = sc->sc_next; 480 sc->sc_next = softfree; 481 softfree = sc; 482 } 483 484 if (sc == NULL) { 485 if (CPU_IN_SET(*softcall_cpuset, cpu_id)) 486 CPUSET_DEL(*softcall_cpuset, cpu_id); 487 488 softcall_state = SOFT_IDLE; 489 ASSERT(softcall_latest_cpuid == cpu_id); 490 softcall_latest_cpuid = -1; 491 492 mutex_exit(&softcall_lock); 493 break; 494 } 495 496 mutex_exit(&softcall_lock); 497 func(arg); 498 mutex_enter(&softcall_lock); 499 500 /* 501 * No longer need softcall processing from current 502 * interrupt handler because either 503 * (a) softcall is in SOFT_IDLE state or 504 * (b) There is a CPU already draining softcall 505 * queue and the current softlevel1 is no 506 * longer required. 507 */ 508 if (softcall_latest_cpuid != cpu_id) { 509 if (CPU_IN_SET(*softcall_cpuset, cpu_id)) 510 CPUSET_DEL(*softcall_cpuset, cpu_id); 511 512 mutex_exit(&softcall_lock); 513 break; 514 } 515 } 516 517 out: 518 if ((func = kdi_softcall_func) != NULL) { 519 kdi_softcall_func = NULL; 520 func(); 521 } 522 } 523