1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/param.h> 28 #include <sys/t_lock.h> 29 #include <sys/systm.h> 30 #include <sys/spl.h> 31 #include <sys/cmn_err.h> 32 #include <sys/debug.h> 33 #include <sys/kdi_impl.h> 34 #include <sys/cpuvar.h> 35 #include <sys/cpuvar.h> 36 #include <sys/archsystm.h> 37 38 /* 39 * Handle software interrupts through 'softcall' mechanism 40 * 41 * At present softcall mechanism uses a global list headed by softhead. 42 * Entries are added to tail and removed from head so as to preserve FIFO 43 * nature of entries in the softcall list. softcall() takes care of adding 44 * entries to the softtail. 45 * 46 * softint must take care of executing the entries in the FIFO 47 * order. It could be called simultaneously from multiple cpus, however only 48 * one instance of softint should process the softcall list with the exception 49 * when CPU is stuck due to high interrupt load and can't execute callbacks. 50 * State diagram is as follows :- 51 * 52 * - Upper half which is same as old state machine 53 * (IDLE->PEND->DRAIN->IDLE) 54 * 55 * - Lower half which steals the entries from softcall queue and execute 56 * in the context of softint interrupt handler. The interrupt handler 57 * is fired on a different CPU by sending a cross-call. 58 * 59 * Starting state is IDLE. 60 * 61 * softint() 62 * 63 * 64 * (c) 65 * ____________________________________________________ 66 * | ^ ^ 67 * v (a) | (b) | 68 * IDLE--------------------->PEND--------------------->DRAIN 69 * ^ | | 70 * | | | 71 * | | | 72 * | | | 73 * | | | 74 * | d d 75 * | | | 76 * | v v 77 * | PEND DRAIN 78 * | (e) & & 79 * |<-----------------------STEAL STEAL 80 * ^ | 81 * | | 82 * | (e) v 83 * |_________________________<__________________________| 84 * 85 * 86 * 87 * Edge (a)->(b)->(c) are same as old state machine and these 88 * are mutually exclusive state. 89 * 90 * a - When an entry is being enqueued to softcall queue then the state 91 * moves from IDLE to PEND. 92 * 93 * b - When interrupt handler has started processing softcall queue. 94 * 95 * c - When interrupt handler finished processing softcall queue, the 96 * state of machines goes back to IDLE. 97 * 98 * d - softcall() generates another softlevel1 iff interrupt handler 99 * hasn't run recently. 100 * 101 * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1 102 * handler exit because we have processed all the entries. 103 * 104 * When CPU is being pinned by higher level interrupts for more than 105 * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1 106 * handler on the other CPU can drain the queue. 107 * 108 * These states are needed for softcall mechanism since Solaris has only 109 * one interface (ie. siron ) as of now for : 110 * 111 * - raising a soft interrupt architecture independently (ie not through 112 * setsoftint(..) ) 113 * - to process the softcall queue. 114 */ 115 116 #define NSOFTCALLS 200 117 118 /* 119 * Defined states for softcall processing. 120 */ 121 #define SOFT_IDLE 0x01 /* no processing is needed */ 122 #define SOFT_PEND 0x02 /* softcall list needs processing */ 123 #define SOFT_DRAIN 0x04 /* list is being processed */ 124 #define SOFT_STEAL 0x08 /* list is being stolen for draining */ 125 126 typedef struct softcall { 127 void (*sc_func)(void *); /* function to call */ 128 void *sc_arg; /* arg to pass to func */ 129 struct softcall *sc_next; /* next in list */ 130 } softcall_t; 131 132 /* 133 * softcall list and state variables. 134 */ 135 static softcall_t *softcalls; 136 static softcall_t *softhead, *softtail, *softfree; 137 static uint_t softcall_state; 138 static clock_t softcall_tick; 139 static clock_t softcall_countstart, softcall_lastpoke; 140 static uint_t softcall_pokecount; 141 142 /* 143 * Max number of pokes per second before increasing softcall_delay 144 */ 145 uint_t softcall_pokemax = 10; 146 147 /* 148 * This ensures that softcall entries don't get stuck for long. It's expressed 149 * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency 150 * is used, softcall_init() ensures that it's still expressed as 1 = 10 milli 151 * seconds. 152 */ 153 unsigned int softcall_delay = 1; 154 155 /* 156 * The last CPU which will drain softcall queue. 157 */ 158 static int softcall_latest_cpuid = -1; 159 160 /* 161 * CPUSET to hold the CPU which is processing softcall queue 162 * currently. There can be more than one CPU having bit set 163 * but it will happen only when they are stuck. 164 */ 165 static cpuset_t *softcall_cpuset = NULL; 166 167 /* 168 * protects softcall lists and control variable softcall_state. 169 */ 170 static kmutex_t softcall_lock; 171 172 static void (*kdi_softcall_func)(void); 173 extern void siron_poke_cpu(cpuset_t); 174 175 extern void siron(void); 176 extern void kdi_siron(void); 177 178 179 void 180 softcall_init(void) 181 { 182 softcall_t *sc; 183 184 softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP); 185 softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP); 186 for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) { 187 sc->sc_next = softfree; 188 softfree = sc; 189 } 190 mutex_init(&softcall_lock, NULL, MUTEX_SPIN, 191 (void *)ipltospl(SPL8)); 192 softcall_state = SOFT_IDLE; 193 softcall_tick = ddi_get_lbolt(); 194 195 /* 196 * Since softcall_delay is expressed as 1 = 10 milliseconds. 197 */ 198 softcall_delay = softcall_delay * (hz/100); 199 CPUSET_ZERO(*softcall_cpuset); 200 } 201 202 /* 203 * Gets called when softcall queue is not moving forward. We choose 204 * a CPU and poke except the ones which are already poked. 205 */ 206 static int 207 softcall_choose_cpu() 208 { 209 cpu_t *cplist = CPU; 210 cpu_t *cp; 211 int intr_load = INT_MAX; 212 int cpuid = -1; 213 cpuset_t poke; 214 int s; 215 216 ASSERT(getpil() >= DISP_LEVEL); 217 ASSERT(ncpus > 1); 218 ASSERT(MUTEX_HELD(&softcall_lock)); 219 220 CPUSET_ZERO(poke); 221 222 /* 223 * The hint is to start from current CPU. 224 */ 225 cp = cplist; 226 do { 227 /* 228 * Don't select this CPU if : 229 * - in cpuset already 230 * - CPU is not accepting interrupts 231 * - CPU is being offlined 232 */ 233 if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) || 234 (cp->cpu_flags & CPU_ENABLE) == 0 || 235 (cp == cpu_inmotion)) 236 continue; 237 #if defined(__x86) 238 /* 239 * Don't select this CPU if a hypervisor indicates it 240 * isn't currently scheduled onto a physical cpu. We are 241 * looking for a cpu that can respond quickly and the time 242 * to get the virtual cpu scheduled and switched to running 243 * state is likely to be relatively lengthy. 244 */ 245 if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU) 246 continue; 247 #endif /* __x86 */ 248 249 /* if CPU is not busy */ 250 if (cp->cpu_intrload == 0) { 251 cpuid = cp->cpu_id; 252 break; 253 } 254 255 if (cp->cpu_intrload < intr_load) { 256 cpuid = cp->cpu_id; 257 intr_load = cp->cpu_intrload; 258 } else if (cp->cpu_intrload == intr_load) { 259 /* 260 * We want to poke CPUs having similar 261 * load because we don't know which CPU is 262 * can acknowledge level1 interrupt. The 263 * list of such CPUs should not be large. 264 */ 265 if (cpuid != -1) { 266 /* 267 * Put the last CPU chosen because 268 * it also has same interrupt load. 269 */ 270 CPUSET_ADD(poke, cpuid); 271 cpuid = -1; 272 } 273 274 CPUSET_ADD(poke, cp->cpu_id); 275 } 276 } while ((cp = cp->cpu_next_onln) != cplist); 277 278 /* if we found a CPU which suits best to poke */ 279 if (cpuid != -1) { 280 CPUSET_ZERO(poke); 281 CPUSET_ADD(poke, cpuid); 282 } 283 284 if (CPUSET_ISNULL(poke)) { 285 mutex_exit(&softcall_lock); 286 return (0); 287 } 288 289 /* 290 * We first set the bit in cpuset and then poke. 291 */ 292 CPUSET_XOR(*softcall_cpuset, poke); 293 mutex_exit(&softcall_lock); 294 295 /* 296 * If softcall() was called at low pil then we may 297 * get preempted before we raise PIL. It should be okay 298 * because we are just going to poke CPUs now or at most 299 * another thread may start choosing CPUs in this routine. 300 */ 301 s = splhigh(); 302 siron_poke_cpu(poke); 303 splx(s); 304 return (1); 305 } 306 307 308 /* 309 * Call function func with argument arg 310 * at some later time at software interrupt priority 311 */ 312 void 313 softcall(void (*func)(void *), void *arg) 314 { 315 softcall_t *sc; 316 clock_t w, now; 317 318 /* 319 * protect against cross-calls 320 */ 321 mutex_enter(&softcall_lock); 322 /* coalesce identical softcalls */ 323 for (sc = softhead; sc != 0; sc = sc->sc_next) { 324 if (sc->sc_func == func && sc->sc_arg == arg) { 325 goto intr; 326 } 327 } 328 329 if ((sc = softfree) == 0) 330 panic("too many softcalls"); 331 332 softfree = sc->sc_next; 333 sc->sc_func = func; 334 sc->sc_arg = arg; 335 sc->sc_next = 0; 336 337 if (softhead) { 338 softtail->sc_next = sc; 339 softtail = sc; 340 } else 341 softhead = softtail = sc; 342 343 intr: 344 if (softcall_state & SOFT_IDLE) { 345 softcall_state = SOFT_PEND; 346 softcall_tick = ddi_get_lbolt(); 347 mutex_exit(&softcall_lock); 348 siron(); 349 } else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) { 350 now = ddi_get_lbolt(); 351 w = now - softcall_tick; 352 if (w <= softcall_delay || ncpus == 1) { 353 mutex_exit(&softcall_lock); 354 return; 355 } 356 /* 357 * Did we poke less than a second ago? 358 */ 359 if (now - softcall_lastpoke < hz) { 360 /* 361 * We did, increment the poke count and 362 * see if we are poking too often 363 */ 364 if (softcall_pokecount++ == 0) 365 softcall_countstart = now; 366 if (softcall_pokecount > softcall_pokemax) { 367 /* 368 * If poking too much increase the delay 369 */ 370 if (now - softcall_countstart <= hz) 371 softcall_delay++; 372 softcall_pokecount = 0; 373 } 374 } else { 375 /* 376 * poke rate has dropped off, reset the poke monitor 377 */ 378 softcall_pokecount = 0; 379 } 380 softcall_lastpoke = now; 381 if (!(softcall_state & SOFT_STEAL)) { 382 softcall_state |= SOFT_STEAL; 383 384 /* 385 * We want to give some more chance before 386 * fishing around again. 387 */ 388 softcall_tick = now; 389 } 390 391 /* softcall_lock will be released by this routine */ 392 (void) softcall_choose_cpu(); 393 } 394 } 395 396 void 397 kdi_softcall(void (*func)(void)) 398 { 399 kdi_softcall_func = func; 400 401 if (softhead == NULL) 402 kdi_siron(); 403 } 404 405 /* 406 * Called to process software interrupts take one off queue, call it, 407 * repeat. 408 * 409 * Note queue may change during call; softcall_lock, state variables 410 * softcall_state and softcall_latest_cpuid ensures that - 411 * - we don't have multiple cpus pulling from the list (thus causing 412 * a violation of FIFO order with an exception when we are stuck). 413 * - we don't miss a new entry having been added to the head. 414 * - we don't miss a wakeup. 415 */ 416 417 void 418 softint(void) 419 { 420 softcall_t *sc = NULL; 421 void (*func)(); 422 caddr_t arg; 423 int cpu_id = CPU->cpu_id; 424 425 /* 426 * Don't process softcall queue if current CPU is quiesced or 427 * offlined. This can happen when a CPU is running pause 428 * thread but softcall already sent a xcall. 429 */ 430 if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) { 431 if (softcall_cpuset != NULL && 432 CPU_IN_SET(*softcall_cpuset, cpu_id)) { 433 CPUSET_DEL(*softcall_cpuset, cpu_id); 434 goto out; 435 } 436 } 437 438 mutex_enter(&softcall_lock); 439 440 if (softcall_state & (SOFT_STEAL|SOFT_PEND)) { 441 softcall_state = SOFT_DRAIN; 442 } else { 443 /* 444 * The check for softcall_cpuset being 445 * NULL is required because it may get 446 * called very early during boot. 447 */ 448 if (softcall_cpuset != NULL && 449 CPU_IN_SET(*softcall_cpuset, cpu_id)) 450 CPUSET_DEL(*softcall_cpuset, cpu_id); 451 mutex_exit(&softcall_lock); 452 goto out; 453 } 454 455 /* 456 * Setting softcall_latest_cpuid to current CPU ensures 457 * that there is only one active softlevel1 handler to 458 * process softcall queues. 459 * 460 * Since softcall_lock lock is dropped before calling 461 * func (callback), we need softcall_latest_cpuid 462 * to prevent two softlevel1 hanlders working on the 463 * queue when the first softlevel1 handler gets 464 * stuck due to high interrupt load. 465 */ 466 softcall_latest_cpuid = cpu_id; 467 468 /* add ourself to the cpuset */ 469 if (!CPU_IN_SET(*softcall_cpuset, cpu_id)) 470 CPUSET_ADD(*softcall_cpuset, cpu_id); 471 472 for (;;) { 473 softcall_tick = ddi_get_lbolt(); 474 if ((sc = softhead) != NULL) { 475 func = sc->sc_func; 476 arg = sc->sc_arg; 477 softhead = sc->sc_next; 478 sc->sc_next = softfree; 479 softfree = sc; 480 } 481 482 if (sc == NULL) { 483 if (CPU_IN_SET(*softcall_cpuset, cpu_id)) 484 CPUSET_DEL(*softcall_cpuset, cpu_id); 485 486 softcall_state = SOFT_IDLE; 487 ASSERT(softcall_latest_cpuid == cpu_id); 488 softcall_latest_cpuid = -1; 489 490 mutex_exit(&softcall_lock); 491 break; 492 } 493 494 mutex_exit(&softcall_lock); 495 func(arg); 496 mutex_enter(&softcall_lock); 497 498 /* 499 * No longer need softcall processing from current 500 * interrupt handler because either 501 * (a) softcall is in SOFT_IDLE state or 502 * (b) There is a CPU already draining softcall 503 * queue and the current softlevel1 is no 504 * longer required. 505 */ 506 if (softcall_latest_cpuid != cpu_id) { 507 if (CPU_IN_SET(*softcall_cpuset, cpu_id)) 508 CPUSET_DEL(*softcall_cpuset, cpu_id); 509 510 mutex_exit(&softcall_lock); 511 break; 512 } 513 } 514 515 out: 516 if ((func = kdi_softcall_func) != NULL) { 517 kdi_softcall_func = NULL; 518 func(); 519 } 520 } 521