1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/types.h>
27 #include <sys/param.h>
28 #include <sys/t_lock.h>
29 #include <sys/systm.h>
30 #include <sys/spl.h>
31 #include <sys/cmn_err.h>
32 #include <sys/debug.h>
33 #include <sys/kdi_impl.h>
34 #include <sys/cpuvar.h>
35 #include <sys/cpuvar.h>
36 #include <sys/archsystm.h>
37
38 /*
39 * Handle software interrupts through 'softcall' mechanism
40 *
41 * At present softcall mechanism uses a global list headed by softhead.
42 * Entries are added to tail and removed from head so as to preserve FIFO
43 * nature of entries in the softcall list. softcall() takes care of adding
44 * entries to the softtail.
45 *
46 * softint must take care of executing the entries in the FIFO
47 * order. It could be called simultaneously from multiple cpus, however only
48 * one instance of softint should process the softcall list with the exception
49 * when CPU is stuck due to high interrupt load and can't execute callbacks.
50 * State diagram is as follows :-
51 *
52 * - Upper half which is same as old state machine
53 * (IDLE->PEND->DRAIN->IDLE)
54 *
55 * - Lower half which steals the entries from softcall queue and execute
56 * in the context of softint interrupt handler. The interrupt handler
57 * is fired on a different CPU by sending a cross-call.
58 *
59 * Starting state is IDLE.
60 *
61 * softint()
62 *
63 *
64 * (c)
65 * ____________________________________________________
66 * | ^ ^
67 * v (a) | (b) |
68 * IDLE--------------------->PEND--------------------->DRAIN
69 * ^ | |
70 * | | |
71 * | | |
72 * | | |
73 * | | |
74 * | d d
75 * | | |
76 * | v v
77 * | PEND DRAIN
78 * | (e) & &
79 * |<-----------------------STEAL STEAL
80 * ^ |
81 * | |
82 * | (e) v
83 * |_________________________<__________________________|
84 *
85 *
86 *
87 * Edge (a)->(b)->(c) are same as old state machine and these
88 * are mutually exclusive state.
89 *
90 * a - When an entry is being enqueued to softcall queue then the state
91 * moves from IDLE to PEND.
92 *
93 * b - When interrupt handler has started processing softcall queue.
94 *
95 * c - When interrupt handler finished processing softcall queue, the
96 * state of machines goes back to IDLE.
97 *
98 * d - softcall() generates another softlevel1 iff interrupt handler
99 * hasn't run recently.
100 *
101 * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1
102 * handler exit because we have processed all the entries.
103 *
104 * When CPU is being pinned by higher level interrupts for more than
105 * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1
106 * handler on the other CPU can drain the queue.
107 *
108 * These states are needed for softcall mechanism since Solaris has only
109 * one interface (ie. siron ) as of now for :
110 *
111 * - raising a soft interrupt architecture independently (ie not through
112 * setsoftint(..) )
113 * - to process the softcall queue.
114 */
115
116 #define NSOFTCALLS 200
117
118 /*
119 * Defined states for softcall processing.
120 */
121 #define SOFT_IDLE 0x01 /* no processing is needed */
122 #define SOFT_PEND 0x02 /* softcall list needs processing */
123 #define SOFT_DRAIN 0x04 /* list is being processed */
124 #define SOFT_STEAL 0x08 /* list is being stolen for draining */
125
126 typedef struct softcall {
127 void (*sc_func)(void *); /* function to call */
128 void *sc_arg; /* arg to pass to func */
129 struct softcall *sc_next; /* next in list */
130 } softcall_t;
131
132 /*
133 * softcall list and state variables.
134 */
135 static softcall_t *softcalls;
136 static softcall_t *softhead, *softtail, *softfree;
137 static uint_t softcall_state;
138 static clock_t softcall_tick;
139 static clock_t softcall_countstart, softcall_lastpoke;
140 static uint_t softcall_pokecount;
141
142 /*
143 * Max number of pokes per second before increasing softcall_delay
144 */
145 uint_t softcall_pokemax = 10;
146
147 /*
148 * This ensures that softcall entries don't get stuck for long. It's expressed
149 * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency
150 * is used, softcall_init() ensures that it's still expressed as 1 = 10 milli
151 * seconds.
152 */
153 unsigned int softcall_delay = 1;
154
155 /*
156 * The last CPU which will drain softcall queue.
157 */
158 static int softcall_latest_cpuid = -1;
159
160 /*
161 * CPUSET to hold the CPU which is processing softcall queue
162 * currently. There can be more than one CPU having bit set
163 * but it will happen only when they are stuck.
164 */
165 static cpuset_t *softcall_cpuset = NULL;
166
167 /*
168 * protects softcall lists and control variable softcall_state.
169 */
170 static kmutex_t softcall_lock;
171
172 static void (*kdi_softcall_func)(void);
173 extern void siron_poke_cpu(cpuset_t);
174
175 extern void siron(void);
176 extern void kdi_siron(void);
177
178
179 void
softcall_init(void)180 softcall_init(void)
181 {
182 softcall_t *sc;
183
184 softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP);
185 softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP);
186 for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) {
187 sc->sc_next = softfree;
188 softfree = sc;
189 }
190 mutex_init(&softcall_lock, NULL, MUTEX_SPIN,
191 (void *)ipltospl(SPL8));
192 softcall_state = SOFT_IDLE;
193 softcall_tick = ddi_get_lbolt();
194
195 /*
196 * Since softcall_delay is expressed as 1 = 10 milliseconds.
197 */
198 softcall_delay = softcall_delay * (hz/100);
199 CPUSET_ZERO(*softcall_cpuset);
200 }
201
202 /*
203 * Gets called when softcall queue is not moving forward. We choose
204 * a CPU and poke except the ones which are already poked.
205 */
206 static int
softcall_choose_cpu()207 softcall_choose_cpu()
208 {
209 cpu_t *cplist = CPU;
210 cpu_t *cp;
211 int intr_load = INT_MAX;
212 int cpuid = -1;
213 cpuset_t poke;
214 int s;
215
216 ASSERT(getpil() >= DISP_LEVEL);
217 ASSERT(ncpus > 1);
218 ASSERT(MUTEX_HELD(&softcall_lock));
219
220 CPUSET_ZERO(poke);
221
222 /*
223 * The hint is to start from current CPU.
224 */
225 cp = cplist;
226 do {
227 /*
228 * Don't select this CPU if :
229 * - in cpuset already
230 * - CPU is not accepting interrupts
231 * - CPU is being offlined
232 */
233 if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) ||
234 (cp->cpu_flags & CPU_ENABLE) == 0 ||
235 (cp == cpu_inmotion))
236 continue;
237 #if defined(__x86)
238 /*
239 * Don't select this CPU if a hypervisor indicates it
240 * isn't currently scheduled onto a physical cpu. We are
241 * looking for a cpu that can respond quickly and the time
242 * to get the virtual cpu scheduled and switched to running
243 * state is likely to be relatively lengthy.
244 */
245 if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU)
246 continue;
247 #endif /* __x86 */
248
249 /* if CPU is not busy */
250 if (cp->cpu_intrload == 0) {
251 cpuid = cp->cpu_id;
252 break;
253 }
254
255 if (cp->cpu_intrload < intr_load) {
256 cpuid = cp->cpu_id;
257 intr_load = cp->cpu_intrload;
258 } else if (cp->cpu_intrload == intr_load) {
259 /*
260 * We want to poke CPUs having similar
261 * load because we don't know which CPU is
262 * can acknowledge level1 interrupt. The
263 * list of such CPUs should not be large.
264 */
265 if (cpuid != -1) {
266 /*
267 * Put the last CPU chosen because
268 * it also has same interrupt load.
269 */
270 CPUSET_ADD(poke, cpuid);
271 cpuid = -1;
272 }
273
274 CPUSET_ADD(poke, cp->cpu_id);
275 }
276 } while ((cp = cp->cpu_next_onln) != cplist);
277
278 /* if we found a CPU which suits best to poke */
279 if (cpuid != -1) {
280 CPUSET_ZERO(poke);
281 CPUSET_ADD(poke, cpuid);
282 }
283
284 if (CPUSET_ISNULL(poke)) {
285 mutex_exit(&softcall_lock);
286 return (0);
287 }
288
289 /*
290 * We first set the bit in cpuset and then poke.
291 */
292 CPUSET_XOR(*softcall_cpuset, poke);
293 mutex_exit(&softcall_lock);
294
295 /*
296 * If softcall() was called at low pil then we may
297 * get preempted before we raise PIL. It should be okay
298 * because we are just going to poke CPUs now or at most
299 * another thread may start choosing CPUs in this routine.
300 */
301 s = splhigh();
302 siron_poke_cpu(poke);
303 splx(s);
304 return (1);
305 }
306
307
308 /*
309 * Call function func with argument arg
310 * at some later time at software interrupt priority
311 */
312 void
softcall(void (* func)(void *),void * arg)313 softcall(void (*func)(void *), void *arg)
314 {
315 softcall_t *sc;
316 clock_t w, now;
317
318 /*
319 * protect against cross-calls
320 */
321 mutex_enter(&softcall_lock);
322 /* coalesce identical softcalls */
323 for (sc = softhead; sc != 0; sc = sc->sc_next) {
324 if (sc->sc_func == func && sc->sc_arg == arg) {
325 goto intr;
326 }
327 }
328
329 if ((sc = softfree) == 0)
330 panic("too many softcalls");
331
332 softfree = sc->sc_next;
333 sc->sc_func = func;
334 sc->sc_arg = arg;
335 sc->sc_next = 0;
336
337 if (softhead) {
338 softtail->sc_next = sc;
339 softtail = sc;
340 } else
341 softhead = softtail = sc;
342
343 intr:
344 if (softcall_state & SOFT_IDLE) {
345 softcall_state = SOFT_PEND;
346 softcall_tick = ddi_get_lbolt();
347 mutex_exit(&softcall_lock);
348 siron();
349 } else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) {
350 now = ddi_get_lbolt();
351 w = now - softcall_tick;
352 if (w <= softcall_delay || ncpus == 1) {
353 mutex_exit(&softcall_lock);
354 return;
355 }
356 /*
357 * Did we poke less than a second ago?
358 */
359 if (now - softcall_lastpoke < hz) {
360 /*
361 * We did, increment the poke count and
362 * see if we are poking too often
363 */
364 if (softcall_pokecount++ == 0)
365 softcall_countstart = now;
366 if (softcall_pokecount > softcall_pokemax) {
367 /*
368 * If poking too much increase the delay
369 */
370 if (now - softcall_countstart <= hz)
371 softcall_delay++;
372 softcall_pokecount = 0;
373 }
374 } else {
375 /*
376 * poke rate has dropped off, reset the poke monitor
377 */
378 softcall_pokecount = 0;
379 }
380 softcall_lastpoke = now;
381 if (!(softcall_state & SOFT_STEAL)) {
382 softcall_state |= SOFT_STEAL;
383
384 /*
385 * We want to give some more chance before
386 * fishing around again.
387 */
388 softcall_tick = now;
389 }
390
391 /* softcall_lock will be released by this routine */
392 (void) softcall_choose_cpu();
393 }
394 }
395
396 void
kdi_softcall(void (* func)(void))397 kdi_softcall(void (*func)(void))
398 {
399 kdi_softcall_func = func;
400
401 if (softhead == NULL)
402 kdi_siron();
403 }
404
405 /*
406 * Called to process software interrupts take one off queue, call it,
407 * repeat.
408 *
409 * Note queue may change during call; softcall_lock, state variables
410 * softcall_state and softcall_latest_cpuid ensures that -
411 * - we don't have multiple cpus pulling from the list (thus causing
412 * a violation of FIFO order with an exception when we are stuck).
413 * - we don't miss a new entry having been added to the head.
414 * - we don't miss a wakeup.
415 */
416
417 void
softint(void)418 softint(void)
419 {
420 softcall_t *sc = NULL;
421 void (*func)();
422 caddr_t arg;
423 int cpu_id = CPU->cpu_id;
424
425 /*
426 * Don't process softcall queue if current CPU is quiesced or
427 * offlined. This can happen when a CPU is running pause
428 * thread but softcall already sent a xcall.
429 */
430 if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) {
431 if (softcall_cpuset != NULL &&
432 CPU_IN_SET(*softcall_cpuset, cpu_id)) {
433 CPUSET_DEL(*softcall_cpuset, cpu_id);
434 goto out;
435 }
436 }
437
438 mutex_enter(&softcall_lock);
439
440 if (softcall_state & (SOFT_STEAL|SOFT_PEND)) {
441 softcall_state = SOFT_DRAIN;
442 } else {
443 /*
444 * The check for softcall_cpuset being
445 * NULL is required because it may get
446 * called very early during boot.
447 */
448 if (softcall_cpuset != NULL &&
449 CPU_IN_SET(*softcall_cpuset, cpu_id))
450 CPUSET_DEL(*softcall_cpuset, cpu_id);
451 mutex_exit(&softcall_lock);
452 goto out;
453 }
454
455 /*
456 * Setting softcall_latest_cpuid to current CPU ensures
457 * that there is only one active softlevel1 handler to
458 * process softcall queues.
459 *
460 * Since softcall_lock lock is dropped before calling
461 * func (callback), we need softcall_latest_cpuid
462 * to prevent two softlevel1 hanlders working on the
463 * queue when the first softlevel1 handler gets
464 * stuck due to high interrupt load.
465 */
466 softcall_latest_cpuid = cpu_id;
467
468 /* add ourself to the cpuset */
469 if (!CPU_IN_SET(*softcall_cpuset, cpu_id))
470 CPUSET_ADD(*softcall_cpuset, cpu_id);
471
472 for (;;) {
473 softcall_tick = ddi_get_lbolt();
474 if ((sc = softhead) != NULL) {
475 func = sc->sc_func;
476 arg = sc->sc_arg;
477 softhead = sc->sc_next;
478 sc->sc_next = softfree;
479 softfree = sc;
480 }
481
482 if (sc == NULL) {
483 if (CPU_IN_SET(*softcall_cpuset, cpu_id))
484 CPUSET_DEL(*softcall_cpuset, cpu_id);
485
486 softcall_state = SOFT_IDLE;
487 ASSERT(softcall_latest_cpuid == cpu_id);
488 softcall_latest_cpuid = -1;
489
490 mutex_exit(&softcall_lock);
491 break;
492 }
493
494 mutex_exit(&softcall_lock);
495 func(arg);
496 mutex_enter(&softcall_lock);
497
498 /*
499 * No longer need softcall processing from current
500 * interrupt handler because either
501 * (a) softcall is in SOFT_IDLE state or
502 * (b) There is a CPU already draining softcall
503 * queue and the current softlevel1 is no
504 * longer required.
505 */
506 if (softcall_latest_cpuid != cpu_id) {
507 if (CPU_IN_SET(*softcall_cpuset, cpu_id))
508 CPUSET_DEL(*softcall_cpuset, cpu_id);
509
510 mutex_exit(&softcall_lock);
511 break;
512 }
513 }
514
515 out:
516 if ((func = kdi_softcall_func) != NULL) {
517 kdi_softcall_func = NULL;
518 func();
519 }
520 }
521