1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/systm.h>
27 #include <sys/archsystm.h>
28 #include <sys/machsystm.h>
29 #include <sys/cpuvar.h>
30 #include <sys/intreg.h>
31 #include <sys/x_call.h>
32 #include <sys/cmn_err.h>
33 #include <sys/membar.h>
34 #include <sys/disp.h>
35 #include <sys/debug.h>
36 #include <sys/privregs.h>
37 #include <sys/xc_impl.h>
38 #include <sys/ivintr.h>
39 #include <sys/dmv.h>
40 #include <sys/sysmacros.h>
41
42 #ifdef TRAPTRACE
43 uint_t x_dstat[NCPU][XC_LOOP_EXIT+1];
44 uint_t x_rstat[NCPU][4];
45 #endif /* TRAPTRACE */
46
47 static uint64_t xc_serv_inum; /* software interrupt number for xc_serv() */
48 static uint64_t xc_loop_inum; /* software interrupt number for xc_loop() */
49 kmutex_t xc_sys_mutex; /* protect xcall session and xc_mbox */
50 int xc_spl_enter[NCPU]; /* protect sending x-call */
51 static int xc_holder = -1; /* the cpu who initiates xc_attention, 0 is valid */
52
53 /*
54 * Mail box for handshaking and xcall request; protected by xc_sys_mutex
55 */
56 static struct xc_mbox {
57 xcfunc_t *xc_func;
58 uint64_t xc_arg1;
59 uint64_t xc_arg2;
60 cpuset_t xc_cpuset;
61 volatile uint_t xc_state;
62 } xc_mbox[NCPU];
63
64 uint64_t xc_tick_limit; /* send_mondo() tick limit value */
65 uint64_t xc_tick_limit_scale = 1; /* scale used to increase the limit */
66 uint64_t xc_tick_jump_limit; /* send_mondo() irregular tick jump limit */
67 uint64_t xc_sync_tick_limit; /* timeout limit for xt_sync() calls */
68
69 /* timeout value for xcalls to be received by the target CPU */
70 uint64_t xc_mondo_time_limit;
71
72 /* timeout value for xcall functions to be executed on the target CPU */
73 uint64_t xc_func_time_limit;
74
75 uint64_t xc_scale = 1; /* scale used to calculate timeout limits */
76 uint64_t xc_mondo_multiplier = 10;
77
78 uint_t sendmondo_in_recover;
79
80 /*
81 * sending x-calls
82 */
83 void init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2);
84 void send_one_mondo(int cpuid);
85 void send_mondo_set(cpuset_t set);
86
87 /*
88 * Adjust xc_attention timeout if a faster cpu is dynamically added.
89 * Ignore the dynamic removal of a cpu that would lower these timeout
90 * values.
91 */
92 static int
xc_func_timeout_adj(cpu_setup_t what,int cpuid)93 xc_func_timeout_adj(cpu_setup_t what, int cpuid)
94 {
95 uint64_t freq = cpunodes[cpuid].clock_freq;
96
97 switch (what) {
98 case CPU_ON:
99 case CPU_INIT:
100 case CPU_CONFIG:
101 case CPU_CPUPART_IN:
102 if (freq * xc_scale > xc_mondo_time_limit) {
103 xc_mondo_time_limit = freq * xc_scale;
104 xc_func_time_limit = xc_mondo_time_limit *
105 xc_mondo_multiplier;
106 }
107 break;
108 case CPU_OFF:
109 case CPU_UNCONFIG:
110 case CPU_CPUPART_OUT:
111 default:
112 break;
113 }
114
115 return (0);
116 }
117
118 /*
119 * xc_init - initialize x-call related locks
120 */
121 void
xc_init(void)122 xc_init(void)
123 {
124 int pix;
125 uint64_t maxfreq = 0;
126
127 mutex_init(&xc_sys_mutex, NULL, MUTEX_SPIN,
128 (void *)ipltospl(XCALL_PIL));
129
130 #ifdef TRAPTRACE
131 /* Initialize for all possible CPUs. */
132 for (pix = 0; pix < NCPU; pix++) {
133 XC_STAT_INIT(pix);
134 }
135 #endif /* TRAPTRACE */
136
137 xc_serv_inum = add_softintr(XCALL_PIL, (softintrfunc)xc_serv, 0,
138 SOFTINT_MT);
139 xc_loop_inum = add_softintr(XCALL_PIL, (softintrfunc)xc_loop, 0,
140 SOFTINT_MT);
141
142 /*
143 * Initialize the calibrated tick limit for send_mondo.
144 * The value represents the maximum tick count to wait.
145 */
146 xc_tick_limit =
147 ((uint64_t)sys_tick_freq * XC_SEND_MONDO_MSEC) / 1000;
148 xc_tick_jump_limit = xc_tick_limit / 32;
149 xc_tick_limit *= xc_tick_limit_scale;
150 xc_sync_tick_limit = xc_tick_limit;
151
152 /*
153 * Maximum number of loops to wait before timing out in xc_attention.
154 */
155 for (pix = 0; pix < NCPU; pix++) {
156 maxfreq = MAX(cpunodes[pix].clock_freq, maxfreq);
157 }
158 xc_mondo_time_limit = maxfreq * xc_scale;
159 register_cpu_setup_func((cpu_setup_func_t *)xc_func_timeout_adj, NULL);
160
161 /*
162 * Maximum number of loops to wait for a xcall function to be
163 * executed on the target CPU.
164 */
165 xc_func_time_limit = xc_mondo_time_limit * xc_mondo_multiplier;
166 }
167
168 /*
169 * The following routines basically provide callers with two kinds of
170 * inter-processor interrupt services:
171 * 1. cross calls (x-calls) - requests are handled at target cpu's TL=0
172 * 2. cross traps (c-traps) - requests are handled at target cpu's TL>0
173 *
174 * Although these routines protect the services from migrating to other cpus
175 * "after" they are called, it is the caller's choice or responsibility to
176 * prevent the cpu migration "before" calling them.
177 *
178 * X-call routines:
179 *
180 * xc_one() - send a request to one processor
181 * xc_some() - send a request to some processors
182 * xc_all() - send a request to all processors
183 *
184 * Their common parameters:
185 * func - a TL=0 handler address
186 * arg1 and arg2 - optional
187 *
188 * The services provided by x-call routines allow callers
189 * to send a request to target cpus to execute a TL=0
190 * handler.
191 * The interface of the registers of the TL=0 handler:
192 * %o0: arg1
193 * %o1: arg2
194 *
195 * X-trap routines:
196 *
197 * xt_one() - send a request to one processor
198 * xt_some() - send a request to some processors
199 * xt_all() - send a request to all processors
200 *
201 * Their common parameters:
202 * func - a TL>0 handler address or an interrupt number
203 * arg1, arg2
204 * optional when "func" is an address;
205 * 0 when "func" is an interrupt number
206 *
207 * If the request of "func" is a kernel address, then
208 * the target cpu will execute the request of "func" with
209 * args at "TL>0" level.
210 * The interface of the registers of the TL>0 handler:
211 * %g1: arg1
212 * %g2: arg2
213 *
214 * If the request of "func" is not a kernel address, then it has
215 * to be an assigned interrupt number through add_softintr().
216 * An interrupt number is an index to the interrupt vector table,
217 * which entry contains an interrupt handler address with its
218 * corresponding interrupt level and argument.
219 * The target cpu will arrange the request to be serviced according
220 * to its pre-registered information.
221 * args are assumed to be zeros in this case.
222 *
223 * In addition, callers are allowed to capture and release cpus by
224 * calling the routines: xc_attention() and xc_dismissed().
225 */
226
227 /*
228 * spl_xcall - set PIL to xcall level
229 */
230 int
spl_xcall(void)231 spl_xcall(void)
232 {
233 return (splr(XCALL_PIL));
234 }
235
236 /*
237 * xt_one - send a "x-trap" to a cpu
238 */
239 void
xt_one(int cix,xcfunc_t * func,uint64_t arg1,uint64_t arg2)240 xt_one(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
241 {
242 if (!CPU_IN_SET(cpu_ready_set, cix)) {
243 return;
244 }
245 xt_one_unchecked(cix, func, arg1, arg2);
246 }
247
248 /*
249 * xt_one_unchecked - send a "x-trap" to a cpu without checking for its
250 * existance in cpu_ready_set
251 */
252 void
xt_one_unchecked(int cix,xcfunc_t * func,uint64_t arg1,uint64_t arg2)253 xt_one_unchecked(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
254 {
255 int lcx;
256 int opl;
257 cpuset_t tset;
258
259 /*
260 * Make sure the function address will not be interpreted as a
261 * dmv interrupt
262 */
263 ASSERT(!DMV_IS_DMV(func));
264
265 /*
266 * It's illegal to send software inums through the cross-trap
267 * interface.
268 */
269 ASSERT((uintptr_t)func >= KERNELBASE);
270
271 CPUSET_ZERO(tset);
272
273 XC_SPL_ENTER(lcx, opl); /* lcx set by the macro */
274
275 CPUSET_ADD(tset, cix);
276
277 if (cix == lcx) {
278 /*
279 * same cpu - use software fast trap
280 */
281 send_self_xcall(CPU, arg1, arg2, func);
282 XC_STAT_INC(x_dstat[lcx][XT_ONE_SELF]);
283 XC_TRACE(XT_ONE_SELF, &tset, func, arg1, arg2);
284 } else { /* other cpu - send a mondo to the target cpu */
285 /*
286 * other cpu - send a mondo to the target cpu
287 */
288 XC_TRACE(XT_ONE_OTHER, &tset, func, arg1, arg2);
289 init_mondo(func, arg1, arg2);
290 send_one_mondo(cix);
291 XC_STAT_INC(x_dstat[lcx][XT_ONE_OTHER]);
292 }
293 XC_SPL_EXIT(lcx, opl);
294 }
295
296 /*
297 * xt_some - send a "x-trap" to some cpus
298 */
299 void
xt_some(cpuset_t cpuset,xcfunc_t * func,uint64_t arg1,uint64_t arg2)300 xt_some(cpuset_t cpuset, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
301 {
302 int lcx;
303 int opl;
304 cpuset_t xc_cpuset, tset;
305
306 /*
307 * Make sure the function address will not be interpreted as a
308 * dmv interrupt
309 */
310 ASSERT(!DMV_IS_DMV(func));
311
312 /*
313 * It's illegal to send software inums through the cross-trap
314 * interface.
315 */
316 ASSERT((uintptr_t)func >= KERNELBASE);
317
318 CPUSET_ZERO(tset);
319
320 XC_SPL_ENTER(lcx, opl); /* lcx set by the macro */
321
322 CPUSET_ADD(tset, lcx);
323
324 /*
325 * only send to the CPU_READY ones
326 */
327 xc_cpuset = cpu_ready_set;
328 CPUSET_AND(xc_cpuset, cpuset);
329
330 /*
331 * send to nobody; just return
332 */
333 if (CPUSET_ISNULL(xc_cpuset)) {
334 XC_SPL_EXIT(lcx, opl);
335 return;
336 }
337
338 /*
339 * don't send mondo to self
340 */
341 if (CPU_IN_SET(xc_cpuset, lcx)) {
342 /*
343 * same cpu - use software fast trap
344 */
345 send_self_xcall(CPU, arg1, arg2, func);
346 XC_STAT_INC(x_dstat[lcx][XT_SOME_SELF]);
347 XC_TRACE(XT_SOME_SELF, &tset, func, arg1, arg2);
348 CPUSET_DEL(xc_cpuset, lcx);
349 if (CPUSET_ISNULL(xc_cpuset)) {
350 XC_SPL_EXIT(lcx, opl);
351 return;
352 }
353 }
354 XC_TRACE(XT_SOME_OTHER, &xc_cpuset, func, arg1, arg2);
355 init_mondo(func, arg1, arg2);
356 send_mondo_set(xc_cpuset);
357 XC_STAT_INC(x_dstat[lcx][XT_SOME_OTHER]);
358
359 XC_SPL_EXIT(lcx, opl);
360 }
361
362 /*
363 * xt_all - send a "x-trap" to all cpus
364 */
365 void
xt_all(xcfunc_t * func,uint64_t arg1,uint64_t arg2)366 xt_all(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
367 {
368 int lcx;
369 int opl;
370 cpuset_t xc_cpuset, tset;
371
372 /*
373 * Make sure the function address will not be interpreted as a
374 * dmv interrupt
375 */
376 ASSERT(!DMV_IS_DMV(func));
377
378 /*
379 * It's illegal to send software inums through the cross-trap
380 * interface.
381 */
382 ASSERT((uintptr_t)func >= KERNELBASE);
383
384 CPUSET_ZERO(tset);
385
386 XC_SPL_ENTER(lcx, opl); /* lcx set by the macro */
387
388 CPUSET_ADD(tset, lcx);
389
390 /*
391 * same cpu - use software fast trap
392 */
393 if (CPU_IN_SET(cpu_ready_set, lcx))
394 send_self_xcall(CPU, arg1, arg2, func);
395
396 XC_TRACE(XT_ALL_OTHER, &cpu_ready_set, func, arg1, arg2);
397
398 /*
399 * don't send mondo to self
400 */
401 xc_cpuset = cpu_ready_set;
402 CPUSET_DEL(xc_cpuset, lcx);
403
404 if (CPUSET_ISNULL(xc_cpuset)) {
405 XC_STAT_INC(x_dstat[lcx][XT_ALL_SELF]);
406 XC_TRACE(XT_ALL_SELF, &tset, func, arg1, arg2);
407 XC_SPL_EXIT(lcx, opl);
408 return;
409 }
410
411 init_mondo(func, arg1, arg2);
412 send_mondo_set(xc_cpuset);
413
414 XC_STAT_INC(x_dstat[lcx][XT_ALL_OTHER]);
415 XC_SPL_EXIT(lcx, opl);
416 }
417
418 /*
419 * xc_one - send a "x-call" to a cpu
420 */
421 void
xc_one(int cix,xcfunc_t * func,uint64_t arg1,uint64_t arg2)422 xc_one(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
423 {
424 int lcx;
425 int opl;
426 uint64_t loop_cnt = 0;
427 cpuset_t tset;
428 int first_time = 1;
429
430 /*
431 * send to nobody; just return
432 */
433 if (!CPU_IN_SET(cpu_ready_set, cix))
434 return;
435
436 ASSERT((uintptr_t)func > KERNELBASE);
437 ASSERT(((uintptr_t)func % PC_ALIGN) == 0);
438
439 CPUSET_ZERO(tset);
440
441 kpreempt_disable();
442
443 XC_SPL_ENTER(lcx, opl); /* lcx set by the macro */
444
445 CPUSET_ADD(tset, cix);
446
447 if (cix == lcx) { /* same cpu just do it */
448 XC_TRACE(XC_ONE_SELF, &tset, func, arg1, arg2);
449 (*func)(arg1, arg2);
450 XC_STAT_INC(x_dstat[lcx][XC_ONE_SELF]);
451 XC_SPL_EXIT(lcx, opl);
452 kpreempt_enable();
453 return;
454 }
455
456 if (xc_holder == lcx) { /* got the xc_sys_mutex already */
457 ASSERT(MUTEX_HELD(&xc_sys_mutex));
458 ASSERT(CPU_IN_SET(xc_mbox[lcx].xc_cpuset, lcx));
459 ASSERT(CPU_IN_SET(xc_mbox[cix].xc_cpuset, cix));
460 ASSERT(xc_mbox[cix].xc_state == XC_WAIT);
461 XC_TRACE(XC_ONE_OTHER_H, &tset, func, arg1, arg2);
462
463 /*
464 * target processor's xc_loop should be waiting
465 * for the work to do; just set up the xc_mbox
466 */
467 XC_SETUP(cix, func, arg1, arg2);
468 membar_stld();
469
470 while (xc_mbox[cix].xc_state != XC_WAIT) {
471 if (loop_cnt++ > xc_func_time_limit) {
472 if (sendmondo_in_recover) {
473 drv_usecwait(1);
474 loop_cnt = 0;
475 continue;
476 }
477 cmn_err(CE_PANIC, "xc_one() timeout, "
478 "xc_state[%d] != XC_WAIT", cix);
479 }
480 }
481 XC_STAT_INC(x_dstat[lcx][XC_ONE_OTHER_H]);
482 XC_SPL_EXIT(lcx, opl);
483 kpreempt_enable();
484 return;
485 }
486
487 /*
488 * Avoid dead lock if someone has sent us a xc_loop request while
489 * we are trying to grab xc_sys_mutex.
490 */
491 XC_SPL_EXIT(lcx, opl);
492
493 /*
494 * At this point, since we don't own xc_sys_mutex,
495 * our pil shouldn't run at or above the XCALL_PIL.
496 */
497 ASSERT(getpil() < XCALL_PIL);
498
499 /*
500 * Since xc_holder is not owned by us, it could be that
501 * no one owns it, or we are not informed to enter into
502 * xc_loop(). In either case, we need to grab the
503 * xc_sys_mutex before we write to the xc_mbox, and
504 * we shouldn't release it until the request is finished.
505 */
506
507 mutex_enter(&xc_sys_mutex);
508 xc_spl_enter[lcx] = 1;
509
510 /*
511 * Since we own xc_sys_mutex now, we are safe to
512 * write to the xc_mbox.
513 */
514 ASSERT(xc_mbox[cix].xc_state == XC_IDLE);
515 XC_TRACE(XC_ONE_OTHER, &tset, func, arg1, arg2);
516 XC_SETUP(cix, func, arg1, arg2);
517 init_mondo(setsoftint_tl1, xc_serv_inum, 0);
518 send_one_mondo(cix);
519 xc_spl_enter[lcx] = 0;
520
521 /* xc_serv does membar_stld */
522 while (xc_mbox[cix].xc_state != XC_IDLE) {
523 if (loop_cnt++ > xc_func_time_limit) {
524 if (sendmondo_in_recover) {
525 drv_usecwait(1);
526 loop_cnt = 0;
527 continue;
528 }
529 if (first_time) {
530 XT_SYNC_ONE(cix);
531 first_time = 0;
532 loop_cnt = 0;
533 continue;
534 }
535 cmn_err(CE_PANIC, "xc_one() timeout, "
536 "xc_state[%d] != XC_IDLE", cix);
537 }
538 }
539 XC_STAT_INC(x_dstat[lcx][XC_ONE_OTHER]);
540 mutex_exit(&xc_sys_mutex);
541
542 kpreempt_enable();
543 }
544
545 /*
546 * xc_some - send a "x-call" to some cpus; sending to self is excluded
547 */
548 void
xc_some(cpuset_t cpuset,xcfunc_t * func,uint64_t arg1,uint64_t arg2)549 xc_some(cpuset_t cpuset, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
550 {
551 int lcx;
552 int opl;
553 cpuset_t xc_cpuset, tset;
554
555 ASSERT((uintptr_t)func > KERNELBASE);
556 ASSERT(((uintptr_t)func % PC_ALIGN) == 0);
557
558 CPUSET_ZERO(tset);
559
560 kpreempt_disable();
561 XC_SPL_ENTER(lcx, opl); /* lcx set by the macro */
562
563 CPUSET_ADD(tset, lcx);
564
565 /*
566 * only send to the CPU_READY ones
567 */
568 xc_cpuset = cpu_ready_set;
569 CPUSET_AND(xc_cpuset, cpuset);
570
571 /*
572 * send to nobody; just return
573 */
574 if (CPUSET_ISNULL(xc_cpuset)) {
575 XC_SPL_EXIT(lcx, opl);
576 kpreempt_enable();
577 return;
578 }
579
580 if (CPU_IN_SET(xc_cpuset, lcx)) {
581 /*
582 * same cpu just do it
583 */
584 (*func)(arg1, arg2);
585 CPUSET_DEL(xc_cpuset, lcx);
586 if (CPUSET_ISNULL(xc_cpuset)) {
587 XC_STAT_INC(x_dstat[lcx][XC_SOME_SELF]);
588 XC_TRACE(XC_SOME_SELF, &tset, func, arg1, arg2);
589 XC_SPL_EXIT(lcx, opl);
590 kpreempt_enable();
591 return;
592 }
593 }
594
595 if (xc_holder == lcx) { /* got the xc_sys_mutex already */
596 cpuset_t mset = xc_mbox[lcx].xc_cpuset;
597
598 CPUSET_AND(mset, cpuset);
599 ASSERT(MUTEX_HELD(&xc_sys_mutex));
600 ASSERT(CPUSET_ISEQUAL(mset, cpuset));
601 SEND_MBOX_ONLY(xc_cpuset, func, arg1, arg2, lcx, XC_WAIT);
602 WAIT_MBOX_DONE(xc_cpuset, lcx, XC_WAIT, 0);
603 XC_STAT_INC(x_dstat[lcx][XC_SOME_OTHER_H]);
604 XC_TRACE(XC_SOME_OTHER_H, &xc_cpuset, func, arg1, arg2);
605 XC_SPL_EXIT(lcx, opl);
606 kpreempt_enable();
607 return;
608 }
609
610 /*
611 * Avoid dead lock if someone has sent us a xc_loop request while
612 * we are trying to grab xc_sys_mutex.
613 */
614 XC_SPL_EXIT(lcx, opl);
615
616 /*
617 * At this point, since we don't own xc_sys_mutex,
618 * our pil shouldn't run at or above the XCALL_PIL.
619 */
620 ASSERT(getpil() < XCALL_PIL);
621
622 /*
623 * grab xc_sys_mutex before writing to the xc_mbox
624 */
625 mutex_enter(&xc_sys_mutex);
626 xc_spl_enter[lcx] = 1;
627
628 XC_TRACE(XC_SOME_OTHER, &xc_cpuset, func, arg1, arg2);
629 init_mondo(setsoftint_tl1, xc_serv_inum, 0);
630 SEND_MBOX_MONDO(xc_cpuset, func, arg1, arg2, XC_IDLE);
631 WAIT_MBOX_DONE(xc_cpuset, lcx, XC_IDLE, 1);
632
633 xc_spl_enter[lcx] = 0;
634 XC_STAT_INC(x_dstat[lcx][XC_SOME_OTHER]);
635 mutex_exit(&xc_sys_mutex);
636 kpreempt_enable();
637 }
638
639 /*
640 * xc_all - send a "x-call" to all cpus
641 */
642 void
xc_all(xcfunc_t * func,uint64_t arg1,uint64_t arg2)643 xc_all(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
644 {
645 int lcx;
646 int opl;
647 cpuset_t xc_cpuset, tset;
648
649 ASSERT((uintptr_t)func > KERNELBASE);
650 ASSERT(((uintptr_t)func % PC_ALIGN) == 0);
651
652 CPUSET_ZERO(tset);
653
654 kpreempt_disable();
655 XC_SPL_ENTER(lcx, opl); /* lcx set by the macro */
656
657 CPUSET_ADD(tset, lcx);
658
659 /*
660 * same cpu just do it
661 */
662 (*func)(arg1, arg2);
663 xc_cpuset = cpu_ready_set;
664 CPUSET_DEL(xc_cpuset, lcx);
665
666 if (CPUSET_ISNULL(xc_cpuset)) {
667 XC_STAT_INC(x_dstat[lcx][XC_ALL_SELF]);
668 XC_TRACE(XC_ALL_SELF, &tset, func, arg1, arg2);
669 XC_SPL_EXIT(lcx, opl);
670 kpreempt_enable();
671 return;
672 }
673
674 if (xc_holder == lcx) { /* got the xc_sys_mutex already */
675 cpuset_t mset = xc_mbox[lcx].xc_cpuset;
676
677 CPUSET_AND(mset, xc_cpuset);
678 ASSERT(MUTEX_HELD(&xc_sys_mutex));
679 ASSERT(CPUSET_ISEQUAL(mset, xc_cpuset));
680 XC_TRACE(XC_ALL_OTHER_H, &xc_cpuset, func, arg1, arg2);
681 SEND_MBOX_ONLY(xc_cpuset, func, arg1, arg2, lcx, XC_WAIT);
682 WAIT_MBOX_DONE(xc_cpuset, lcx, XC_WAIT, 0);
683 XC_STAT_INC(x_dstat[lcx][XC_ALL_OTHER_H]);
684 XC_SPL_EXIT(lcx, opl);
685 kpreempt_enable();
686 return;
687 }
688
689 /*
690 * Avoid dead lock if someone has sent us a xc_loop request while
691 * we are trying to grab xc_sys_mutex.
692 */
693 XC_SPL_EXIT(lcx, opl);
694
695 /*
696 * At this point, since we don't own xc_sys_mutex,
697 * our pil shouldn't run at or above the XCALL_PIL.
698 */
699 ASSERT(getpil() < XCALL_PIL);
700
701 /*
702 * grab xc_sys_mutex before writing to the xc_mbox
703 */
704 mutex_enter(&xc_sys_mutex);
705 xc_spl_enter[lcx] = 1;
706
707 XC_TRACE(XC_ALL_OTHER, &xc_cpuset, func, arg1, arg2);
708 init_mondo(setsoftint_tl1, xc_serv_inum, 0);
709 SEND_MBOX_MONDO(xc_cpuset, func, arg1, arg2, XC_IDLE);
710 WAIT_MBOX_DONE(xc_cpuset, lcx, XC_IDLE, 1);
711
712 xc_spl_enter[lcx] = 0;
713 XC_STAT_INC(x_dstat[lcx][XC_ALL_OTHER]);
714 mutex_exit(&xc_sys_mutex);
715 kpreempt_enable();
716 }
717
718 /*
719 * xc_attention - paired with xc_dismissed()
720 *
721 * xt_attention() holds the xc_sys_mutex and xc_dismissed() releases it
722 * called when an initiator wants to capture some/all cpus for a critical
723 * session.
724 */
725 void
xc_attention(cpuset_t cpuset)726 xc_attention(cpuset_t cpuset)
727 {
728 int pix, lcx;
729 cpuset_t xc_cpuset, tmpset;
730 cpuset_t recv_cpuset;
731 uint64_t loop_cnt = 0;
732 int first_time = 1;
733
734 CPUSET_ZERO(recv_cpuset);
735
736 /*
737 * don't migrate the cpu until xc_dismissed() is finished
738 */
739 ASSERT(getpil() < XCALL_PIL);
740 mutex_enter(&xc_sys_mutex);
741 lcx = (int)(CPU->cpu_id);
742 ASSERT(x_dstat[lcx][XC_ATTENTION] ==
743 x_dstat[lcx][XC_DISMISSED]);
744 ASSERT(xc_holder == -1);
745 xc_mbox[lcx].xc_cpuset = cpuset;
746 xc_holder = lcx; /* no membar; only current cpu needs the right lcx */
747
748 /*
749 * only send to the CPU_READY ones
750 */
751 xc_cpuset = cpu_ready_set;
752 CPUSET_AND(xc_cpuset, cpuset);
753
754 /*
755 * don't send mondo to self
756 */
757 CPUSET_DEL(xc_cpuset, lcx);
758
759 XC_STAT_INC(x_dstat[lcx][XC_ATTENTION]);
760 XC_TRACE(XC_ATTENTION, &xc_cpuset, NULL, 0, 0);
761
762 if (CPUSET_ISNULL(xc_cpuset))
763 return;
764
765 xc_spl_enter[lcx] = 1;
766 /*
767 * inform the target processors to enter into xc_loop()
768 */
769 init_mondo(setsoftint_tl1, xc_loop_inum, 0);
770 SEND_MBOX_MONDO_XC_ENTER(xc_cpuset);
771 xc_spl_enter[lcx] = 0;
772
773 /*
774 * make sure target processors have entered into xc_loop()
775 */
776 while (!CPUSET_ISEQUAL(recv_cpuset, xc_cpuset)) {
777 tmpset = xc_cpuset;
778 for (pix = 0; pix < NCPU; pix++) {
779 if (CPU_IN_SET(tmpset, pix)) {
780 /*
781 * membar_stld() is done in xc_loop
782 */
783 if (xc_mbox[pix].xc_state == XC_WAIT) {
784 CPUSET_ADD(recv_cpuset, pix);
785 }
786 CPUSET_DEL(tmpset, pix);
787 if (CPUSET_ISNULL(tmpset)) {
788 break;
789 }
790 }
791 }
792 if (loop_cnt++ > xc_mondo_time_limit) {
793 if (sendmondo_in_recover) {
794 drv_usecwait(1);
795 loop_cnt = 0;
796 continue;
797 }
798 if (first_time) {
799 XT_SYNC_SOME(xc_cpuset);
800 first_time = 0;
801 loop_cnt = 0;
802 continue;
803 }
804 cmn_err(CE_PANIC, "xc_attention() timeout");
805 }
806 }
807
808 /*
809 * xc_sys_mutex remains held until xc_dismissed() is finished
810 */
811 }
812
813 /*
814 * xc_dismissed - paired with xc_attention()
815 *
816 * Called after the critical session is finished.
817 */
818 void
xc_dismissed(cpuset_t cpuset)819 xc_dismissed(cpuset_t cpuset)
820 {
821 int pix;
822 int lcx = (int)(CPU->cpu_id);
823 cpuset_t xc_cpuset, tmpset;
824 cpuset_t recv_cpuset;
825 uint64_t loop_cnt = 0;
826
827 ASSERT(lcx == xc_holder);
828 ASSERT(CPUSET_ISEQUAL(xc_mbox[lcx].xc_cpuset, cpuset));
829 ASSERT(getpil() >= XCALL_PIL);
830 CPUSET_ZERO(xc_mbox[lcx].xc_cpuset);
831 CPUSET_ZERO(recv_cpuset);
832 membar_stld();
833
834 XC_STAT_INC(x_dstat[lcx][XC_DISMISSED]);
835 ASSERT(x_dstat[lcx][XC_DISMISSED] == x_dstat[lcx][XC_ATTENTION]);
836
837 /*
838 * only send to the CPU_READY ones
839 */
840 xc_cpuset = cpu_ready_set;
841 CPUSET_AND(xc_cpuset, cpuset);
842
843 /*
844 * exclude itself
845 */
846 CPUSET_DEL(xc_cpuset, lcx);
847 XC_TRACE(XC_DISMISSED, &xc_cpuset, NULL, 0, 0);
848 if (CPUSET_ISNULL(xc_cpuset)) {
849 xc_holder = -1;
850 mutex_exit(&xc_sys_mutex);
851 return;
852 }
853
854 /*
855 * inform other processors to get out of xc_loop()
856 */
857 tmpset = xc_cpuset;
858 for (pix = 0; pix < NCPU; pix++) {
859 if (CPU_IN_SET(tmpset, pix)) {
860 xc_mbox[pix].xc_state = XC_EXIT;
861 membar_stld();
862 CPUSET_DEL(tmpset, pix);
863 if (CPUSET_ISNULL(tmpset)) {
864 break;
865 }
866 }
867 }
868
869 /*
870 * make sure target processors have exited from xc_loop()
871 */
872 while (!CPUSET_ISEQUAL(recv_cpuset, xc_cpuset)) {
873 tmpset = xc_cpuset;
874 for (pix = 0; pix < NCPU; pix++) {
875 if (CPU_IN_SET(tmpset, pix)) {
876 /*
877 * membar_stld() is done in xc_loop
878 */
879 if (xc_mbox[pix].xc_state == XC_IDLE) {
880 CPUSET_ADD(recv_cpuset, pix);
881 }
882 CPUSET_DEL(tmpset, pix);
883 if (CPUSET_ISNULL(tmpset)) {
884 break;
885 }
886 }
887 }
888 if (loop_cnt++ > xc_func_time_limit) {
889 if (sendmondo_in_recover) {
890 drv_usecwait(1);
891 loop_cnt = 0;
892 continue;
893 }
894 cmn_err(CE_PANIC, "xc_dismissed() timeout");
895 }
896 }
897 xc_holder = -1;
898 mutex_exit(&xc_sys_mutex);
899 }
900
901 /*
902 * xc_serv - "x-call" handler at TL=0; serves only one x-call request
903 * runs at XCALL_PIL level.
904 */
905 uint_t
xc_serv(void)906 xc_serv(void)
907 {
908 int lcx = (int)(CPU->cpu_id);
909 struct xc_mbox *xmp;
910 xcfunc_t *func;
911 uint64_t arg1, arg2;
912 cpuset_t tset;
913
914 ASSERT(getpil() == XCALL_PIL);
915 CPUSET_ZERO(tset);
916 CPUSET_ADD(tset, lcx);
917 flush_windows();
918 xmp = &xc_mbox[lcx];
919 ASSERT(lcx != xc_holder);
920 ASSERT(xmp->xc_state == XC_DOIT);
921 func = xmp->xc_func;
922 XC_TRACE(XC_SERV, &tset, func, xmp->xc_arg1, xmp->xc_arg2);
923 if (func != NULL) {
924 arg1 = xmp->xc_arg1;
925 arg2 = xmp->xc_arg2;
926 (*func)(arg1, arg2);
927 }
928 XC_STAT_INC(x_rstat[lcx][XC_SERV]);
929 XC_TRACE(XC_SERV, &tset, func, arg1, arg2);
930 xmp->xc_state = XC_IDLE;
931 membar_stld();
932 return (1);
933 }
934
935 /*
936 * if == 1, an xc_loop timeout will cause a panic
937 * otherwise print a warning
938 */
939 uint_t xc_loop_panic = 0;
940
941 /*
942 * xc_loop - "x-call" handler at TL=0; capture the cpu for a critial
943 * session, or serve multiple x-call requests runs at XCALL_PIL level.
944 */
945 uint_t
xc_loop(void)946 xc_loop(void)
947 {
948 int lcx = (int)(CPU->cpu_id);
949 struct xc_mbox *xmp;
950 xcfunc_t *func;
951 uint64_t arg1, arg2;
952 uint64_t loop_cnt = 0;
953 cpuset_t tset;
954
955 ASSERT(getpil() == XCALL_PIL);
956
957 CPUSET_ZERO(tset);
958 flush_windows();
959
960 /*
961 * Some one must have owned the xc_sys_mutex;
962 * no further interrupt (at XCALL_PIL or below) can
963 * be taken by this processor until xc_loop exits.
964 *
965 * The owner of xc_sys_mutex (or xc_holder) can expect
966 * its xc/xt requests are handled as follows:
967 * xc requests use xc_mbox's handshaking for their services
968 * xt requests at TL>0 will be handled immediately
969 * xt requests at TL=0:
970 * if their handlers'pils are <= XCALL_PIL, then
971 * they will be handled after xc_loop exits
972 * (so, they probably should not be used)
973 * else they will be handled immediately
974 *
975 * For those who are not informed to enter xc_loop, if they
976 * send xc/xt requests to this processor at this moment,
977 * the requests will be handled as follows:
978 * xc requests will be handled after they grab xc_sys_mutex
979 * xt requests at TL>0 will be handled immediately
980 * xt requests at TL=0:
981 * if their handlers'pils are <= XCALL_PIL, then
982 * they will be handled after xc_loop exits
983 * else they will be handled immediately
984 */
985 xmp = &xc_mbox[lcx];
986 ASSERT(lcx != xc_holder);
987 ASSERT(xmp->xc_state == XC_ENTER);
988 xmp->xc_state = XC_WAIT;
989 CPUSET_ADD(tset, lcx);
990 membar_stld();
991 XC_STAT_INC(x_rstat[lcx][XC_LOOP]);
992 XC_TRACE(XC_LOOP_ENTER, &tset, NULL, 0, 0);
993 while (xmp->xc_state != XC_EXIT) {
994 if (xmp->xc_state == XC_DOIT) {
995 func = xmp->xc_func;
996 arg1 = xmp->xc_arg1;
997 arg2 = xmp->xc_arg2;
998 XC_TRACE(XC_LOOP_DOIT, &tset, func, arg1, arg2);
999 if (func != NULL)
1000 (*func)(arg1, arg2);
1001 xmp->xc_state = XC_WAIT;
1002 membar_stld();
1003 /*
1004 * reset the timeout counter
1005 * since some work was done
1006 */
1007 loop_cnt = 0;
1008 } else {
1009 /* patience is a virtue... */
1010 loop_cnt++;
1011 }
1012
1013 if (loop_cnt > xc_func_time_limit) {
1014 if (sendmondo_in_recover) {
1015 drv_usecwait(1);
1016 loop_cnt = 0;
1017 continue;
1018 }
1019 cmn_err(xc_loop_panic ? CE_PANIC : CE_WARN,
1020 "xc_loop() timeout");
1021 /*
1022 * if the above displayed a warning,
1023 * reset the timeout counter and be patient
1024 */
1025 loop_cnt = 0;
1026 }
1027 }
1028 ASSERT(xmp->xc_state == XC_EXIT);
1029 ASSERT(xc_holder != -1);
1030 XC_TRACE(XC_LOOP_EXIT, &tset, NULL, 0, 0);
1031 xmp->xc_state = XC_IDLE;
1032 membar_stld();
1033 return (1);
1034 }
1035