xref: /titanic_41/usr/src/uts/common/disp/fss.c (revision 7f02c26644541f89bc301b94cc6fa9bc1c1e85a1)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2013, Joyent, Inc. All rights reserved.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/cred.h>
31 #include <sys/proc.h>
32 #include <sys/strsubr.h>
33 #include <sys/priocntl.h>
34 #include <sys/class.h>
35 #include <sys/disp.h>
36 #include <sys/procset.h>
37 #include <sys/debug.h>
38 #include <sys/kmem.h>
39 #include <sys/errno.h>
40 #include <sys/systm.h>
41 #include <sys/schedctl.h>
42 #include <sys/vmsystm.h>
43 #include <sys/atomic.h>
44 #include <sys/project.h>
45 #include <sys/modctl.h>
46 #include <sys/fss.h>
47 #include <sys/fsspriocntl.h>
48 #include <sys/cpupart.h>
49 #include <sys/zone.h>
50 #include <vm/rm.h>
51 #include <vm/seg_kmem.h>
52 #include <sys/tnf_probe.h>
53 #include <sys/policy.h>
54 #include <sys/sdt.h>
55 #include <sys/cpucaps.h>
56 
57 /*
58  * The fair share scheduling class ensures that collections of processes
59  * (zones and projects) each get their configured share of CPU.  This is in
60  * contrast to the TS class which considers individual processes.
61  *
62  * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on
63  * projects using the project.cpu-shares rctl.  By default the value is 1
64  * and it can range from 0 - 64k.  A value of 0 means that processes in the
65  * collection will only get CPU resources when there are no other processes
66  * that need CPU. The cpu-share is used as one of the inputs to calculate a
67  * thread's "user-mode" priority (umdpri) for the scheduler.  The umdpri falls
68  * in the range 0-59.  FSS calculates other, internal, priorities which are not
69  * visible outside of the FSS class.
70  *
71  * The FSS class should approximate TS behavior when there are excess CPU
72  * resources.  When there is a backlog of runnable processes, then the share
73  * is used as input into the runnable process's priority calculation, where
74  * the final umdpri is used by the scheduler to determine when the process runs.
75  *
76  * Projects in a zone compete with each other for CPU time, receiving CPU
77  * allocation within a zone proportional to the project's share; at a higher
78  * level zones compete with each other, receiving allocation in a pset
79  * proportional to the zone's share.
80  *
81  * The FSS priority calculation consists of several parts.
82  *
83  * 1) Once per second the fss_update function runs. The first thing it does is
84  *    call fss_decay_usage. This function does three things.
85  *
86  * a) fss_decay_usage first decays the maxfsspri value for the pset.  This
87  *    value is used in the per-process priority calculation described in step
88  *    (2b).  The maxfsspri is decayed using the following formula:
89  *
90  *                      maxfsspri * fss_nice_decay[NZERO])
91  *        maxfsspri =  ------------------------------------
92  *                            FSS_DECAY_BASE
93  *
94  *
95  *     - NZERO is the default process priority (i.e. 20)
96  *
97  *    The fss_nice_decay array is a fixed set of values used to adjust the
98  *    decay rate of processes based on their nice value.  Entries in this
99  *    array are initialized in fss_init using the following formula:
100  *
101  *                        (FSS_DECAY_MAX - FSS_DECAY_MIN) * i
102  *       FSS_DECAY_MIN + -------------------------------------
103  *                               FSS_NICE_RANGE - 1
104  *
105  *     - FSS_DECAY_MIN is 82 = approximates 65% (82/128)
106  *     - FSS_DECAY_MAX is 108 = approximates 85% (108/128)
107  *     - FSS_NICE_RANGE is 40 (range is 0 - 39)
108  *
109  * b) The second thing fss_decay_usage does is update each project's "usage"
110  *    for the last second and then recalculates the project's "share usage".
111  *
112  *    The usage value is the recent CPU usage for all of the threads in the
113  *    project. It is decayed and updated this way:
114  *
115  *                  (usage * FSS_DECAY_USG)
116  *        usage =  ------------------------- + ticks;
117  *                       FSS_DECAY_BASE
118  *
119  *     - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide
120  *     - FSS_DECAY_USG is 96 - approximates 75% (96/128)
121  *     - ticks is updated whenever a process in this project is running
122  *       when the scheduler's tick processing fires. This is not a simple
123  *       counter, the values are based on the entries in the fss_nice_tick
124  *       array (see section 3 below). ticks is then reset to 0 so it can track
125  *       the next seconds worth of nice-adjusted time for the project.
126  *
127  * c) The third thing fss_decay_usage does is update each project's "share
128  *    usage" (shusage). This is the normalized usage value for the project and
129  *    is calculated this way:
130  *
131  *                pset_shares^2    zone_int_shares^2
132  *        usage * ------------- * ------------------
133  *                kpj_shares^2	   zone_ext_shares^2
134  *
135  *    - usage - see (1b) for more details
136  *    - pset_shares is the total of all *active* zone shares in the pset (by
137  *      default there is only one pset)
138  *    - kpj_shares is the individual project's share (project.cpu-shares rctl)
139  *    - zone_int_shares is the sum of shares of all active projects within the
140  *      zone (the zone-internal total)
141  *    - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl)
142  *
143  *    The shusage is used in step (2b) to calculate the thread's new internal
144  *    priority. A larger shusage value leads to a lower priority.
145  *
146  * 2) The fss_update function then calls fss_update_list to update the priority
147  *    of all threads. This does two things.
148  *
149  * a) First the thread's internal priority is decayed using the following
150  *    formula:
151  *
152  *                  fsspri * fss_nice_decay[nice_value])
153  *        fsspri =  ------------------------------------
154  *                            FSS_DECAY_BASE
155  *
156  *     - FSS_DECAY_BASE is 128 as described above
157  *
158  * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri
159  *    to update the user-mode priority (umdpri) of the runnable thread.
160  *    Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP)
161  *    are not updated at this time. The updated user-mode priority can cause
162  *    threads to change their position in the run queue.
163  *
164  *    The process's new internal fsspri is calculated using the following
165  *    formula. All runnable threads in the project will use the same shusage
166  *    and nrunnable values in their calculation.
167  *
168  *        fsspri += shusage * nrunnable * ticks
169  *
170  *     - shusage is the project's share usage, calculated in (1c)
171  *     - nrunnable is the number of runnable threads in the project
172  *     - ticks is the number of ticks this thread ran since the last fss_newpri
173  *       invocation.
174  *
175  *    Finally the process's new user-mode priority is calculated using the
176  *    following formula:
177  *
178  *                              (fsspri * umdprirange)
179  *        umdpri = maxumdpri - ------------------------
180  *                                    maxfsspri
181  *
182  *     - maxumdpri is MINCLSYSPRI - 1 (i.e. 59)
183  *     - umdprirange is maxumdpri - 1 (i.e. 58)
184  *     - maxfsspri is the largest fsspri seen so far, as we're iterating all
185  *       runnable processes
186  *
187  *    Thus, a higher internal priority (fsspri) leads to a lower user-mode
188  *    priority which means the thread runs less. The fsspri is higher when
189  *    the project's normalized share usage is higher, when the project has
190  *    more runnable threads, or when the thread has accumulated more run-time.
191  *
192  *    This code has various checks to ensure the resulting umdpri is in the
193  *    range 1-59.  See fss_newpri for more details.
194  *
195  * To reiterate, the above processing is performed once per second to recompute
196  * the runnable thread user-mode priorities.
197  *
198  * 3) The final major component in the priority calculation is the tick
199  *    processing which occurs on a thread that is running when the clock
200  *    calls fss_tick.
201  *
202  *    A thread can run continuously in user-land (compute-bound) for the
203  *    fss_quantum (see "dispadmin -c FSS -g" for the configurable properties).
204  *    The fss_quantum defaults to 11 (i.e. 11 ticks).
205  *
206  *    Once the quantum has been consumed, the thread will call fss_newpri to
207  *    recompute its umdpri priority, as described above in (2b). Threads that
208  *    were T_ONPROC at the one second interval when runnable thread priorities
209  *    were recalculated will have their umdpri priority recalculated when their
210  *    quanta expires.
211  *
212  *    To ensure that runnable threads within a project see the expected
213  *    round-robin behavior, there is a special case in fss_newpri for a thread
214  *    that has run for its quanta within the one second update interval.  See
215  *    the handling for the quanta_up parameter within fss_newpri.
216  *
217  *    Also of interest, the fss_tick code increments the project's tick value
218  *    using the fss_nice_tick array entry for the thread's nice value. The idea
219  *    behind the fss_nice_tick array is that the cost of a tick is lower at
220  *    positive nice values (so that it doesn't increase the project's usage
221  *    as much as normal) with a 50% drop at the maximum level and a 50%
222  *    increase at the minimum level. See (1b). The fss_nice_tick array is
223  *    initialized in fss_init using the following formula:
224  *
225  *         FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i)
226  *        --------------------------------------------------
227  *                          FSS_NICE_RANGE
228  *
229  *     - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0
230  *
231  * FSS Data Structures:
232  *
233  *                 fsszone
234  *                  -----           -----
235  *  -----          |     |         |     |
236  * |     |-------->|     |<------->|     |<---->...
237  * |     |          -----           -----
238  * |     |          ^    ^            ^
239  * |     |---       |     \            \
240  *  -----    |      |      \            \
241  * fsspset   |      |       \            \
242  *           |      |        \            \
243  *           |    -----       -----       -----
244  *            -->|     |<--->|     |<--->|     |
245  *               |     |     |     |     |     |
246  *                -----       -----       -----
247  *               fssproj
248  *
249  * That is, fsspsets contain a list of fsszone's that are currently active in
250  * the pset, and a list of fssproj's, corresponding to projects with runnable
251  * threads on the pset.  fssproj's in turn point to the fsszone which they
252  * are a member of.
253  *
254  * An fssproj_t is removed when there are no threads in it.
255  *
256  * An fsszone_t is removed when there are no projects with threads in it.
257  */
258 
259 static pri_t fss_init(id_t, int, classfuncs_t **);
260 
261 static struct sclass fss = {
262 	"FSS",
263 	fss_init,
264 	0
265 };
266 
267 extern struct mod_ops mod_schedops;
268 
269 /*
270  * Module linkage information for the kernel.
271  */
272 static struct modlsched modlsched = {
273 	&mod_schedops, "fair share scheduling class", &fss
274 };
275 
276 static struct modlinkage modlinkage = {
277 	MODREV_1, (void *)&modlsched, NULL
278 };
279 
280 #define	FSS_MAXUPRI	60
281 
282 /*
283  * The fssproc_t structures are kept in an array of circular doubly linked
284  * lists.  A hash on the thread pointer is used to determine which list each
285  * thread should be placed in.  Each list has a dummy "head" which is never
286  * removed, so the list is never empty.  fss_update traverses these lists to
287  * update the priorities of threads that have been waiting on the run queue.
288  */
289 #define	FSS_LISTS		16 /* number of lists, must be power of 2 */
290 #define	FSS_LIST_HASH(t)	(((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
291 #define	FSS_LIST_NEXT(i)	(((i) + 1) & (FSS_LISTS - 1))
292 
293 #define	FSS_LIST_INSERT(fssproc)				\
294 {								\
295 	int index = FSS_LIST_HASH(fssproc->fss_tp);		\
296 	kmutex_t *lockp = &fss_listlock[index];			\
297 	fssproc_t *headp = &fss_listhead[index];		\
298 	mutex_enter(lockp);					\
299 	fssproc->fss_next = headp->fss_next;			\
300 	fssproc->fss_prev = headp;				\
301 	headp->fss_next->fss_prev = fssproc;			\
302 	headp->fss_next = fssproc;				\
303 	mutex_exit(lockp);					\
304 }
305 
306 #define	FSS_LIST_DELETE(fssproc)				\
307 {								\
308 	int index = FSS_LIST_HASH(fssproc->fss_tp);		\
309 	kmutex_t *lockp = &fss_listlock[index];			\
310 	mutex_enter(lockp);					\
311 	fssproc->fss_prev->fss_next = fssproc->fss_next;	\
312 	fssproc->fss_next->fss_prev = fssproc->fss_prev;	\
313 	mutex_exit(lockp);					\
314 }
315 
316 #define	FSS_TICK_COST	1000	/* tick cost for threads with nice level = 0 */
317 
318 /*
319  * Decay rate percentages are based on n/128 rather than n/100 so  that
320  * calculations can avoid having to do an integer divide by 100 (divide
321  * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
322  *
323  * FSS_DECAY_MIN	=  83/128 ~= 65%
324  * FSS_DECAY_MAX	= 108/128 ~= 85%
325  * FSS_DECAY_USG	=  96/128 ~= 75%
326  */
327 #define	FSS_DECAY_MIN	83	/* fsspri decay pct for threads w/ nice -20 */
328 #define	FSS_DECAY_MAX	108	/* fsspri decay pct for threads w/ nice +19 */
329 #define	FSS_DECAY_USG	96	/* fssusage decay pct for projects */
330 #define	FSS_DECAY_BASE	128	/* base for decay percentages above */
331 
332 #define	FSS_NICE_MIN	0
333 #define	FSS_NICE_MAX	(2 * NZERO - 1)
334 #define	FSS_NICE_RANGE	(FSS_NICE_MAX - FSS_NICE_MIN + 1)
335 
336 static int	fss_nice_tick[FSS_NICE_RANGE];
337 static int	fss_nice_decay[FSS_NICE_RANGE];
338 
339 static pri_t	fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
340 static pri_t	fss_maxumdpri; /* maximum user mode fss priority */
341 static pri_t	fss_maxglobpri;	/* maximum global priority used by fss class */
342 static pri_t	fss_minglobpri;	/* minimum global priority */
343 
344 static fssproc_t fss_listhead[FSS_LISTS];
345 static kmutex_t	fss_listlock[FSS_LISTS];
346 
347 static fsspset_t *fsspsets;
348 static kmutex_t fsspsets_lock;	/* protects fsspsets */
349 
350 static id_t	fss_cid;
351 
352 static time_t	fss_minrun = 2;	/* t_pri becomes 59 within 2 secs */
353 static time_t	fss_minslp = 2;	/* min time on sleep queue for hardswap */
354 static int	fss_quantum = 11;
355 
356 static void	fss_newpri(fssproc_t *, boolean_t);
357 static void	fss_update(void *);
358 static int	fss_update_list(int);
359 static void	fss_change_priority(kthread_t *, fssproc_t *);
360 
361 static int	fss_admin(caddr_t, cred_t *);
362 static int	fss_getclinfo(void *);
363 static int	fss_parmsin(void *);
364 static int	fss_parmsout(void *, pc_vaparms_t *);
365 static int	fss_vaparmsin(void *, pc_vaparms_t *);
366 static int	fss_vaparmsout(void *, pc_vaparms_t *);
367 static int	fss_getclpri(pcpri_t *);
368 static int	fss_alloc(void **, int);
369 static void	fss_free(void *);
370 
371 static int	fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
372 static void	fss_exitclass(void *);
373 static int	fss_canexit(kthread_t *, cred_t *);
374 static int	fss_fork(kthread_t *, kthread_t *, void *);
375 static void	fss_forkret(kthread_t *, kthread_t *);
376 static void	fss_parmsget(kthread_t *, void *);
377 static int	fss_parmsset(kthread_t *, void *, id_t, cred_t *);
378 static void	fss_stop(kthread_t *, int, int);
379 static void	fss_exit(kthread_t *);
380 static void	fss_active(kthread_t *);
381 static void	fss_inactive(kthread_t *);
382 static pri_t	fss_swapin(kthread_t *, int);
383 static pri_t	fss_swapout(kthread_t *, int);
384 static void	fss_trapret(kthread_t *);
385 static void	fss_preempt(kthread_t *);
386 static void	fss_setrun(kthread_t *);
387 static void	fss_sleep(kthread_t *);
388 static void	fss_tick(kthread_t *);
389 static void	fss_wakeup(kthread_t *);
390 static int	fss_donice(kthread_t *, cred_t *, int, int *);
391 static int	fss_doprio(kthread_t *, cred_t *, int, int *);
392 static pri_t	fss_globpri(kthread_t *);
393 static void	fss_yield(kthread_t *);
394 static void	fss_nullsys();
395 
396 static struct classfuncs fss_classfuncs = {
397 	/* class functions */
398 	fss_admin,
399 	fss_getclinfo,
400 	fss_parmsin,
401 	fss_parmsout,
402 	fss_vaparmsin,
403 	fss_vaparmsout,
404 	fss_getclpri,
405 	fss_alloc,
406 	fss_free,
407 
408 	/* thread functions */
409 	fss_enterclass,
410 	fss_exitclass,
411 	fss_canexit,
412 	fss_fork,
413 	fss_forkret,
414 	fss_parmsget,
415 	fss_parmsset,
416 	fss_stop,
417 	fss_exit,
418 	fss_active,
419 	fss_inactive,
420 	fss_swapin,
421 	fss_swapout,
422 	fss_trapret,
423 	fss_preempt,
424 	fss_setrun,
425 	fss_sleep,
426 	fss_tick,
427 	fss_wakeup,
428 	fss_donice,
429 	fss_globpri,
430 	fss_nullsys,	/* set_process_group */
431 	fss_yield,
432 	fss_doprio,
433 };
434 
435 int
_init()436 _init()
437 {
438 	return (mod_install(&modlinkage));
439 }
440 
441 int
_fini()442 _fini()
443 {
444 	return (EBUSY);
445 }
446 
447 int
_info(struct modinfo * modinfop)448 _info(struct modinfo *modinfop)
449 {
450 	return (mod_info(&modlinkage, modinfop));
451 }
452 
453 /*ARGSUSED*/
454 static int
fss_project_walker(kproject_t * kpj,void * buf)455 fss_project_walker(kproject_t *kpj, void *buf)
456 {
457 	return (0);
458 }
459 
460 void *
fss_allocbuf(int op,int type)461 fss_allocbuf(int op, int type)
462 {
463 	fssbuf_t *fssbuf;
464 	void **fsslist;
465 	int cnt;
466 	int i;
467 	size_t size;
468 
469 	ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
470 	ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
471 	ASSERT(MUTEX_HELD(&cpu_lock));
472 
473 	fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
474 	switch (op) {
475 	case FSS_NPSET_BUF:
476 		cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
477 		break;
478 	case FSS_NPROJ_BUF:
479 		cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
480 		break;
481 	case FSS_ONE_BUF:
482 		cnt = 1;
483 		break;
484 	}
485 
486 	switch (type) {
487 	case FSS_ALLOC_PROJ:
488 		size = sizeof (fssproj_t);
489 		break;
490 	case FSS_ALLOC_ZONE:
491 		size = sizeof (fsszone_t);
492 		break;
493 	}
494 	fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
495 	fssbuf->fssb_size = cnt;
496 	fssbuf->fssb_list = fsslist;
497 	for (i = 0; i < cnt; i++)
498 		fsslist[i] = kmem_zalloc(size, KM_SLEEP);
499 	return (fssbuf);
500 }
501 
502 void
fss_freebuf(fssbuf_t * fssbuf,int type)503 fss_freebuf(fssbuf_t *fssbuf, int type)
504 {
505 	void **fsslist;
506 	int i;
507 	size_t size;
508 
509 	ASSERT(fssbuf != NULL);
510 	ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
511 	fsslist = fssbuf->fssb_list;
512 
513 	switch (type) {
514 	case FSS_ALLOC_PROJ:
515 		size = sizeof (fssproj_t);
516 		break;
517 	case FSS_ALLOC_ZONE:
518 		size = sizeof (fsszone_t);
519 		break;
520 	}
521 
522 	for (i = 0; i < fssbuf->fssb_size; i++) {
523 		if (fsslist[i] != NULL)
524 			kmem_free(fsslist[i], size);
525 	}
526 	kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
527 	kmem_free(fssbuf, sizeof (fssbuf_t));
528 }
529 
530 static fsspset_t *
fss_find_fsspset(cpupart_t * cpupart)531 fss_find_fsspset(cpupart_t *cpupart)
532 {
533 	int i;
534 	fsspset_t *fsspset = NULL;
535 	int found = 0;
536 
537 	ASSERT(cpupart != NULL);
538 	ASSERT(MUTEX_HELD(&fsspsets_lock));
539 
540 	/*
541 	 * Search for the cpupart pointer in the array of fsspsets.
542 	 */
543 	for (i = 0; i < max_ncpus; i++) {
544 		fsspset = &fsspsets[i];
545 		if (fsspset->fssps_cpupart == cpupart) {
546 			ASSERT(fsspset->fssps_nproj > 0);
547 			found = 1;
548 			break;
549 		}
550 	}
551 	if (found == 0) {
552 		/*
553 		 * If we didn't find anything, then use the first
554 		 * available slot in the fsspsets array.
555 		 */
556 		for (i = 0; i < max_ncpus; i++) {
557 			fsspset = &fsspsets[i];
558 			if (fsspset->fssps_cpupart == NULL) {
559 				ASSERT(fsspset->fssps_nproj == 0);
560 				found = 1;
561 				break;
562 			}
563 		}
564 		fsspset->fssps_cpupart = cpupart;
565 	}
566 	ASSERT(found == 1);
567 	return (fsspset);
568 }
569 
570 static void
fss_del_fsspset(fsspset_t * fsspset)571 fss_del_fsspset(fsspset_t *fsspset)
572 {
573 	ASSERT(MUTEX_HELD(&fsspsets_lock));
574 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
575 	ASSERT(fsspset->fssps_nproj == 0);
576 	ASSERT(fsspset->fssps_list == NULL);
577 	ASSERT(fsspset->fssps_zones == NULL);
578 	fsspset->fssps_cpupart = NULL;
579 	fsspset->fssps_maxfsspri = 0;
580 	fsspset->fssps_shares = 0;
581 }
582 
583 /*
584  * The following routine returns a pointer to the fsszone structure which
585  * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
586  */
587 static fsszone_t *
fss_find_fsszone(fsspset_t * fsspset,zone_t * zone)588 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
589 {
590 	fsszone_t *fsszone;
591 
592 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
593 
594 	if (fsspset->fssps_list != NULL) {
595 		/*
596 		 * There are projects/zones active on this cpu partition
597 		 * already.  Try to find our zone among them.
598 		 */
599 		fsszone = fsspset->fssps_zones;
600 		do {
601 			if (fsszone->fssz_zone == zone) {
602 				return (fsszone);
603 			}
604 			fsszone = fsszone->fssz_next;
605 		} while (fsszone != fsspset->fssps_zones);
606 	}
607 	return (NULL);
608 }
609 
610 /*
611  * The following routine links new fsszone structure into doubly linked list of
612  * zones active on the specified cpu partition.
613  */
614 static void
fss_insert_fsszone(fsspset_t * fsspset,zone_t * zone,fsszone_t * fsszone)615 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
616 {
617 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
618 
619 	fsszone->fssz_zone = zone;
620 	fsszone->fssz_rshares = zone->zone_shares;
621 
622 	if (fsspset->fssps_zones == NULL) {
623 		/*
624 		 * This will be the first fsszone for this fsspset
625 		 */
626 		fsszone->fssz_next = fsszone->fssz_prev = fsszone;
627 		fsspset->fssps_zones = fsszone;
628 	} else {
629 		/*
630 		 * Insert this fsszone to the doubly linked list.
631 		 */
632 		fsszone_t *fssz_head = fsspset->fssps_zones;
633 
634 		fsszone->fssz_next = fssz_head;
635 		fsszone->fssz_prev = fssz_head->fssz_prev;
636 		fssz_head->fssz_prev->fssz_next = fsszone;
637 		fssz_head->fssz_prev = fsszone;
638 		fsspset->fssps_zones = fsszone;
639 	}
640 }
641 
642 /*
643  * The following routine removes a single fsszone structure from the doubly
644  * linked list of zones active on the specified cpu partition.  Note that
645  * global fsspsets_lock must be held in case this fsszone structure is the last
646  * on the above mentioned list.  Also note that the fsszone structure is not
647  * freed here, it is the responsibility of the caller to call kmem_free for it.
648  */
649 static void
fss_remove_fsszone(fsspset_t * fsspset,fsszone_t * fsszone)650 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
651 {
652 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
653 	ASSERT(fsszone->fssz_nproj == 0);
654 	ASSERT(fsszone->fssz_shares == 0);
655 	ASSERT(fsszone->fssz_runnable == 0);
656 
657 	if (fsszone->fssz_next != fsszone) {
658 		/*
659 		 * This is not the last zone in the list.
660 		 */
661 		fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
662 		fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
663 		if (fsspset->fssps_zones == fsszone)
664 			fsspset->fssps_zones = fsszone->fssz_next;
665 	} else {
666 		/*
667 		 * This was the last zone active in this cpu partition.
668 		 */
669 		fsspset->fssps_zones = NULL;
670 	}
671 }
672 
673 /*
674  * The following routine returns a pointer to the fssproj structure
675  * which belongs to project kpj and cpu partition fsspset, if such structure
676  * exists.
677  */
678 static fssproj_t *
fss_find_fssproj(fsspset_t * fsspset,kproject_t * kpj)679 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
680 {
681 	fssproj_t *fssproj;
682 
683 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
684 
685 	if (fsspset->fssps_list != NULL) {
686 		/*
687 		 * There are projects running on this cpu partition already.
688 		 * Try to find our project among them.
689 		 */
690 		fssproj = fsspset->fssps_list;
691 		do {
692 			if (fssproj->fssp_proj == kpj) {
693 				ASSERT(fssproj->fssp_pset == fsspset);
694 				return (fssproj);
695 			}
696 			fssproj = fssproj->fssp_next;
697 		} while (fssproj != fsspset->fssps_list);
698 	}
699 	return (NULL);
700 }
701 
702 /*
703  * The following routine links new fssproj structure into doubly linked list
704  * of projects running on the specified cpu partition.
705  */
706 static void
fss_insert_fssproj(fsspset_t * fsspset,kproject_t * kpj,fsszone_t * fsszone,fssproj_t * fssproj)707 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
708     fssproj_t *fssproj)
709 {
710 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
711 
712 	fssproj->fssp_pset = fsspset;
713 	fssproj->fssp_proj = kpj;
714 	fssproj->fssp_shares = kpj->kpj_shares;
715 
716 	fsspset->fssps_nproj++;
717 
718 	if (fsspset->fssps_list == NULL) {
719 		/*
720 		 * This will be the first fssproj for this fsspset
721 		 */
722 		fssproj->fssp_next = fssproj->fssp_prev = fssproj;
723 		fsspset->fssps_list = fssproj;
724 	} else {
725 		/*
726 		 * Insert this fssproj to the doubly linked list.
727 		 */
728 		fssproj_t *fssp_head = fsspset->fssps_list;
729 
730 		fssproj->fssp_next = fssp_head;
731 		fssproj->fssp_prev = fssp_head->fssp_prev;
732 		fssp_head->fssp_prev->fssp_next = fssproj;
733 		fssp_head->fssp_prev = fssproj;
734 		fsspset->fssps_list = fssproj;
735 	}
736 	fssproj->fssp_fsszone = fsszone;
737 	fsszone->fssz_nproj++;
738 	ASSERT(fsszone->fssz_nproj != 0);
739 }
740 
741 /*
742  * The following routine removes a single fssproj structure from the doubly
743  * linked list of projects running on the specified cpu partition.  Note that
744  * global fsspsets_lock must be held in case if this fssproj structure is the
745  * last on the above mentioned list.  Also note that the fssproj structure is
746  * not freed here, it is the responsibility of the caller to call kmem_free
747  * for it.
748  */
749 static void
fss_remove_fssproj(fsspset_t * fsspset,fssproj_t * fssproj)750 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
751 {
752 	fsszone_t *fsszone;
753 
754 	ASSERT(MUTEX_HELD(&fsspsets_lock));
755 	ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
756 	ASSERT(fssproj->fssp_runnable == 0);
757 
758 	fsspset->fssps_nproj--;
759 
760 	fsszone = fssproj->fssp_fsszone;
761 	fsszone->fssz_nproj--;
762 
763 	if (fssproj->fssp_next != fssproj) {
764 		/*
765 		 * This is not the last part in the list.
766 		 */
767 		fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
768 		fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
769 		if (fsspset->fssps_list == fssproj)
770 			fsspset->fssps_list = fssproj->fssp_next;
771 		if (fsszone->fssz_nproj == 0)
772 			fss_remove_fsszone(fsspset, fsszone);
773 	} else {
774 		/*
775 		 * This was the last project part running
776 		 * at this cpu partition.
777 		 */
778 		fsspset->fssps_list = NULL;
779 		ASSERT(fsspset->fssps_nproj == 0);
780 		ASSERT(fsszone->fssz_nproj == 0);
781 		fss_remove_fsszone(fsspset, fsszone);
782 		fss_del_fsspset(fsspset);
783 	}
784 }
785 
786 static void
fss_inactive(kthread_t * t)787 fss_inactive(kthread_t *t)
788 {
789 	fssproc_t *fssproc;
790 	fssproj_t *fssproj;
791 	fsspset_t *fsspset;
792 	fsszone_t *fsszone;
793 
794 	ASSERT(THREAD_LOCK_HELD(t));
795 	fssproc = FSSPROC(t);
796 	fssproj = FSSPROC2FSSPROJ(fssproc);
797 	if (fssproj == NULL)	/* if this thread already exited */
798 		return;
799 	fsspset = FSSPROJ2FSSPSET(fssproj);
800 	fsszone = fssproj->fssp_fsszone;
801 	disp_lock_enter_high(&fsspset->fssps_displock);
802 	ASSERT(fssproj->fssp_runnable > 0);
803 	if (--fssproj->fssp_runnable == 0) {
804 		fsszone->fssz_shares -= fssproj->fssp_shares;
805 		if (--fsszone->fssz_runnable == 0)
806 			fsspset->fssps_shares -= fsszone->fssz_rshares;
807 	}
808 	ASSERT(fssproc->fss_runnable == 1);
809 	fssproc->fss_runnable = 0;
810 	disp_lock_exit_high(&fsspset->fssps_displock);
811 }
812 
813 static void
fss_active(kthread_t * t)814 fss_active(kthread_t *t)
815 {
816 	fssproc_t *fssproc;
817 	fssproj_t *fssproj;
818 	fsspset_t *fsspset;
819 	fsszone_t *fsszone;
820 
821 	ASSERT(THREAD_LOCK_HELD(t));
822 	fssproc = FSSPROC(t);
823 	fssproj = FSSPROC2FSSPROJ(fssproc);
824 	if (fssproj == NULL)	/* if this thread already exited */
825 		return;
826 	fsspset = FSSPROJ2FSSPSET(fssproj);
827 	fsszone = fssproj->fssp_fsszone;
828 	disp_lock_enter_high(&fsspset->fssps_displock);
829 	if (++fssproj->fssp_runnable == 1) {
830 		fsszone->fssz_shares += fssproj->fssp_shares;
831 		if (++fsszone->fssz_runnable == 1)
832 			fsspset->fssps_shares += fsszone->fssz_rshares;
833 	}
834 	ASSERT(fssproc->fss_runnable == 0);
835 	fssproc->fss_runnable = 1;
836 	disp_lock_exit_high(&fsspset->fssps_displock);
837 }
838 
839 /*
840  * Fair share scheduler initialization. Called by dispinit() at boot time.
841  * We can ignore clparmsz argument since we know that the smallest possible
842  * parameter buffer is big enough for us.
843  */
844 /*ARGSUSED*/
845 static pri_t
fss_init(id_t cid,int clparmsz,classfuncs_t ** clfuncspp)846 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
847 {
848 	int i;
849 
850 	ASSERT(MUTEX_HELD(&cpu_lock));
851 
852 	fss_cid = cid;
853 	fss_maxumdpri = minclsyspri - 1;
854 	fss_maxglobpri = minclsyspri;
855 	fss_minglobpri = 0;
856 	fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
857 
858 	/*
859 	 * Initialize the fssproc hash table.
860 	 */
861 	for (i = 0; i < FSS_LISTS; i++)
862 		fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
863 		    &fss_listhead[i];
864 
865 	*clfuncspp = &fss_classfuncs;
866 
867 	/*
868 	 * Fill in fss_nice_tick and fss_nice_decay arrays:
869 	 * The cost of a tick is lower at positive nice values (so that it
870 	 * will not increase its project's usage as much as normal) with 50%
871 	 * drop at the maximum level and 50% increase at the minimum level.
872 	 * The fsspri decay is slower at positive nice values.  fsspri values
873 	 * of processes with negative nice levels must decay faster to receive
874 	 * time slices more frequently than normal.
875 	 */
876 	for (i = 0; i < FSS_NICE_RANGE; i++) {
877 		fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
878 		    - i)) / FSS_NICE_RANGE;
879 		fss_nice_decay[i] = FSS_DECAY_MIN +
880 		    ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
881 		    (FSS_NICE_RANGE - 1);
882 	}
883 
884 	return (fss_maxglobpri);
885 }
886 
887 /*
888  * Calculate the new fss_umdpri based on the usage, the normalized share usage
889  * and the number of active threads.  Reset the tick counter for this thread.
890  *
891  * When calculating the new priority using the standard formula we can hit
892  * a scenario where we don't have good round-robin behavior.  This would be
893  * most commonly seen when there is a zone with lots of runnable threads.
894  * In the bad scenario we will see the following behavior when using the
895  * standard formula and these conditions:
896  *
897  *	- there are multiple runnable threads in the zone (project)
898  *	- the fssps_maxfsspri is a very large value
899  *	- (we also know all of these threads will use the project's
900  *	    fssp_shusage)
901  *
902  * Under these conditions, a thread with a low fss_fsspri value is chosen
903  * to run and the thread gets a high fss_umdpri.  This thread can run for
904  * its full quanta (fss_timeleft) at which time fss_newpri is called to
905  * calculate the thread's new priority.
906  *
907  * In this case, because the newly calculated fsspri value is much smaller
908  * (orders of magnitude) than the fssps_maxfsspri value, if we used the
909  * standard formula the thread will still get a high fss_umdpri value and
910  * will run again for another quanta, even though there are other runnable
911  * threads in the project.
912  *
913  * For a thread that is runnable for a long time, the thread can continue
914  * to run for many quanta (totaling many seconds) before the thread's fsspri
915  * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back
916  * down to 1.  This behavior also keeps the fssps_maxfsspr at a high value,
917  * so that the next runnable thread might repeat this cycle.
918  *
919  * This leads to the case where we don't have round-robin behavior at quanta
920  * granularity, but instead, runnable threads within the project only run
921  * at several second intervals.
922  *
923  * To prevent this scenario from occuring, when a thread has consumed its
924  * quanta and there are multiple runnable threads in the project, we
925  * immediately cause the thread to hit fssps_maxfsspri so that it gets
926  * reset back to 1 and another runnable thread in the project can run.
927  */
928 static void
fss_newpri(fssproc_t * fssproc,boolean_t quanta_up)929 fss_newpri(fssproc_t *fssproc, boolean_t quanta_up)
930 {
931 	kthread_t *tp;
932 	fssproj_t *fssproj;
933 	fsspset_t *fsspset;
934 	fsszone_t *fsszone;
935 	fsspri_t fsspri, maxfsspri;
936 	uint32_t n_runnable;
937 	pri_t invpri;
938 	uint32_t ticks;
939 
940 	tp = fssproc->fss_tp;
941 	ASSERT(tp != NULL);
942 
943 	if (tp->t_cid != fss_cid)
944 		return;
945 
946 	ASSERT(THREAD_LOCK_HELD(tp));
947 
948 	fssproj = FSSPROC2FSSPROJ(fssproc);
949 	fsszone = FSSPROJ2FSSZONE(fssproj);
950 	if (fssproj == NULL)
951 		/*
952 		 * No need to change priority of exited threads.
953 		 */
954 		return;
955 
956 	fsspset = FSSPROJ2FSSPSET(fssproj);
957 	disp_lock_enter_high(&fsspset->fssps_displock);
958 
959 	ticks = fssproc->fss_ticks;
960 	fssproc->fss_ticks = 0;
961 
962 	if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
963 		/*
964 		 * Special case: threads with no shares.
965 		 */
966 		fssproc->fss_umdpri = fss_minglobpri;
967 		disp_lock_exit_high(&fsspset->fssps_displock);
968 		return;
969 	}
970 
971 	maxfsspri = fsspset->fssps_maxfsspri;
972 	n_runnable = fssproj->fssp_runnable;
973 
974 	if (quanta_up && n_runnable > 1) {
975 		fsspri = maxfsspri;
976 	} else {
977 		/*
978 		 * fsspri += fssp_shusage * nrunnable * ticks
979 		 * If all three values are non-0, this typically calculates to
980 		 * a large number (sometimes > 1M, sometimes > 100B) due to
981 		 * fssp_shusage which can be > 1T.
982 		 */
983 		fsspri = fssproc->fss_fsspri;
984 		fsspri += fssproj->fssp_shusage * n_runnable * ticks;
985 	}
986 
987 	fssproc->fss_fsspri = fsspri;
988 
989 	/*
990 	 * fss_maxumdpri is normally 59, since FSS priorities are 0-59.
991 	 * If the previous calculation resulted in 0 (e.g. was 0 and added 0
992 	 * because ticks == 0), then instead of 0, we use the largest priority,
993 	 * which is still small in comparison to the large numbers we typically
994 	 * see.
995 	 */
996 	if (fsspri < fss_maxumdpri)
997 		fsspri = fss_maxumdpri;	/* so that maxfsspri is != 0 */
998 
999 	/*
1000 	 * The general priority formula:
1001 	 *
1002 	 *			(fsspri * umdprirange)
1003 	 *   pri = maxumdpri - ------------------------
1004 	 *				maxfsspri
1005 	 *
1006 	 * If this thread's fsspri is greater than the previous largest
1007 	 * fsspri, then record it as the new high and priority for this
1008 	 * thread will be one (the lowest priority assigned to a thread
1009 	 * that has non-zero shares). Because of this check, maxfsspri can
1010 	 * change as this function is called via the
1011 	 * fss_update -> fss_update_list -> fss_newpri code path to update
1012 	 * all runnable threads. See the code in fss_update for how we
1013 	 * mitigate this issue.
1014 	 *
1015 	 * Note that this formula cannot produce out of bounds priority
1016 	 * values (0-59); if it is changed, additional checks may need to be
1017 	 * added.
1018 	 */
1019 	if (fsspri >= maxfsspri) {
1020 		fsspset->fssps_maxfsspri = fsspri;
1021 		disp_lock_exit_high(&fsspset->fssps_displock);
1022 		fssproc->fss_umdpri = 1;
1023 	} else {
1024 		disp_lock_exit_high(&fsspset->fssps_displock);
1025 		invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
1026 		fssproc->fss_umdpri = fss_maxumdpri - invpri;
1027 	}
1028 }
1029 
1030 /*
1031  * Decays usages of all running projects, resets their tick counters and
1032  * calcluates the projects normalized share usage. Called once per second from
1033  * fss_update().
1034  */
1035 static void
fss_decay_usage()1036 fss_decay_usage()
1037 {
1038 	uint32_t zone_ext_shares, zone_int_shares;
1039 	uint32_t kpj_shares, pset_shares;
1040 	fsspset_t *fsspset;
1041 	fssproj_t *fssproj;
1042 	fsszone_t *fsszone;
1043 	fsspri_t maxfsspri;
1044 	int psetid;
1045 	struct zone *zp;
1046 
1047 	mutex_enter(&fsspsets_lock);
1048 	/*
1049 	 * Go through all active processor sets and decay usages of projects
1050 	 * running on them.
1051 	 */
1052 	for (psetid = 0; psetid < max_ncpus; psetid++) {
1053 		fsspset = &fsspsets[psetid];
1054 		mutex_enter(&fsspset->fssps_lock);
1055 
1056 		fsspset->fssps_gen++;
1057 
1058 		if (fsspset->fssps_cpupart == NULL ||
1059 		    (fssproj = fsspset->fssps_list) == NULL) {
1060 			mutex_exit(&fsspset->fssps_lock);
1061 			continue;
1062 		}
1063 
1064 		/*
1065 		 * Decay maxfsspri for this cpu partition with the
1066 		 * fastest possible decay rate.
1067 		 */
1068 		disp_lock_enter(&fsspset->fssps_displock);
1069 
1070 		pset_shares = fsspset->fssps_shares;
1071 
1072 		maxfsspri = (fsspset->fssps_maxfsspri *
1073 		    fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
1074 		if (maxfsspri < fss_maxumdpri)
1075 			maxfsspri = fss_maxumdpri;
1076 		fsspset->fssps_maxfsspri = maxfsspri;
1077 
1078 		do {
1079 			fsszone = fssproj->fssp_fsszone;
1080 			zp = fsszone->fssz_zone;
1081 
1082 			/*
1083 			 * Reset zone's FSS stats if they are from a
1084 			 * previous cycle.
1085 			 */
1086 			if (fsspset->fssps_gen != zp->zone_fss_gen) {
1087 				zp->zone_fss_gen = fsspset->fssps_gen;
1088 				zp->zone_run_ticks = 0;
1089 			}
1090 
1091 			/*
1092 			 * Decay project usage, then add in this cycle's
1093 			 * nice tick value.
1094 			 */
1095 			fssproj->fssp_usage =
1096 			    (fssproj->fssp_usage * FSS_DECAY_USG) /
1097 			    FSS_DECAY_BASE +
1098 			    fssproj->fssp_ticks;
1099 
1100 			fssproj->fssp_ticks = 0;
1101 			zp->zone_run_ticks += fssproj->fssp_tick_cnt;
1102 			fssproj->fssp_tick_cnt = 0;
1103 
1104 			/*
1105 			 * Readjust the project's number of shares if it has
1106 			 * changed since we checked it last time.
1107 			 */
1108 			kpj_shares = fssproj->fssp_proj->kpj_shares;
1109 			if (fssproj->fssp_shares != kpj_shares) {
1110 				if (fssproj->fssp_runnable != 0) {
1111 					fsszone->fssz_shares -=
1112 					    fssproj->fssp_shares;
1113 					fsszone->fssz_shares += kpj_shares;
1114 				}
1115 				fssproj->fssp_shares = kpj_shares;
1116 			}
1117 
1118 			/*
1119 			 * Readjust the zone's number of shares if it
1120 			 * has changed since we checked it last time.
1121 			 */
1122 			zone_ext_shares = zp->zone_shares;
1123 			if (fsszone->fssz_rshares != zone_ext_shares) {
1124 				if (fsszone->fssz_runnable != 0) {
1125 					fsspset->fssps_shares -=
1126 					    fsszone->fssz_rshares;
1127 					fsspset->fssps_shares +=
1128 					    zone_ext_shares;
1129 					pset_shares = fsspset->fssps_shares;
1130 				}
1131 				fsszone->fssz_rshares = zone_ext_shares;
1132 			}
1133 			zone_int_shares = fsszone->fssz_shares;
1134 
1135 			/*
1136 			 * If anything is runnable in the project, track the
1137 			 * overall project share percent for monitoring useage.
1138 			 */
1139 			if (fssproj->fssp_runnable > 0) {
1140 				uint32_t zone_shr_pct;
1141 				uint32_t int_shr_pct;
1142 
1143 				/*
1144 				 * Times 1000 to get tenths of a percent
1145 				 *
1146 				 *		  zone_ext_shares
1147 				 * zone_shr_pct = ---------------
1148 				 *		  pset_shares
1149 				 *
1150 				 *		  kpj_shares
1151 				 * int_shr_pct =  ---------------
1152 				 *		  zone_int_shares
1153 				 */
1154 				if (pset_shares == 0 || zone_int_shares == 0) {
1155 					fssproj->fssp_shr_pct = 0;
1156 				} else {
1157 					zone_shr_pct =
1158 					    (zone_ext_shares * 1000) /
1159 					    pset_shares;
1160 					int_shr_pct = (kpj_shares * 1000) /
1161 					    zone_int_shares;
1162 					fssproj->fssp_shr_pct =
1163 					    (zone_shr_pct * int_shr_pct) /
1164 					    1000;
1165 				}
1166 			} else {
1167 				DTRACE_PROBE1(fss__prj__norun, fssproj_t *,
1168 				    fssproj);
1169 			}
1170 
1171 			/*
1172 			 * Calculate fssp_shusage value to be used
1173 			 * for fsspri increments for the next second.
1174 			 */
1175 			if (kpj_shares == 0 || zone_ext_shares == 0) {
1176 				fssproj->fssp_shusage = 0;
1177 			} else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
1178 				uint32_t zone_shr_pct;
1179 
1180 				/*
1181 				 * Project 0 in the global zone has 50%
1182 				 * of its zone. See calculation above for
1183 				 * the zone's share percent.
1184 				 */
1185 				if (pset_shares == 0)
1186 					zone_shr_pct = 1000;
1187 				else
1188 					zone_shr_pct =
1189 					    (zone_ext_shares * 1000) /
1190 					    pset_shares;
1191 
1192 				fssproj->fssp_shr_pct = zone_shr_pct / 2;
1193 
1194 				fssproj->fssp_shusage = (fssproj->fssp_usage *
1195 				    zone_int_shares * zone_int_shares) /
1196 				    (zone_ext_shares * zone_ext_shares);
1197 			} else {
1198 				/*
1199 				 * Thread's priority is based on its project's
1200 				 * normalized usage (shusage) value which gets
1201 				 * calculated this way:
1202 				 *
1203 				 *	   pset_shares^2    zone_int_shares^2
1204 				 * usage * ------------- * ------------------
1205 				 *	   kpj_shares^2	    zone_ext_shares^2
1206 				 *
1207 				 * Where zone_int_shares is the sum of shares
1208 				 * of all active projects within the zone (and
1209 				 * the pset), and zone_ext_shares is the number
1210 				 * of zone shares (ie, zone.cpu-shares).
1211 				 *
1212 				 * If there is only one zone active on the pset
1213 				 * the above reduces to:
1214 				 *
1215 				 * 			zone_int_shares^2
1216 				 * shusage = usage * ---------------------
1217 				 * 			kpj_shares^2
1218 				 *
1219 				 * If there's only one project active in the
1220 				 * zone this formula reduces to:
1221 				 *
1222 				 *			pset_shares^2
1223 				 * shusage = usage * ----------------------
1224 				 *			zone_ext_shares^2
1225 				 *
1226 				 * shusage is one input to calculating fss_pri
1227 				 * in fss_newpri(). Larger values tend toward
1228 				 * lower priorities for processes in the proj.
1229 				 */
1230 				fssproj->fssp_shusage = fssproj->fssp_usage *
1231 				    pset_shares * zone_int_shares;
1232 				fssproj->fssp_shusage /=
1233 				    kpj_shares * zone_ext_shares;
1234 				fssproj->fssp_shusage *=
1235 				    pset_shares * zone_int_shares;
1236 				fssproj->fssp_shusage /=
1237 				    kpj_shares * zone_ext_shares;
1238 			}
1239 			fssproj = fssproj->fssp_next;
1240 		} while (fssproj != fsspset->fssps_list);
1241 
1242 		disp_lock_exit(&fsspset->fssps_displock);
1243 		mutex_exit(&fsspset->fssps_lock);
1244 	}
1245 	mutex_exit(&fsspsets_lock);
1246 }
1247 
1248 static void
fss_change_priority(kthread_t * t,fssproc_t * fssproc)1249 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
1250 {
1251 	pri_t new_pri;
1252 
1253 	ASSERT(THREAD_LOCK_HELD(t));
1254 	new_pri = fssproc->fss_umdpri;
1255 	ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
1256 
1257 	t->t_cpri = fssproc->fss_upri;
1258 	fssproc->fss_flags &= ~FSSRESTORE;
1259 	if (t == curthread || t->t_state == TS_ONPROC) {
1260 		/*
1261 		 * curthread is always onproc
1262 		 */
1263 		cpu_t *cp = t->t_disp_queue->disp_cpu;
1264 		THREAD_CHANGE_PRI(t, new_pri);
1265 		if (t == cp->cpu_dispthread)
1266 			cp->cpu_dispatch_pri = DISP_PRIO(t);
1267 		if (DISP_MUST_SURRENDER(t)) {
1268 			fssproc->fss_flags |= FSSBACKQ;
1269 			cpu_surrender(t);
1270 		} else {
1271 			fssproc->fss_timeleft = fss_quantum;
1272 		}
1273 	} else {
1274 		/*
1275 		 * When the priority of a thread is changed, it may be
1276 		 * necessary to adjust its position on a sleep queue or
1277 		 * dispatch queue.  The function thread_change_pri accomplishes
1278 		 * this.
1279 		 */
1280 		if (thread_change_pri(t, new_pri, 0)) {
1281 			/*
1282 			 * The thread was on a run queue.
1283 			 */
1284 			fssproc->fss_timeleft = fss_quantum;
1285 		} else {
1286 			fssproc->fss_flags |= FSSBACKQ;
1287 		}
1288 	}
1289 }
1290 
1291 /*
1292  * Update priorities of all fair-sharing threads that are currently runnable
1293  * at a user mode priority based on the number of shares and current usage.
1294  * Called once per second via timeout which we reset here.
1295  *
1296  * There are several lists of fair-sharing threads broken up by a hash on the
1297  * thread pointer.  Each list has its own lock.  This avoids blocking all
1298  * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
1299  * fss_update traverses each list in turn.
1300  *
1301  * Each time we're run (once/second) we may start at the next list and iterate
1302  * through all of the lists. By starting with a different list, we mitigate any
1303  * effects we would see updating the fssps_maxfsspri value in fss_newpri.
1304  */
1305 static void
fss_update(void * arg)1306 fss_update(void *arg)
1307 {
1308 	int i;
1309 	int new_marker = -1;
1310 	static int fss_update_marker;
1311 
1312 	/*
1313 	 * Decay and update usages for all projects.
1314 	 */
1315 	fss_decay_usage();
1316 
1317 	/*
1318 	 * Start with the fss_update_marker list, then do the rest.
1319 	 */
1320 	i = fss_update_marker;
1321 
1322 	/*
1323 	 * Go around all threads, set new priorities and decay
1324 	 * per-thread CPU usages.
1325 	 */
1326 	do {
1327 		/*
1328 		 * If this is the first list after the current marker to have
1329 		 * threads with priority updates, advance the marker to this
1330 		 * list for the next time fss_update runs.
1331 		 */
1332 		if (fss_update_list(i) &&
1333 		    new_marker == -1 && i != fss_update_marker)
1334 			new_marker = i;
1335 	} while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1336 
1337 	/*
1338 	 * Advance marker for the next fss_update call
1339 	 */
1340 	if (new_marker != -1)
1341 		fss_update_marker = new_marker;
1342 
1343 	(void) timeout(fss_update, arg, hz);
1344 }
1345 
1346 /*
1347  * Updates priority for a list of threads.  Returns 1 if the priority of one
1348  * of the threads was actually updated, 0 if none were for various reasons
1349  * (thread is no longer in the FSS class, is not runnable, has the preemption
1350  * control no-preempt bit set, etc.)
1351  */
1352 static int
fss_update_list(int i)1353 fss_update_list(int i)
1354 {
1355 	fssproc_t *fssproc;
1356 	fssproj_t *fssproj;
1357 	fsspri_t fsspri;
1358 	pri_t fss_umdpri;
1359 	kthread_t *t;
1360 	int updated = 0;
1361 
1362 	mutex_enter(&fss_listlock[i]);
1363 	for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1364 	    fssproc = fssproc->fss_next) {
1365 		t = fssproc->fss_tp;
1366 		/*
1367 		 * Lock the thread and verify the state.
1368 		 */
1369 		thread_lock(t);
1370 		/*
1371 		 * Skip the thread if it is no longer in the FSS class or
1372 		 * is running with kernel mode priority.
1373 		 */
1374 		if (t->t_cid != fss_cid)
1375 			goto next;
1376 		if ((fssproc->fss_flags & FSSKPRI) != 0)
1377 			goto next;
1378 
1379 		fssproj = FSSPROC2FSSPROJ(fssproc);
1380 		if (fssproj == NULL)
1381 			goto next;
1382 
1383 		if (fssproj->fssp_shares != 0) {
1384 			/*
1385 			 * Decay fsspri value.
1386 			 */
1387 			fsspri = fssproc->fss_fsspri;
1388 			fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1389 			    FSS_DECAY_BASE;
1390 			fssproc->fss_fsspri = fsspri;
1391 		}
1392 
1393 		if (t->t_schedctl && schedctl_get_nopreempt(t))
1394 			goto next;
1395 		if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1396 			/*
1397 			 * Make next syscall/trap call fss_trapret
1398 			 */
1399 			t->t_trapret = 1;
1400 			aston(t);
1401 			if (t->t_state == TS_ONPROC)
1402 				DTRACE_PROBE1(fss__onproc, fssproc_t *,
1403 				    fssproc);
1404 			goto next;
1405 		}
1406 		fss_newpri(fssproc, B_FALSE);
1407 		updated = 1;
1408 
1409 		fss_umdpri = fssproc->fss_umdpri;
1410 
1411 		/*
1412 		 * Only dequeue the thread if it needs to be moved; otherwise
1413 		 * it should just round-robin here.
1414 		 */
1415 		if (t->t_pri != fss_umdpri)
1416 			fss_change_priority(t, fssproc);
1417 next:
1418 		thread_unlock(t);
1419 	}
1420 	mutex_exit(&fss_listlock[i]);
1421 	return (updated);
1422 }
1423 
1424 /*ARGSUSED*/
1425 static int
fss_admin(caddr_t uaddr,cred_t * reqpcredp)1426 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1427 {
1428 	fssadmin_t fssadmin;
1429 
1430 	if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1431 		return (EFAULT);
1432 
1433 	switch (fssadmin.fss_cmd) {
1434 	case FSS_SETADMIN:
1435 		if (secpolicy_dispadm(reqpcredp) != 0)
1436 			return (EPERM);
1437 		if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1438 			return (EINVAL);
1439 		fss_quantum = fssadmin.fss_quantum;
1440 		break;
1441 	case FSS_GETADMIN:
1442 		fssadmin.fss_quantum = fss_quantum;
1443 		if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1444 			return (EFAULT);
1445 		break;
1446 	default:
1447 		return (EINVAL);
1448 	}
1449 	return (0);
1450 }
1451 
1452 static int
fss_getclinfo(void * infop)1453 fss_getclinfo(void *infop)
1454 {
1455 	fssinfo_t *fssinfo = (fssinfo_t *)infop;
1456 	fssinfo->fss_maxupri = fss_maxupri;
1457 	return (0);
1458 }
1459 
1460 static int
fss_parmsin(void * parmsp)1461 fss_parmsin(void *parmsp)
1462 {
1463 	fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1464 
1465 	/*
1466 	 * Check validity of parameters.
1467 	 */
1468 	if ((fssparmsp->fss_uprilim > fss_maxupri ||
1469 	    fssparmsp->fss_uprilim < -fss_maxupri) &&
1470 	    fssparmsp->fss_uprilim != FSS_NOCHANGE)
1471 		return (EINVAL);
1472 
1473 	if ((fssparmsp->fss_upri > fss_maxupri ||
1474 	    fssparmsp->fss_upri < -fss_maxupri) &&
1475 	    fssparmsp->fss_upri != FSS_NOCHANGE)
1476 		return (EINVAL);
1477 
1478 	return (0);
1479 }
1480 
1481 /*ARGSUSED*/
1482 static int
fss_parmsout(void * parmsp,pc_vaparms_t * vaparmsp)1483 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1484 {
1485 	return (0);
1486 }
1487 
1488 static int
fss_vaparmsin(void * parmsp,pc_vaparms_t * vaparmsp)1489 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1490 {
1491 	fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1492 	int priflag = 0;
1493 	int limflag = 0;
1494 	uint_t cnt;
1495 	pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1496 
1497 	/*
1498 	 * FSS_NOCHANGE (-32768) is outside of the range of values for
1499 	 * fss_uprilim and fss_upri.  If the structure fssparms_t is changed,
1500 	 * FSS_NOCHANGE should be replaced by a flag word.
1501 	 */
1502 	fssparmsp->fss_uprilim = FSS_NOCHANGE;
1503 	fssparmsp->fss_upri = FSS_NOCHANGE;
1504 
1505 	/*
1506 	 * Get the varargs parameter and check validity of parameters.
1507 	 */
1508 	if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1509 		return (EINVAL);
1510 
1511 	for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1512 		switch (vpp->pc_key) {
1513 		case FSS_KY_UPRILIM:
1514 			if (limflag++)
1515 				return (EINVAL);
1516 			fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1517 			if (fssparmsp->fss_uprilim > fss_maxupri ||
1518 			    fssparmsp->fss_uprilim < -fss_maxupri)
1519 				return (EINVAL);
1520 			break;
1521 		case FSS_KY_UPRI:
1522 			if (priflag++)
1523 				return (EINVAL);
1524 			fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1525 			if (fssparmsp->fss_upri > fss_maxupri ||
1526 			    fssparmsp->fss_upri < -fss_maxupri)
1527 				return (EINVAL);
1528 			break;
1529 		default:
1530 			return (EINVAL);
1531 		}
1532 	}
1533 
1534 	if (vaparmsp->pc_vaparmscnt == 0) {
1535 		/*
1536 		 * Use default parameters.
1537 		 */
1538 		fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1539 	}
1540 
1541 	return (0);
1542 }
1543 
1544 /*
1545  * Copy all selected fair-sharing class parameters to the user.  The parameters
1546  * are specified by a key.
1547  */
1548 static int
fss_vaparmsout(void * parmsp,pc_vaparms_t * vaparmsp)1549 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1550 {
1551 	fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1552 	int priflag = 0;
1553 	int limflag = 0;
1554 	uint_t cnt;
1555 	pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1556 
1557 	ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1558 
1559 	if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1560 		return (EINVAL);
1561 
1562 	for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1563 		switch (vpp->pc_key) {
1564 		case FSS_KY_UPRILIM:
1565 			if (limflag++)
1566 				return (EINVAL);
1567 			if (copyout(&fssparmsp->fss_uprilim,
1568 			    (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1569 				return (EFAULT);
1570 			break;
1571 		case FSS_KY_UPRI:
1572 			if (priflag++)
1573 				return (EINVAL);
1574 			if (copyout(&fssparmsp->fss_upri,
1575 			    (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1576 				return (EFAULT);
1577 			break;
1578 		default:
1579 			return (EINVAL);
1580 		}
1581 	}
1582 
1583 	return (0);
1584 }
1585 
1586 /*
1587  * Return the user mode scheduling priority range.
1588  */
1589 static int
fss_getclpri(pcpri_t * pcprip)1590 fss_getclpri(pcpri_t *pcprip)
1591 {
1592 	pcprip->pc_clpmax = fss_maxupri;
1593 	pcprip->pc_clpmin = -fss_maxupri;
1594 	return (0);
1595 }
1596 
1597 static int
fss_alloc(void ** p,int flag)1598 fss_alloc(void **p, int flag)
1599 {
1600 	void *bufp;
1601 
1602 	if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1603 		return (ENOMEM);
1604 	} else {
1605 		*p = bufp;
1606 		return (0);
1607 	}
1608 }
1609 
1610 static void
fss_free(void * bufp)1611 fss_free(void *bufp)
1612 {
1613 	if (bufp)
1614 		kmem_free(bufp, sizeof (fssproc_t));
1615 }
1616 
1617 /*
1618  * Thread functions
1619  */
1620 static int
fss_enterclass(kthread_t * t,id_t cid,void * parmsp,cred_t * reqpcredp,void * bufp)1621 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1622     void *bufp)
1623 {
1624 	fssparms_t	*fssparmsp = (fssparms_t *)parmsp;
1625 	fssproc_t	*fssproc;
1626 	pri_t		reqfssuprilim;
1627 	pri_t		reqfssupri;
1628 	static uint32_t fssexists = 0;
1629 	fsspset_t	*fsspset;
1630 	fssproj_t	*fssproj;
1631 	fsszone_t	*fsszone;
1632 	kproject_t	*kpj;
1633 	zone_t		*zone;
1634 	int		fsszone_allocated = 0;
1635 
1636 	fssproc = (fssproc_t *)bufp;
1637 	ASSERT(fssproc != NULL);
1638 
1639 	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1640 
1641 	/*
1642 	 * Only root can move threads to FSS class.
1643 	 */
1644 	if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1645 		return (EPERM);
1646 	/*
1647 	 * Initialize the fssproc structure.
1648 	 */
1649 	fssproc->fss_umdpri = fss_maxumdpri / 2;
1650 
1651 	if (fssparmsp == NULL) {
1652 		/*
1653 		 * Use default values.
1654 		 */
1655 		fssproc->fss_nice = NZERO;
1656 		fssproc->fss_uprilim = fssproc->fss_upri = 0;
1657 	} else {
1658 		/*
1659 		 * Use supplied values.
1660 		 */
1661 		if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1662 			reqfssuprilim = 0;
1663 		} else {
1664 			if (fssparmsp->fss_uprilim > 0 &&
1665 			    secpolicy_setpriority(reqpcredp) != 0)
1666 				return (EPERM);
1667 			reqfssuprilim = fssparmsp->fss_uprilim;
1668 		}
1669 		if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1670 			reqfssupri = reqfssuprilim;
1671 		} else {
1672 			if (fssparmsp->fss_upri > 0 &&
1673 			    secpolicy_setpriority(reqpcredp) != 0)
1674 				return (EPERM);
1675 			/*
1676 			 * Set the user priority to the requested value or
1677 			 * the upri limit, whichever is lower.
1678 			 */
1679 			reqfssupri = fssparmsp->fss_upri;
1680 			if (reqfssupri > reqfssuprilim)
1681 				reqfssupri = reqfssuprilim;
1682 		}
1683 		fssproc->fss_uprilim = reqfssuprilim;
1684 		fssproc->fss_upri = reqfssupri;
1685 		fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1686 		if (fssproc->fss_nice > FSS_NICE_MAX)
1687 			fssproc->fss_nice = FSS_NICE_MAX;
1688 	}
1689 
1690 	fssproc->fss_timeleft = fss_quantum;
1691 	fssproc->fss_tp = t;
1692 	cpucaps_sc_init(&fssproc->fss_caps);
1693 
1694 	/*
1695 	 * Put a lock on our fsspset structure.
1696 	 */
1697 	mutex_enter(&fsspsets_lock);
1698 	fsspset = fss_find_fsspset(t->t_cpupart);
1699 	mutex_enter(&fsspset->fssps_lock);
1700 	mutex_exit(&fsspsets_lock);
1701 
1702 	zone = ttoproc(t)->p_zone;
1703 	if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1704 		if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1705 		    == NULL) {
1706 			mutex_exit(&fsspset->fssps_lock);
1707 			return (ENOMEM);
1708 		} else {
1709 			fsszone_allocated = 1;
1710 			fss_insert_fsszone(fsspset, zone, fsszone);
1711 		}
1712 	}
1713 	kpj = ttoproj(t);
1714 	if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1715 		if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1716 		    == NULL) {
1717 			if (fsszone_allocated) {
1718 				fss_remove_fsszone(fsspset, fsszone);
1719 				kmem_free(fsszone, sizeof (fsszone_t));
1720 			}
1721 			mutex_exit(&fsspset->fssps_lock);
1722 			return (ENOMEM);
1723 		} else {
1724 			fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1725 		}
1726 	}
1727 	fssproj->fssp_threads++;
1728 	fssproc->fss_proj = fssproj;
1729 
1730 	/*
1731 	 * Reset priority. Process goes to a "user mode" priority here
1732 	 * regardless of whether or not it has slept since entering the kernel.
1733 	 */
1734 	thread_lock(t);
1735 	t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1736 	t->t_cid = cid;
1737 	t->t_cldata = (void *)fssproc;
1738 	t->t_schedflag |= TS_RUNQMATCH;
1739 	fss_change_priority(t, fssproc);
1740 	if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1741 	    t->t_state == TS_WAIT)
1742 		fss_active(t);
1743 	thread_unlock(t);
1744 
1745 	mutex_exit(&fsspset->fssps_lock);
1746 
1747 	/*
1748 	 * Link new structure into fssproc list.
1749 	 */
1750 	FSS_LIST_INSERT(fssproc);
1751 
1752 	/*
1753 	 * If this is the first fair-sharing thread to occur since boot,
1754 	 * we set up the initial call to fss_update() here. Use an atomic
1755 	 * compare-and-swap since that's easier and faster than a mutex
1756 	 * (but check with an ordinary load first since most of the time
1757 	 * this will already be done).
1758 	 */
1759 	if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
1760 		(void) timeout(fss_update, NULL, hz);
1761 
1762 	return (0);
1763 }
1764 
1765 /*
1766  * Remove fssproc_t from the list.
1767  */
1768 static void
fss_exitclass(void * procp)1769 fss_exitclass(void *procp)
1770 {
1771 	fssproc_t *fssproc = (fssproc_t *)procp;
1772 	fssproj_t *fssproj;
1773 	fsspset_t *fsspset;
1774 	fsszone_t *fsszone;
1775 	kthread_t *t = fssproc->fss_tp;
1776 
1777 	/*
1778 	 * We should be either getting this thread off the deathrow or
1779 	 * this thread has already moved to another scheduling class and
1780 	 * we're being called with its old cldata buffer pointer.  In both
1781 	 * cases, the content of this buffer can not be changed while we're
1782 	 * here.
1783 	 */
1784 	mutex_enter(&fsspsets_lock);
1785 	thread_lock(t);
1786 	if (t->t_cid != fss_cid) {
1787 		/*
1788 		 * We're being called as a result of the priocntl() system
1789 		 * call -- someone is trying to move our thread to another
1790 		 * scheduling class. We can't call fss_inactive() here
1791 		 * because our thread's t_cldata pointer already points
1792 		 * to another scheduling class specific data.
1793 		 */
1794 		ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1795 
1796 		fssproj = FSSPROC2FSSPROJ(fssproc);
1797 		fsspset = FSSPROJ2FSSPSET(fssproj);
1798 		fsszone = fssproj->fssp_fsszone;
1799 
1800 		if (fssproc->fss_runnable) {
1801 			disp_lock_enter_high(&fsspset->fssps_displock);
1802 			if (--fssproj->fssp_runnable == 0) {
1803 				fsszone->fssz_shares -= fssproj->fssp_shares;
1804 				if (--fsszone->fssz_runnable == 0)
1805 					fsspset->fssps_shares -=
1806 					    fsszone->fssz_rshares;
1807 			}
1808 			disp_lock_exit_high(&fsspset->fssps_displock);
1809 		}
1810 		thread_unlock(t);
1811 
1812 		mutex_enter(&fsspset->fssps_lock);
1813 		if (--fssproj->fssp_threads == 0) {
1814 			fss_remove_fssproj(fsspset, fssproj);
1815 			if (fsszone->fssz_nproj == 0)
1816 				kmem_free(fsszone, sizeof (fsszone_t));
1817 			kmem_free(fssproj, sizeof (fssproj_t));
1818 		}
1819 		mutex_exit(&fsspset->fssps_lock);
1820 
1821 	} else {
1822 		ASSERT(t->t_state == TS_FREE);
1823 		/*
1824 		 * We're being called from thread_free() when our thread
1825 		 * is removed from the deathrow. There is nothing we need
1826 		 * do here since everything should've been done earlier
1827 		 * in fss_exit().
1828 		 */
1829 		thread_unlock(t);
1830 	}
1831 	mutex_exit(&fsspsets_lock);
1832 
1833 	FSS_LIST_DELETE(fssproc);
1834 	fss_free(fssproc);
1835 }
1836 
1837 /*ARGSUSED*/
1838 static int
fss_canexit(kthread_t * t,cred_t * credp)1839 fss_canexit(kthread_t *t, cred_t *credp)
1840 {
1841 	/*
1842 	 * A thread is allowed to exit FSS only if we have sufficient
1843 	 * privileges.
1844 	 */
1845 	if (credp != NULL && secpolicy_setpriority(credp) != 0)
1846 		return (EPERM);
1847 	else
1848 		return (0);
1849 }
1850 
1851 /*
1852  * Initialize fair-share class specific proc structure for a child.
1853  */
1854 static int
fss_fork(kthread_t * pt,kthread_t * ct,void * bufp)1855 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1856 {
1857 	fssproc_t *pfssproc;	/* ptr to parent's fssproc structure	*/
1858 	fssproc_t *cfssproc;	/* ptr to child's fssproc structure	*/
1859 	fssproj_t *fssproj;
1860 	fsspset_t *fsspset;
1861 
1862 	ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1863 	ASSERT(ct->t_state == TS_STOPPED);
1864 
1865 	cfssproc = (fssproc_t *)bufp;
1866 	ASSERT(cfssproc != NULL);
1867 	bzero(cfssproc, sizeof (fssproc_t));
1868 
1869 	thread_lock(pt);
1870 	pfssproc = FSSPROC(pt);
1871 	fssproj = FSSPROC2FSSPROJ(pfssproc);
1872 	fsspset = FSSPROJ2FSSPSET(fssproj);
1873 	thread_unlock(pt);
1874 
1875 	mutex_enter(&fsspset->fssps_lock);
1876 	/*
1877 	 * Initialize child's fssproc structure.
1878 	 */
1879 	thread_lock(pt);
1880 	ASSERT(FSSPROJ(pt) == fssproj);
1881 	cfssproc->fss_proj = fssproj;
1882 	cfssproc->fss_timeleft = fss_quantum;
1883 	cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1884 	cfssproc->fss_fsspri = 0;
1885 	cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1886 	cfssproc->fss_upri = pfssproc->fss_upri;
1887 	cfssproc->fss_tp = ct;
1888 	cfssproc->fss_nice = pfssproc->fss_nice;
1889 	cpucaps_sc_init(&cfssproc->fss_caps);
1890 
1891 	cfssproc->fss_flags =
1892 	    pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
1893 	ct->t_cldata = (void *)cfssproc;
1894 	ct->t_schedflag |= TS_RUNQMATCH;
1895 	thread_unlock(pt);
1896 
1897 	fssproj->fssp_threads++;
1898 	mutex_exit(&fsspset->fssps_lock);
1899 
1900 	/*
1901 	 * Link new structure into fssproc hash table.
1902 	 */
1903 	FSS_LIST_INSERT(cfssproc);
1904 	return (0);
1905 }
1906 
1907 /*
1908  * Child is placed at back of dispatcher queue and parent gives up processor
1909  * so that the child runs first after the fork. This allows the child
1910  * immediately execing to break the multiple use of copy on write pages with no
1911  * disk home. The parent will get to steal them back rather than uselessly
1912  * copying them.
1913  */
1914 static void
fss_forkret(kthread_t * t,kthread_t * ct)1915 fss_forkret(kthread_t *t, kthread_t *ct)
1916 {
1917 	proc_t *pp = ttoproc(t);
1918 	proc_t *cp = ttoproc(ct);
1919 	fssproc_t *fssproc;
1920 
1921 	ASSERT(t == curthread);
1922 	ASSERT(MUTEX_HELD(&pidlock));
1923 
1924 	/*
1925 	 * Grab the child's p_lock before dropping pidlock to ensure the
1926 	 * process does not disappear before we set it running.
1927 	 */
1928 	mutex_enter(&cp->p_lock);
1929 	continuelwps(cp);
1930 	mutex_exit(&cp->p_lock);
1931 
1932 	mutex_enter(&pp->p_lock);
1933 	mutex_exit(&pidlock);
1934 	continuelwps(pp);
1935 
1936 	thread_lock(t);
1937 
1938 	fssproc = FSSPROC(t);
1939 	fss_newpri(fssproc, B_FALSE);
1940 	fssproc->fss_timeleft = fss_quantum;
1941 	t->t_pri = fssproc->fss_umdpri;
1942 	ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1943 	fssproc->fss_flags &= ~FSSKPRI;
1944 	THREAD_TRANSITION(t);
1945 
1946 	/*
1947 	 * We don't want to call fss_setrun(t) here because it may call
1948 	 * fss_active, which we don't need.
1949 	 */
1950 	fssproc->fss_flags &= ~FSSBACKQ;
1951 
1952 	if (t->t_disp_time != ddi_get_lbolt())
1953 		setbackdq(t);
1954 	else
1955 		setfrontdq(t);
1956 
1957 	thread_unlock(t);
1958 	/*
1959 	 * Safe to drop p_lock now since it is safe to change
1960 	 * the scheduling class after this point.
1961 	 */
1962 	mutex_exit(&pp->p_lock);
1963 
1964 	swtch();
1965 }
1966 
1967 /*
1968  * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1969  * the buffer pointed by fssparmsp.
1970  */
1971 static void
fss_parmsget(kthread_t * t,void * parmsp)1972 fss_parmsget(kthread_t *t, void *parmsp)
1973 {
1974 	fssproc_t *fssproc = FSSPROC(t);
1975 	fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1976 
1977 	fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1978 	fssparmsp->fss_upri = fssproc->fss_upri;
1979 }
1980 
1981 /*ARGSUSED*/
1982 static int
fss_parmsset(kthread_t * t,void * parmsp,id_t reqpcid,cred_t * reqpcredp)1983 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1984 {
1985 	char		nice;
1986 	pri_t		reqfssuprilim;
1987 	pri_t		reqfssupri;
1988 	fssproc_t	*fssproc = FSSPROC(t);
1989 	fssparms_t	*fssparmsp = (fssparms_t *)parmsp;
1990 
1991 	ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1992 
1993 	if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1994 		reqfssuprilim = fssproc->fss_uprilim;
1995 	else
1996 		reqfssuprilim = fssparmsp->fss_uprilim;
1997 
1998 	if (fssparmsp->fss_upri == FSS_NOCHANGE)
1999 		reqfssupri = fssproc->fss_upri;
2000 	else
2001 		reqfssupri = fssparmsp->fss_upri;
2002 
2003 	/*
2004 	 * Make sure the user priority doesn't exceed the upri limit.
2005 	 */
2006 	if (reqfssupri > reqfssuprilim)
2007 		reqfssupri = reqfssuprilim;
2008 
2009 	/*
2010 	 * Basic permissions enforced by generic kernel code for all classes
2011 	 * require that a thread attempting to change the scheduling parameters
2012 	 * of a target thread be privileged or have a real or effective UID
2013 	 * matching that of the target thread. We are not called unless these
2014 	 * basic permission checks have already passed. The fair-sharing class
2015 	 * requires in addition that the calling thread be privileged if it
2016 	 * is attempting to raise the upri limit above its current value.
2017 	 * This may have been checked previously but if our caller passed us
2018 	 * a non-NULL credential pointer we assume it hasn't and we check it
2019 	 * here.
2020 	 */
2021 	if ((reqpcredp != NULL) &&
2022 	    (reqfssuprilim > fssproc->fss_uprilim) &&
2023 	    secpolicy_raisepriority(reqpcredp) != 0)
2024 		return (EPERM);
2025 
2026 	/*
2027 	 * Set fss_nice to the nice value corresponding to the user priority we
2028 	 * are setting.  Note that setting the nice field of the parameter
2029 	 * struct won't affect upri or nice.
2030 	 */
2031 	nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
2032 	if (nice > FSS_NICE_MAX)
2033 		nice = FSS_NICE_MAX;
2034 
2035 	thread_lock(t);
2036 
2037 	fssproc->fss_uprilim = reqfssuprilim;
2038 	fssproc->fss_upri = reqfssupri;
2039 	fssproc->fss_nice = nice;
2040 	fss_newpri(fssproc, B_FALSE);
2041 
2042 	if ((fssproc->fss_flags & FSSKPRI) != 0) {
2043 		thread_unlock(t);
2044 		return (0);
2045 	}
2046 
2047 	fss_change_priority(t, fssproc);
2048 	thread_unlock(t);
2049 	return (0);
2050 
2051 }
2052 
2053 /*
2054  * The thread is being stopped.
2055  */
2056 /*ARGSUSED*/
2057 static void
fss_stop(kthread_t * t,int why,int what)2058 fss_stop(kthread_t *t, int why, int what)
2059 {
2060 	ASSERT(THREAD_LOCK_HELD(t));
2061 	ASSERT(t == curthread);
2062 
2063 	fss_inactive(t);
2064 }
2065 
2066 /*
2067  * The current thread is exiting, do necessary adjustments to its project
2068  */
2069 static void
fss_exit(kthread_t * t)2070 fss_exit(kthread_t *t)
2071 {
2072 	fsspset_t *fsspset;
2073 	fssproj_t *fssproj;
2074 	fssproc_t *fssproc;
2075 	fsszone_t *fsszone;
2076 	int free = 0;
2077 
2078 	/*
2079 	 * Thread t here is either a current thread (in which case we hold
2080 	 * its process' p_lock), or a thread being destroyed by forklwp_fail(),
2081 	 * in which case we hold pidlock and thread is no longer on the
2082 	 * thread list.
2083 	 */
2084 	ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
2085 
2086 	fssproc = FSSPROC(t);
2087 	fssproj = FSSPROC2FSSPROJ(fssproc);
2088 	fsspset = FSSPROJ2FSSPSET(fssproj);
2089 	fsszone = fssproj->fssp_fsszone;
2090 
2091 	mutex_enter(&fsspsets_lock);
2092 	mutex_enter(&fsspset->fssps_lock);
2093 
2094 	thread_lock(t);
2095 	disp_lock_enter_high(&fsspset->fssps_displock);
2096 	if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
2097 		if (--fssproj->fssp_runnable == 0) {
2098 			fsszone->fssz_shares -= fssproj->fssp_shares;
2099 			if (--fsszone->fssz_runnable == 0)
2100 				fsspset->fssps_shares -= fsszone->fssz_rshares;
2101 		}
2102 		ASSERT(fssproc->fss_runnable == 1);
2103 		fssproc->fss_runnable = 0;
2104 	}
2105 	if (--fssproj->fssp_threads == 0) {
2106 		fss_remove_fssproj(fsspset, fssproj);
2107 		free = 1;
2108 	}
2109 	disp_lock_exit_high(&fsspset->fssps_displock);
2110 	fssproc->fss_proj = NULL;	/* mark this thread as already exited */
2111 	thread_unlock(t);
2112 
2113 	if (free) {
2114 		if (fsszone->fssz_nproj == 0)
2115 			kmem_free(fsszone, sizeof (fsszone_t));
2116 		kmem_free(fssproj, sizeof (fssproj_t));
2117 	}
2118 	mutex_exit(&fsspset->fssps_lock);
2119 	mutex_exit(&fsspsets_lock);
2120 
2121 	/*
2122 	 * A thread could be exiting in between clock ticks, so we need to
2123 	 * calculate how much CPU time it used since it was charged last time.
2124 	 *
2125 	 * CPU caps are not enforced on exiting processes - it is usually
2126 	 * desirable to exit as soon as possible to free resources.
2127 	 */
2128 	if (CPUCAPS_ON()) {
2129 		thread_lock(t);
2130 		fssproc = FSSPROC(t);
2131 		(void) cpucaps_charge(t, &fssproc->fss_caps,
2132 		    CPUCAPS_CHARGE_ONLY);
2133 		thread_unlock(t);
2134 	}
2135 }
2136 
2137 static void
fss_nullsys()2138 fss_nullsys()
2139 {
2140 }
2141 
2142 /*
2143  * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
2144  * swapped in. Otherwise, it returns the thread's effective priority based
2145  * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
2146  */
2147 /*ARGSUSED*/
2148 static pri_t
fss_swapin(kthread_t * t,int flags)2149 fss_swapin(kthread_t *t, int flags)
2150 {
2151 	fssproc_t *fssproc = FSSPROC(t);
2152 	long epri = -1;
2153 	proc_t *pp = ttoproc(t);
2154 
2155 	ASSERT(THREAD_LOCK_HELD(t));
2156 
2157 	if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
2158 		time_t swapout_time;
2159 
2160 		swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
2161 		if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) {
2162 			epri = (long)DISP_PRIO(t) + swapout_time;
2163 		} else {
2164 			/*
2165 			 * Threads which have been out for a long time,
2166 			 * have high user mode priority and are associated
2167 			 * with a small address space are more deserving.
2168 			 */
2169 			epri = fssproc->fss_umdpri;
2170 			ASSERT(epri >= 0 && epri <= fss_maxumdpri);
2171 			epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
2172 		}
2173 		/*
2174 		 * Scale epri so that SHRT_MAX / 2 represents zero priority.
2175 		 */
2176 		epri += SHRT_MAX / 2;
2177 		if (epri < 0)
2178 			epri = 0;
2179 		else if (epri > SHRT_MAX)
2180 			epri = SHRT_MAX;
2181 	}
2182 	return ((pri_t)epri);
2183 }
2184 
2185 /*
2186  * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
2187  * be swapped out. Otherwise, it returns the thread's effective priority
2188  * based on if the swapper is in softswap or hardswap mode.
2189  */
2190 static pri_t
fss_swapout(kthread_t * t,int flags)2191 fss_swapout(kthread_t *t, int flags)
2192 {
2193 	fssproc_t *fssproc = FSSPROC(t);
2194 	long epri = -1;
2195 	proc_t *pp = ttoproc(t);
2196 	time_t swapin_time;
2197 
2198 	ASSERT(THREAD_LOCK_HELD(t));
2199 
2200 	if (INHERITED(t) ||
2201 	    (fssproc->fss_flags & FSSKPRI) ||
2202 	    (t->t_proc_flag & TP_LWPEXIT) ||
2203 	    (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
2204 	    !(t->t_schedflag & TS_LOAD) ||
2205 	    !(SWAP_OK(t)))
2206 		return (-1);
2207 
2208 	ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
2209 
2210 	swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
2211 
2212 	if (flags == SOFTSWAP) {
2213 		if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
2214 			epri = 0;
2215 		} else {
2216 			return ((pri_t)epri);
2217 		}
2218 	} else {
2219 		pri_t pri;
2220 
2221 		if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
2222 		    (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
2223 			pri = fss_maxumdpri;
2224 			epri = swapin_time -
2225 			    (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
2226 		} else {
2227 			return ((pri_t)epri);
2228 		}
2229 	}
2230 
2231 	/*
2232 	 * Scale epri so that SHRT_MAX / 2 represents zero priority.
2233 	 */
2234 	epri += SHRT_MAX / 2;
2235 	if (epri < 0)
2236 		epri = 0;
2237 	else if (epri > SHRT_MAX)
2238 		epri = SHRT_MAX;
2239 
2240 	return ((pri_t)epri);
2241 }
2242 
2243 /*
2244  * If thread is currently at a kernel mode priority (has slept) and is
2245  * returning to the userland we assign it the appropriate user mode priority
2246  * and time quantum here.  If we're lowering the thread's priority below that
2247  * of other runnable threads then we will set runrun via cpu_surrender() to
2248  * cause preemption.
2249  */
2250 static void
fss_trapret(kthread_t * t)2251 fss_trapret(kthread_t *t)
2252 {
2253 	fssproc_t *fssproc = FSSPROC(t);
2254 	cpu_t *cp = CPU;
2255 
2256 	ASSERT(THREAD_LOCK_HELD(t));
2257 	ASSERT(t == curthread);
2258 	ASSERT(cp->cpu_dispthread == t);
2259 	ASSERT(t->t_state == TS_ONPROC);
2260 
2261 	t->t_kpri_req = 0;
2262 	if (fssproc->fss_flags & FSSKPRI) {
2263 		/*
2264 		 * If thread has blocked in the kernel
2265 		 */
2266 		THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2267 		cp->cpu_dispatch_pri = DISP_PRIO(t);
2268 		ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2269 		fssproc->fss_flags &= ~FSSKPRI;
2270 
2271 		if (DISP_MUST_SURRENDER(t))
2272 			cpu_surrender(t);
2273 	}
2274 
2275 	/*
2276 	 * Swapout lwp if the swapper is waiting for this thread to reach
2277 	 * a safe point.
2278 	 */
2279 	if (t->t_schedflag & TS_SWAPENQ) {
2280 		thread_unlock(t);
2281 		swapout_lwp(ttolwp(t));
2282 		thread_lock(t);
2283 	}
2284 }
2285 
2286 /*
2287  * Arrange for thread to be placed in appropriate location on dispatcher queue.
2288  * This is called with the current thread in TS_ONPROC and locked.
2289  */
2290 static void
fss_preempt(kthread_t * t)2291 fss_preempt(kthread_t *t)
2292 {
2293 	fssproc_t *fssproc = FSSPROC(t);
2294 	klwp_t *lwp;
2295 	uint_t flags;
2296 
2297 	ASSERT(t == curthread);
2298 	ASSERT(THREAD_LOCK_HELD(curthread));
2299 	ASSERT(t->t_state == TS_ONPROC);
2300 
2301 	/*
2302 	 * If preempted in the kernel, make sure the thread has a kernel
2303 	 * priority if needed.
2304 	 */
2305 	lwp = curthread->t_lwp;
2306 	if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
2307 		fssproc->fss_flags |= FSSKPRI;
2308 		THREAD_CHANGE_PRI(t, minclsyspri);
2309 		ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2310 		t->t_trapret = 1;	/* so that fss_trapret will run */
2311 		aston(t);
2312 	}
2313 
2314 	/*
2315 	 * This thread may be placed on wait queue by CPU Caps. In this case we
2316 	 * do not need to do anything until it is removed from the wait queue.
2317 	 * Do not enforce CPU caps on threads running at a kernel priority
2318 	 */
2319 	if (CPUCAPS_ON()) {
2320 		(void) cpucaps_charge(t, &fssproc->fss_caps,
2321 		    CPUCAPS_CHARGE_ENFORCE);
2322 
2323 		if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2324 			return;
2325 	}
2326 
2327 	/*
2328 	 * If preempted in user-land mark the thread as swappable because it
2329 	 * cannot be holding any kernel locks.
2330 	 */
2331 	ASSERT(t->t_schedflag & TS_DONT_SWAP);
2332 	if (lwp != NULL && lwp->lwp_state == LWP_USER)
2333 		t->t_schedflag &= ~TS_DONT_SWAP;
2334 
2335 	/*
2336 	 * Check to see if we're doing "preemption control" here.  If
2337 	 * we are, and if the user has requested that this thread not
2338 	 * be preempted, and if preemptions haven't been put off for
2339 	 * too long, let the preemption happen here but try to make
2340 	 * sure the thread is rescheduled as soon as possible.  We do
2341 	 * this by putting it on the front of the highest priority run
2342 	 * queue in the FSS class.  If the preemption has been put off
2343 	 * for too long, clear the "nopreempt" bit and let the thread
2344 	 * be preempted.
2345 	 */
2346 	if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2347 		if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2348 			DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2349 			if (!(fssproc->fss_flags & FSSKPRI)) {
2350 				/*
2351 				 * If not already remembered, remember current
2352 				 * priority for restoration in fss_yield().
2353 				 */
2354 				if (!(fssproc->fss_flags & FSSRESTORE)) {
2355 					fssproc->fss_scpri = t->t_pri;
2356 					fssproc->fss_flags |= FSSRESTORE;
2357 				}
2358 				THREAD_CHANGE_PRI(t, fss_maxumdpri);
2359 				t->t_schedflag |= TS_DONT_SWAP;
2360 			}
2361 			schedctl_set_yield(t, 1);
2362 			setfrontdq(t);
2363 			return;
2364 		} else {
2365 			if (fssproc->fss_flags & FSSRESTORE) {
2366 				THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2367 				fssproc->fss_flags &= ~FSSRESTORE;
2368 			}
2369 			schedctl_set_nopreempt(t, 0);
2370 			DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2371 			/*
2372 			 * Fall through and be preempted below.
2373 			 */
2374 		}
2375 	}
2376 
2377 	flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2378 
2379 	if (flags == FSSBACKQ) {
2380 		fssproc->fss_timeleft = fss_quantum;
2381 		fssproc->fss_flags &= ~FSSBACKQ;
2382 		setbackdq(t);
2383 	} else if (flags == (FSSBACKQ | FSSKPRI)) {
2384 		fssproc->fss_flags &= ~FSSBACKQ;
2385 		setbackdq(t);
2386 	} else {
2387 		setfrontdq(t);
2388 	}
2389 }
2390 
2391 /*
2392  * Called when a thread is waking up and is to be placed on the run queue.
2393  */
2394 static void
fss_setrun(kthread_t * t)2395 fss_setrun(kthread_t *t)
2396 {
2397 	fssproc_t *fssproc = FSSPROC(t);
2398 
2399 	ASSERT(THREAD_LOCK_HELD(t));	/* t should be in transition */
2400 
2401 	if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
2402 		fss_active(t);
2403 
2404 	fssproc->fss_timeleft = fss_quantum;
2405 
2406 	fssproc->fss_flags &= ~FSSBACKQ;
2407 	/*
2408 	 * If previously were running at the kernel priority then keep that
2409 	 * priority and the fss_timeleft doesn't matter.
2410 	 */
2411 	if ((fssproc->fss_flags & FSSKPRI) == 0)
2412 		THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2413 
2414 	if (t->t_disp_time != ddi_get_lbolt())
2415 		setbackdq(t);
2416 	else
2417 		setfrontdq(t);
2418 }
2419 
2420 /*
2421  * Prepare thread for sleep. We reset the thread priority so it will run at the
2422  * kernel priority level when it wakes up.
2423  */
2424 static void
fss_sleep(kthread_t * t)2425 fss_sleep(kthread_t *t)
2426 {
2427 	fssproc_t *fssproc = FSSPROC(t);
2428 
2429 	ASSERT(t == curthread);
2430 	ASSERT(THREAD_LOCK_HELD(t));
2431 
2432 	ASSERT(t->t_state == TS_ONPROC);
2433 
2434 	/*
2435 	 * Account for time spent on CPU before going to sleep.
2436 	 */
2437 	(void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2438 
2439 	fss_inactive(t);
2440 
2441 	/*
2442 	 * Assign a system priority to the thread and arrange for it to be
2443 	 * retained when the thread is next placed on the run queue (i.e.,
2444 	 * when it wakes up) instead of being given a new pri.  Also arrange
2445 	 * for trapret processing as the thread leaves the system call so it
2446 	 * will drop back to normal priority range.
2447 	 */
2448 	if (t->t_kpri_req) {
2449 		THREAD_CHANGE_PRI(t, minclsyspri);
2450 		fssproc->fss_flags |= FSSKPRI;
2451 		t->t_trapret = 1;	/* so that fss_trapret will run */
2452 		aston(t);
2453 	} else if (fssproc->fss_flags & FSSKPRI) {
2454 		/*
2455 		 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2456 		 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2457 		 * then slept again all without finishing the current system
2458 		 * call so trapret won't have cleared FSSKPRI
2459 		 */
2460 		fssproc->fss_flags &= ~FSSKPRI;
2461 		THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2462 		if (DISP_MUST_SURRENDER(curthread))
2463 			cpu_surrender(t);
2464 	}
2465 	t->t_stime = ddi_get_lbolt();	/* time stamp for the swapper */
2466 }
2467 
2468 /*
2469  * A tick interrupt has ocurrend on a running thread. Check to see if our
2470  * time slice has expired.  We must also clear the TS_DONT_SWAP flag in
2471  * t_schedflag if the thread is eligible to be swapped out.
2472  */
2473 static void
fss_tick(kthread_t * t)2474 fss_tick(kthread_t *t)
2475 {
2476 	fssproc_t *fssproc;
2477 	fssproj_t *fssproj;
2478 	klwp_t *lwp;
2479 	boolean_t call_cpu_surrender = B_FALSE;
2480 	boolean_t cpucaps_enforce = B_FALSE;
2481 
2482 	ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2483 
2484 	/*
2485 	 * It's safe to access fsspset and fssproj structures because we're
2486 	 * holding our p_lock here.
2487 	 */
2488 	thread_lock(t);
2489 	fssproc = FSSPROC(t);
2490 	fssproj = FSSPROC2FSSPROJ(fssproc);
2491 	if (fssproj != NULL) {
2492 		fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2493 		disp_lock_enter_high(&fsspset->fssps_displock);
2494 		fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2495 		fssproj->fssp_tick_cnt++;
2496 		fssproc->fss_ticks++;
2497 		disp_lock_exit_high(&fsspset->fssps_displock);
2498 	}
2499 
2500 	/*
2501 	 * Keep track of thread's project CPU usage.  Note that projects
2502 	 * get charged even when threads are running in the kernel.
2503 	 * Do not surrender CPU if running in the SYS class.
2504 	 */
2505 	if (CPUCAPS_ON()) {
2506 		cpucaps_enforce = cpucaps_charge(t,
2507 		    &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
2508 		    !(fssproc->fss_flags & FSSKPRI);
2509 	}
2510 
2511 	/*
2512 	 * A thread's execution time for threads running in the SYS class
2513 	 * is not tracked.
2514 	 */
2515 	if ((fssproc->fss_flags & FSSKPRI) == 0) {
2516 		/*
2517 		 * If thread is not in kernel mode, decrement its fss_timeleft
2518 		 */
2519 		if (--fssproc->fss_timeleft <= 0) {
2520 			pri_t new_pri;
2521 
2522 			/*
2523 			 * If we're doing preemption control and trying to
2524 			 * avoid preempting this thread, just note that the
2525 			 * thread should yield soon and let it keep running
2526 			 * (unless it's been a while).
2527 			 */
2528 			if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2529 				if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2530 					DTRACE_SCHED1(schedctl__nopreempt,
2531 					    kthread_t *, t);
2532 					schedctl_set_yield(t, 1);
2533 					thread_unlock_nopreempt(t);
2534 					return;
2535 				}
2536 			}
2537 			fssproc->fss_flags &= ~FSSRESTORE;
2538 
2539 			fss_newpri(fssproc, B_TRUE);
2540 			new_pri = fssproc->fss_umdpri;
2541 			ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2542 
2543 			/*
2544 			 * When the priority of a thread is changed, it may
2545 			 * be necessary to adjust its position on a sleep queue
2546 			 * or dispatch queue. The function thread_change_pri
2547 			 * accomplishes this.
2548 			 */
2549 			if (thread_change_pri(t, new_pri, 0)) {
2550 				if ((t->t_schedflag & TS_LOAD) &&
2551 				    (lwp = t->t_lwp) &&
2552 				    lwp->lwp_state == LWP_USER)
2553 					t->t_schedflag &= ~TS_DONT_SWAP;
2554 				fssproc->fss_timeleft = fss_quantum;
2555 			} else {
2556 				call_cpu_surrender = B_TRUE;
2557 			}
2558 		} else if (t->t_state == TS_ONPROC &&
2559 		    t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2560 			/*
2561 			 * If there is a higher-priority thread which is
2562 			 * waiting for a processor, then thread surrenders
2563 			 * the processor.
2564 			 */
2565 			call_cpu_surrender = B_TRUE;
2566 		}
2567 	}
2568 
2569 	if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2570 		/*
2571 		 * The thread used more than half of its quantum, so assume that
2572 		 * it used the whole quantum.
2573 		 *
2574 		 * Update thread's priority just before putting it on the wait
2575 		 * queue so that it gets charged for the CPU time from its
2576 		 * quantum even before that quantum expires.
2577 		 */
2578 		fss_newpri(fssproc, B_FALSE);
2579 		if (t->t_pri != fssproc->fss_umdpri)
2580 			fss_change_priority(t, fssproc);
2581 
2582 		/*
2583 		 * We need to call cpu_surrender for this thread due to cpucaps
2584 		 * enforcement, but fss_change_priority may have already done
2585 		 * so. In this case FSSBACKQ is set and there is no need to call
2586 		 * cpu-surrender again.
2587 		 */
2588 		if (!(fssproc->fss_flags & FSSBACKQ))
2589 			call_cpu_surrender = B_TRUE;
2590 	}
2591 
2592 	if (call_cpu_surrender) {
2593 		fssproc->fss_flags |= FSSBACKQ;
2594 		cpu_surrender(t);
2595 	}
2596 
2597 	thread_unlock_nopreempt(t);	/* clock thread can't be preempted */
2598 }
2599 
2600 /*
2601  * Processes waking up go to the back of their queue.  We don't need to assign
2602  * a time quantum here because thread is still at a kernel mode priority and
2603  * the time slicing is not done for threads running in the kernel after
2604  * sleeping.  The proper time quantum will be assigned by fss_trapret before the
2605  * thread returns to user mode.
2606  */
2607 static void
fss_wakeup(kthread_t * t)2608 fss_wakeup(kthread_t *t)
2609 {
2610 	fssproc_t *fssproc;
2611 
2612 	ASSERT(THREAD_LOCK_HELD(t));
2613 	ASSERT(t->t_state == TS_SLEEP);
2614 
2615 	fss_active(t);
2616 
2617 	t->t_stime = ddi_get_lbolt();		/* time stamp for the swapper */
2618 	fssproc = FSSPROC(t);
2619 	fssproc->fss_flags &= ~FSSBACKQ;
2620 
2621 	if (fssproc->fss_flags & FSSKPRI) {
2622 		/*
2623 		 * If we already have a kernel priority assigned, then we
2624 		 * just use it.
2625 		 */
2626 		setbackdq(t);
2627 	} else if (t->t_kpri_req) {
2628 		/*
2629 		 * Give thread a priority boost if we were asked.
2630 		 */
2631 		fssproc->fss_flags |= FSSKPRI;
2632 		THREAD_CHANGE_PRI(t, minclsyspri);
2633 		setbackdq(t);
2634 		t->t_trapret = 1;	/* so that fss_trapret will run */
2635 		aston(t);
2636 	} else {
2637 		/*
2638 		 * Otherwise, we recalculate the priority.
2639 		 */
2640 		if (t->t_disp_time == ddi_get_lbolt()) {
2641 			setfrontdq(t);
2642 		} else {
2643 			fssproc->fss_timeleft = fss_quantum;
2644 			THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2645 			setbackdq(t);
2646 		}
2647 	}
2648 }
2649 
2650 /*
2651  * fss_donice() is called when a nice(1) command is issued on the thread to
2652  * alter the priority. The nice(1) command exists in Solaris for compatibility.
2653  * Thread priority adjustments should be done via priocntl(1).
2654  */
2655 static int
fss_donice(kthread_t * t,cred_t * cr,int incr,int * retvalp)2656 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2657 {
2658 	int newnice;
2659 	fssproc_t *fssproc = FSSPROC(t);
2660 	fssparms_t fssparms;
2661 
2662 	/*
2663 	 * If there is no change to priority, just return current setting.
2664 	 */
2665 	if (incr == 0) {
2666 		if (retvalp)
2667 			*retvalp = fssproc->fss_nice - NZERO;
2668 		return (0);
2669 	}
2670 
2671 	if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2672 		return (EPERM);
2673 
2674 	/*
2675 	 * Specifying a nice increment greater than the upper limit of
2676 	 * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2677 	 * value being set to the upper limit.  We check for this before
2678 	 * computing the new value because otherwise we could get overflow
2679 	 * if a privileged user specified some ridiculous increment.
2680 	 */
2681 	if (incr > FSS_NICE_MAX)
2682 		incr = FSS_NICE_MAX;
2683 
2684 	newnice = fssproc->fss_nice + incr;
2685 	if (newnice > FSS_NICE_MAX)
2686 		newnice = FSS_NICE_MAX;
2687 	else if (newnice < FSS_NICE_MIN)
2688 		newnice = FSS_NICE_MIN;
2689 
2690 	fssparms.fss_uprilim = fssparms.fss_upri =
2691 	    -((newnice - NZERO) * fss_maxupri) / NZERO;
2692 
2693 	/*
2694 	 * Reset the uprilim and upri values of the thread.
2695 	 */
2696 	(void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2697 
2698 	/*
2699 	 * Although fss_parmsset already reset fss_nice it may not have been
2700 	 * set to precisely the value calculated above because fss_parmsset
2701 	 * determines the nice value from the user priority and we may have
2702 	 * truncated during the integer conversion from nice value to user
2703 	 * priority and back. We reset fss_nice to the value we calculated
2704 	 * above.
2705 	 */
2706 	fssproc->fss_nice = (char)newnice;
2707 
2708 	if (retvalp)
2709 		*retvalp = newnice - NZERO;
2710 	return (0);
2711 }
2712 
2713 /*
2714  * Increment the priority of the specified thread by incr and
2715  * return the new value in *retvalp.
2716  */
2717 static int
fss_doprio(kthread_t * t,cred_t * cr,int incr,int * retvalp)2718 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2719 {
2720 	int newpri;
2721 	fssproc_t *fssproc = FSSPROC(t);
2722 	fssparms_t fssparms;
2723 
2724 	/*
2725 	 * If there is no change to priority, just return current setting.
2726 	 */
2727 	if (incr == 0) {
2728 		*retvalp = fssproc->fss_upri;
2729 		return (0);
2730 	}
2731 
2732 	newpri = fssproc->fss_upri + incr;
2733 	if (newpri > fss_maxupri || newpri < -fss_maxupri)
2734 		return (EINVAL);
2735 
2736 	*retvalp = newpri;
2737 	fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2738 
2739 	/*
2740 	 * Reset the uprilim and upri values of the thread.
2741 	 */
2742 	return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2743 }
2744 
2745 /*
2746  * Return the global scheduling priority that would be assigned to a thread
2747  * entering the fair-sharing class with the fss_upri.
2748  */
2749 /*ARGSUSED*/
2750 static pri_t
fss_globpri(kthread_t * t)2751 fss_globpri(kthread_t *t)
2752 {
2753 	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2754 
2755 	return (fss_maxumdpri / 2);
2756 }
2757 
2758 /*
2759  * Called from the yield(2) system call when a thread is yielding (surrendering)
2760  * the processor. The kernel thread is placed at the back of a dispatch queue.
2761  */
2762 static void
fss_yield(kthread_t * t)2763 fss_yield(kthread_t *t)
2764 {
2765 	fssproc_t *fssproc = FSSPROC(t);
2766 
2767 	ASSERT(t == curthread);
2768 	ASSERT(THREAD_LOCK_HELD(t));
2769 
2770 	/*
2771 	 * Collect CPU usage spent before yielding
2772 	 */
2773 	(void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2774 
2775 	/*
2776 	 * Clear the preemption control "yield" bit since the user is
2777 	 * doing a yield.
2778 	 */
2779 	if (t->t_schedctl)
2780 		schedctl_set_yield(t, 0);
2781 	/*
2782 	 * If fss_preempt() artifically increased the thread's priority
2783 	 * to avoid preemption, restore the original priority now.
2784 	 */
2785 	if (fssproc->fss_flags & FSSRESTORE) {
2786 		THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2787 		fssproc->fss_flags &= ~FSSRESTORE;
2788 	}
2789 	if (fssproc->fss_timeleft < 0) {
2790 		/*
2791 		 * Time slice was artificially extended to avoid preemption,
2792 		 * so pretend we're preempting it now.
2793 		 */
2794 		DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2795 		fssproc->fss_timeleft = fss_quantum;
2796 	}
2797 	fssproc->fss_flags &= ~FSSBACKQ;
2798 	setbackdq(t);
2799 }
2800 
2801 void
fss_changeproj(kthread_t * t,void * kp,void * zp,fssbuf_t * projbuf,fssbuf_t * zonebuf)2802 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2803     fssbuf_t *zonebuf)
2804 {
2805 	kproject_t *kpj_new = kp;
2806 	zone_t *zone = zp;
2807 	fssproj_t *fssproj_old, *fssproj_new;
2808 	fsspset_t *fsspset;
2809 	kproject_t *kpj_old;
2810 	fssproc_t *fssproc;
2811 	fsszone_t *fsszone_old, *fsszone_new;
2812 	int free = 0;
2813 	int id;
2814 
2815 	ASSERT(MUTEX_HELD(&cpu_lock));
2816 	ASSERT(MUTEX_HELD(&pidlock));
2817 	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2818 
2819 	if (t->t_cid != fss_cid)
2820 		return;
2821 
2822 	fssproc = FSSPROC(t);
2823 	mutex_enter(&fsspsets_lock);
2824 	fssproj_old = FSSPROC2FSSPROJ(fssproc);
2825 	if (fssproj_old == NULL) {
2826 		mutex_exit(&fsspsets_lock);
2827 		return;
2828 	}
2829 
2830 	fsspset = FSSPROJ2FSSPSET(fssproj_old);
2831 	mutex_enter(&fsspset->fssps_lock);
2832 	kpj_old = FSSPROJ2KPROJ(fssproj_old);
2833 	fsszone_old = fssproj_old->fssp_fsszone;
2834 
2835 	ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2836 
2837 	if (kpj_old == kpj_new) {
2838 		mutex_exit(&fsspset->fssps_lock);
2839 		mutex_exit(&fsspsets_lock);
2840 		return;
2841 	}
2842 
2843 	if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2844 		/*
2845 		 * If the zone for the new project is not currently active on
2846 		 * the cpu partition we're on, get one of the pre-allocated
2847 		 * buffers and link it in our per-pset zone list.  Such buffers
2848 		 * should already exist.
2849 		 */
2850 		for (id = 0; id < zonebuf->fssb_size; id++) {
2851 			if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2852 				fss_insert_fsszone(fsspset, zone, fsszone_new);
2853 				zonebuf->fssb_list[id] = NULL;
2854 				break;
2855 			}
2856 		}
2857 	}
2858 	ASSERT(fsszone_new != NULL);
2859 	if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2860 		/*
2861 		 * If our new project is not currently running
2862 		 * on the cpu partition we're on, get one of the
2863 		 * pre-allocated buffers and link it in our new cpu
2864 		 * partition doubly linked list. Such buffers should already
2865 		 * exist.
2866 		 */
2867 		for (id = 0; id < projbuf->fssb_size; id++) {
2868 			if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2869 				fss_insert_fssproj(fsspset, kpj_new,
2870 				    fsszone_new, fssproj_new);
2871 				projbuf->fssb_list[id] = NULL;
2872 				break;
2873 			}
2874 		}
2875 	}
2876 	ASSERT(fssproj_new != NULL);
2877 
2878 	thread_lock(t);
2879 	if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2880 	    t->t_state == TS_WAIT)
2881 		fss_inactive(t);
2882 	ASSERT(fssproj_old->fssp_threads > 0);
2883 	if (--fssproj_old->fssp_threads == 0) {
2884 		fss_remove_fssproj(fsspset, fssproj_old);
2885 		free = 1;
2886 	}
2887 	fssproc->fss_proj = fssproj_new;
2888 	fssproc->fss_fsspri = 0;
2889 	fssproj_new->fssp_threads++;
2890 	if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2891 	    t->t_state == TS_WAIT)
2892 		fss_active(t);
2893 	thread_unlock(t);
2894 	if (free) {
2895 		if (fsszone_old->fssz_nproj == 0)
2896 			kmem_free(fsszone_old, sizeof (fsszone_t));
2897 		kmem_free(fssproj_old, sizeof (fssproj_t));
2898 	}
2899 
2900 	mutex_exit(&fsspset->fssps_lock);
2901 	mutex_exit(&fsspsets_lock);
2902 }
2903 
2904 void
fss_changepset(kthread_t * t,void * newcp,fssbuf_t * projbuf,fssbuf_t * zonebuf)2905 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2906     fssbuf_t *zonebuf)
2907 {
2908 	fsspset_t *fsspset_old, *fsspset_new;
2909 	fssproj_t *fssproj_old, *fssproj_new;
2910 	fsszone_t *fsszone_old, *fsszone_new;
2911 	fssproc_t *fssproc;
2912 	kproject_t *kpj;
2913 	zone_t *zone;
2914 	int id;
2915 
2916 	ASSERT(MUTEX_HELD(&cpu_lock));
2917 	ASSERT(MUTEX_HELD(&pidlock));
2918 	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2919 
2920 	if (t->t_cid != fss_cid)
2921 		return;
2922 
2923 	fssproc = FSSPROC(t);
2924 	zone = ttoproc(t)->p_zone;
2925 	mutex_enter(&fsspsets_lock);
2926 	fssproj_old = FSSPROC2FSSPROJ(fssproc);
2927 	if (fssproj_old == NULL) {
2928 		mutex_exit(&fsspsets_lock);
2929 		return;
2930 	}
2931 	fsszone_old = fssproj_old->fssp_fsszone;
2932 	fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2933 	kpj = FSSPROJ2KPROJ(fssproj_old);
2934 
2935 	if (fsspset_old->fssps_cpupart == newcp) {
2936 		mutex_exit(&fsspsets_lock);
2937 		return;
2938 	}
2939 
2940 	ASSERT(ttoproj(t) == kpj);
2941 
2942 	fsspset_new = fss_find_fsspset(newcp);
2943 
2944 	mutex_enter(&fsspset_new->fssps_lock);
2945 	if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2946 		for (id = 0; id < zonebuf->fssb_size; id++) {
2947 			if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2948 				fss_insert_fsszone(fsspset_new, zone,
2949 				    fsszone_new);
2950 				zonebuf->fssb_list[id] = NULL;
2951 				break;
2952 			}
2953 		}
2954 	}
2955 	ASSERT(fsszone_new != NULL);
2956 	if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2957 		for (id = 0; id < projbuf->fssb_size; id++) {
2958 			if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2959 				fss_insert_fssproj(fsspset_new, kpj,
2960 				    fsszone_new, fssproj_new);
2961 				projbuf->fssb_list[id] = NULL;
2962 				break;
2963 			}
2964 		}
2965 	}
2966 	ASSERT(fssproj_new != NULL);
2967 
2968 	fssproj_new->fssp_threads++;
2969 	thread_lock(t);
2970 	if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2971 	    t->t_state == TS_WAIT)
2972 		fss_inactive(t);
2973 	fssproc->fss_proj = fssproj_new;
2974 	fssproc->fss_fsspri = 0;
2975 	if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2976 	    t->t_state == TS_WAIT)
2977 		fss_active(t);
2978 	thread_unlock(t);
2979 	mutex_exit(&fsspset_new->fssps_lock);
2980 
2981 	mutex_enter(&fsspset_old->fssps_lock);
2982 	if (--fssproj_old->fssp_threads == 0) {
2983 		fss_remove_fssproj(fsspset_old, fssproj_old);
2984 		if (fsszone_old->fssz_nproj == 0)
2985 			kmem_free(fsszone_old, sizeof (fsszone_t));
2986 		kmem_free(fssproj_old, sizeof (fssproj_t));
2987 	}
2988 	mutex_exit(&fsspset_old->fssps_lock);
2989 
2990 	mutex_exit(&fsspsets_lock);
2991 }
2992