xref: /illumos-gate/usr/src/uts/common/disp/fx.c (revision 2570281cf351044b6936651ce26dbe1f801dcbd8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2013, Joyent, Inc. All rights reserved.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/cred.h>
31 #include <sys/proc.h>
32 #include <sys/session.h>
33 #include <sys/strsubr.h>
34 #include <sys/user.h>
35 #include <sys/priocntl.h>
36 #include <sys/class.h>
37 #include <sys/disp.h>
38 #include <sys/procset.h>
39 #include <sys/debug.h>
40 #include <sys/kmem.h>
41 #include <sys/errno.h>
42 #include <sys/fx.h>
43 #include <sys/fxpriocntl.h>
44 #include <sys/cpuvar.h>
45 #include <sys/systm.h>
46 #include <sys/vtrace.h>
47 #include <sys/schedctl.h>
48 #include <sys/sunddi.h>
49 #include <sys/spl.h>
50 #include <sys/modctl.h>
51 #include <sys/policy.h>
52 #include <sys/sdt.h>
53 #include <sys/cpupart.h>
54 #include <sys/cpucaps.h>
55 
56 static pri_t fx_init(id_t, int, classfuncs_t **);
57 
58 static struct sclass csw = {
59 	"FX",
60 	fx_init,
61 	0
62 };
63 
64 static struct modlsched modlsched = {
65 	&mod_schedops, "Fixed priority sched class", &csw
66 };
67 
68 static struct modlinkage modlinkage = {
69 	MODREV_1, (void *)&modlsched, NULL
70 };
71 
72 
73 /*
74  * control flags (kparms->fx_cflags).
75  */
76 #define	FX_DOUPRILIM	0x01    /* change user priority limit */
77 #define	FX_DOUPRI	0x02    /* change user priority */
78 #define	FX_DOTQ		0x04    /* change FX time quantum */
79 
80 
81 #define	FXMAXUPRI 60		/* maximum user priority setting */
82 
83 #define	FX_MAX_UNPRIV_PRI	0	/* maximum unpriviledge priority */
84 
85 /*
86  * The fxproc_t structures that have a registered callback vector,
87  * are also kept in an array of circular doubly linked lists. A hash on
88  * the thread id (from ddi_get_kt_did()) is used to determine which list
89  * each of such fxproc structures should be placed. Each list has a dummy
90  * "head" which is never removed, so the list is never empty.
91  */
92 
93 #define	FX_CB_LISTS 16		/* number of lists, must be power of 2 */
94 #define	FX_CB_LIST_HASH(ktid)	((uint_t)ktid & (FX_CB_LISTS - 1))
95 
96 /* Insert fxproc into callback list */
97 #define	FX_CB_LIST_INSERT(fxpp)						\
98 {									\
99 	int index = FX_CB_LIST_HASH(fxpp->fx_ktid);			\
100 	kmutex_t *lockp = &fx_cb_list_lock[index];			\
101 	fxproc_t *headp = &fx_cb_plisthead[index];			\
102 	mutex_enter(lockp);						\
103 	fxpp->fx_cb_next = headp->fx_cb_next;				\
104 	fxpp->fx_cb_prev = headp;					\
105 	headp->fx_cb_next->fx_cb_prev = fxpp;				\
106 	headp->fx_cb_next = fxpp;					\
107 	mutex_exit(lockp);						\
108 }
109 
110 /*
111  * Remove thread from callback list.
112  */
113 #define	FX_CB_LIST_DELETE(fxpp)						\
114 {									\
115 	int index = FX_CB_LIST_HASH(fxpp->fx_ktid);			\
116 	kmutex_t *lockp = &fx_cb_list_lock[index];			\
117 	mutex_enter(lockp);						\
118 	fxpp->fx_cb_prev->fx_cb_next = fxpp->fx_cb_next;		\
119 	fxpp->fx_cb_next->fx_cb_prev = fxpp->fx_cb_prev;		\
120 	mutex_exit(lockp);						\
121 }
122 
123 #define	FX_HAS_CB(fxpp)	(fxpp->fx_callback != NULL)
124 
125 /* adjust x to be between 0 and fx_maxumdpri */
126 
127 #define	FX_ADJUST_PRI(pri)						\
128 {									\
129 	if (pri < 0)							\
130 		pri = 0;						\
131 	else if (pri > fx_maxumdpri)					\
132 		pri = fx_maxumdpri;					\
133 }
134 
135 #define	FX_ADJUST_QUANTUM(q)						\
136 {									\
137 	if (q > INT_MAX)						\
138 		q = INT_MAX;						\
139 	else if (q <= 0)						\
140 		q = FX_TQINF;						\
141 }
142 
143 #define	FX_ISVALID(pri, quantum) \
144 	(((pri >= 0) || (pri == FX_CB_NOCHANGE)) &&			\
145 	    ((quantum >= 0) || (quantum == FX_NOCHANGE) ||		\
146 		(quantum == FX_TQDEF) || (quantum == FX_TQINF)))
147 
148 
149 static id_t	fx_cid;		/* fixed priority class ID */
150 static fxdpent_t *fx_dptbl;	/* fixed priority disp parameter table */
151 
152 static pri_t	fx_maxupri = FXMAXUPRI;
153 static pri_t	fx_maxumdpri;	/* max user mode fixed priority */
154 
155 static pri_t	fx_maxglobpri;	/* maximum global priority used by fx class */
156 static kmutex_t	fx_dptblock;	/* protects fixed priority dispatch table */
157 
158 
159 static kmutex_t	fx_cb_list_lock[FX_CB_LISTS];	/* protects list of fxprocs */
160 						/* that have callbacks */
161 static fxproc_t	fx_cb_plisthead[FX_CB_LISTS];	/* dummy fxproc at head of */
162 						/* list of fxprocs with */
163 						/* callbacks */
164 
165 static int	fx_admin(caddr_t, cred_t *);
166 static int	fx_getclinfo(void *);
167 static int	fx_parmsin(void *);
168 static int	fx_parmsout(void *, pc_vaparms_t *);
169 static int	fx_vaparmsin(void *, pc_vaparms_t *);
170 static int	fx_vaparmsout(void *, pc_vaparms_t *);
171 static int	fx_getclpri(pcpri_t *);
172 static int	fx_alloc(void **, int);
173 static void	fx_free(void *);
174 static int	fx_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
175 static void	fx_exitclass(void *);
176 static int	fx_canexit(kthread_t *, cred_t *);
177 static int	fx_fork(kthread_t *, kthread_t *, void *);
178 static void	fx_forkret(kthread_t *, kthread_t *);
179 static void	fx_parmsget(kthread_t *, void *);
180 static int	fx_parmsset(kthread_t *, void *, id_t, cred_t *);
181 static void	fx_stop(kthread_t *, int, int);
182 static void	fx_exit(kthread_t *);
183 static pri_t	fx_swapin(kthread_t *, int);
184 static pri_t	fx_swapout(kthread_t *, int);
185 static void	fx_trapret(kthread_t *);
186 static void	fx_preempt(kthread_t *);
187 static void	fx_setrun(kthread_t *);
188 static void	fx_sleep(kthread_t *);
189 static void	fx_tick(kthread_t *);
190 static void	fx_wakeup(kthread_t *);
191 static int	fx_donice(kthread_t *, cred_t *, int, int *);
192 static int	fx_doprio(kthread_t *, cred_t *, int, int *);
193 static pri_t	fx_globpri(kthread_t *);
194 static void	fx_yield(kthread_t *);
195 static void	fx_nullsys();
196 
197 extern fxdpent_t *fx_getdptbl(void);
198 
199 static void	fx_change_priority(kthread_t *, fxproc_t *);
200 static fxproc_t *fx_list_lookup(kt_did_t);
201 static void fx_list_release(fxproc_t *);
202 
203 
204 static struct classfuncs fx_classfuncs = {
205 	/* class functions */
206 	fx_admin,
207 	fx_getclinfo,
208 	fx_parmsin,
209 	fx_parmsout,
210 	fx_vaparmsin,
211 	fx_vaparmsout,
212 	fx_getclpri,
213 	fx_alloc,
214 	fx_free,
215 
216 	/* thread functions */
217 	fx_enterclass,
218 	fx_exitclass,
219 	fx_canexit,
220 	fx_fork,
221 	fx_forkret,
222 	fx_parmsget,
223 	fx_parmsset,
224 	fx_stop,
225 	fx_exit,
226 	fx_nullsys,	/* active */
227 	fx_nullsys,	/* inactive */
228 	fx_swapin,
229 	fx_swapout,
230 	fx_trapret,
231 	fx_preempt,
232 	fx_setrun,
233 	fx_sleep,
234 	fx_tick,
235 	fx_wakeup,
236 	fx_donice,
237 	fx_globpri,
238 	fx_nullsys,	/* set_process_group */
239 	fx_yield,
240 	fx_doprio,
241 };
242 
243 
244 int
_init()245 _init()
246 {
247 	return (mod_install(&modlinkage));
248 }
249 
250 int
_fini()251 _fini()
252 {
253 	return (EBUSY);
254 }
255 
256 int
_info(struct modinfo * modinfop)257 _info(struct modinfo *modinfop)
258 {
259 	return (mod_info(&modlinkage, modinfop));
260 }
261 
262 /*
263  * Fixed priority class initialization. Called by dispinit() at boot time.
264  * We can ignore the clparmsz argument since we know that the smallest
265  * possible parameter buffer is big enough for us.
266  */
267 /* ARGSUSED */
268 static pri_t
fx_init(id_t cid,int clparmsz,classfuncs_t ** clfuncspp)269 fx_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
270 {
271 	int i;
272 	extern pri_t fx_getmaxumdpri(void);
273 
274 	fx_dptbl = fx_getdptbl();
275 	fx_maxumdpri = fx_getmaxumdpri();
276 	fx_maxglobpri = fx_dptbl[fx_maxumdpri].fx_globpri;
277 
278 	fx_cid = cid;		/* Record our class ID */
279 
280 	/*
281 	 * Initialize the hash table for fxprocs with callbacks
282 	 */
283 	for (i = 0; i < FX_CB_LISTS; i++) {
284 		fx_cb_plisthead[i].fx_cb_next = fx_cb_plisthead[i].fx_cb_prev =
285 		    &fx_cb_plisthead[i];
286 	}
287 
288 	/*
289 	 * We're required to return a pointer to our classfuncs
290 	 * structure and the highest global priority value we use.
291 	 */
292 	*clfuncspp = &fx_classfuncs;
293 	return (fx_maxglobpri);
294 }
295 
296 /*
297  * Get or reset the fx_dptbl values per the user's request.
298  */
299 static int
fx_admin(caddr_t uaddr,cred_t * reqpcredp)300 fx_admin(caddr_t uaddr, cred_t *reqpcredp)
301 {
302 	fxadmin_t	fxadmin;
303 	fxdpent_t	*tmpdpp;
304 	int		userdpsz;
305 	int		i;
306 	size_t		fxdpsz;
307 
308 	if (get_udatamodel() == DATAMODEL_NATIVE) {
309 		if (copyin(uaddr, &fxadmin, sizeof (fxadmin_t)))
310 			return (EFAULT);
311 	}
312 #ifdef _SYSCALL32_IMPL
313 	else {
314 		/* get fxadmin struct from ILP32 caller */
315 		fxadmin32_t fxadmin32;
316 		if (copyin(uaddr, &fxadmin32, sizeof (fxadmin32_t)))
317 			return (EFAULT);
318 		fxadmin.fx_dpents =
319 		    (struct fxdpent *)(uintptr_t)fxadmin32.fx_dpents;
320 		fxadmin.fx_ndpents = fxadmin32.fx_ndpents;
321 		fxadmin.fx_cmd = fxadmin32.fx_cmd;
322 	}
323 #endif /* _SYSCALL32_IMPL */
324 
325 	fxdpsz = (fx_maxumdpri + 1) * sizeof (fxdpent_t);
326 
327 	switch (fxadmin.fx_cmd) {
328 	case FX_GETDPSIZE:
329 		fxadmin.fx_ndpents = fx_maxumdpri + 1;
330 
331 		if (get_udatamodel() == DATAMODEL_NATIVE) {
332 			if (copyout(&fxadmin, uaddr, sizeof (fxadmin_t)))
333 				return (EFAULT);
334 		}
335 #ifdef _SYSCALL32_IMPL
336 		else {
337 			/* return fxadmin struct to ILP32 caller */
338 			fxadmin32_t fxadmin32;
339 			fxadmin32.fx_dpents =
340 			    (caddr32_t)(uintptr_t)fxadmin.fx_dpents;
341 			fxadmin32.fx_ndpents = fxadmin.fx_ndpents;
342 			fxadmin32.fx_cmd = fxadmin.fx_cmd;
343 			if (copyout(&fxadmin32, uaddr, sizeof (fxadmin32_t)))
344 				return (EFAULT);
345 		}
346 #endif /* _SYSCALL32_IMPL */
347 		break;
348 
349 	case FX_GETDPTBL:
350 		userdpsz = MIN(fxadmin.fx_ndpents * sizeof (fxdpent_t),
351 		    fxdpsz);
352 		if (copyout(fx_dptbl, fxadmin.fx_dpents, userdpsz))
353 			return (EFAULT);
354 
355 		fxadmin.fx_ndpents = userdpsz / sizeof (fxdpent_t);
356 
357 		if (get_udatamodel() == DATAMODEL_NATIVE) {
358 			if (copyout(&fxadmin, uaddr, sizeof (fxadmin_t)))
359 				return (EFAULT);
360 		}
361 #ifdef _SYSCALL32_IMPL
362 		else {
363 			/* return fxadmin struct to ILP32 callers */
364 			fxadmin32_t fxadmin32;
365 			fxadmin32.fx_dpents =
366 			    (caddr32_t)(uintptr_t)fxadmin.fx_dpents;
367 			fxadmin32.fx_ndpents = fxadmin.fx_ndpents;
368 			fxadmin32.fx_cmd = fxadmin.fx_cmd;
369 			if (copyout(&fxadmin32, uaddr, sizeof (fxadmin32_t)))
370 				return (EFAULT);
371 		}
372 #endif /* _SYSCALL32_IMPL */
373 		break;
374 
375 	case FX_SETDPTBL:
376 		/*
377 		 * We require that the requesting process has sufficient
378 		 * privileges. We also require that the table supplied by
379 		 * the user exactly match the current fx_dptbl in size.
380 		 */
381 		if (secpolicy_dispadm(reqpcredp) != 0) {
382 			return (EPERM);
383 		}
384 		if (fxadmin.fx_ndpents * sizeof (fxdpent_t) != fxdpsz) {
385 			return (EINVAL);
386 		}
387 
388 		/*
389 		 * We read the user supplied table into a temporary buffer
390 		 * where it is validated before being copied over the
391 		 * fx_dptbl.
392 		 */
393 		tmpdpp = kmem_alloc(fxdpsz, KM_SLEEP);
394 		if (copyin(fxadmin.fx_dpents, tmpdpp, fxdpsz)) {
395 			kmem_free(tmpdpp, fxdpsz);
396 			return (EFAULT);
397 		}
398 		for (i = 0; i < fxadmin.fx_ndpents; i++) {
399 
400 			/*
401 			 * Validate the user supplied values. All we are doing
402 			 * here is verifying that the values are within their
403 			 * allowable ranges and will not panic the system. We
404 			 * make no attempt to ensure that the resulting
405 			 * configuration makes sense or results in reasonable
406 			 * performance.
407 			 */
408 			if (tmpdpp[i].fx_quantum <= 0 &&
409 			    tmpdpp[i].fx_quantum != FX_TQINF) {
410 				kmem_free(tmpdpp, fxdpsz);
411 				return (EINVAL);
412 			}
413 		}
414 
415 		/*
416 		 * Copy the user supplied values over the current fx_dptbl
417 		 * values. The fx_globpri member is read-only so we don't
418 		 * overwrite it.
419 		 */
420 		mutex_enter(&fx_dptblock);
421 		for (i = 0; i < fxadmin.fx_ndpents; i++) {
422 			fx_dptbl[i].fx_quantum = tmpdpp[i].fx_quantum;
423 		}
424 		mutex_exit(&fx_dptblock);
425 		kmem_free(tmpdpp, fxdpsz);
426 		break;
427 
428 	default:
429 		return (EINVAL);
430 	}
431 	return (0);
432 }
433 
434 /*
435  * Allocate a fixed priority class specific thread structure and
436  * initialize it with the parameters supplied. Also move the thread
437  * to specified priority.
438  */
439 static int
fx_enterclass(kthread_t * t,id_t cid,void * parmsp,cred_t * reqpcredp,void * bufp)440 fx_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
441     void *bufp)
442 {
443 	fxkparms_t	*fxkparmsp = (fxkparms_t *)parmsp;
444 	fxproc_t	*fxpp;
445 	pri_t		reqfxupri;
446 	pri_t		reqfxuprilim;
447 
448 	fxpp = (fxproc_t *)bufp;
449 	ASSERT(fxpp != NULL);
450 
451 	/*
452 	 * Initialize the fxproc structure.
453 	 */
454 	fxpp->fx_flags = 0;
455 	fxpp->fx_callback = NULL;
456 	fxpp->fx_cookie = 0;
457 
458 	if (fxkparmsp == NULL) {
459 		/*
460 		 * Use default values.
461 		 */
462 		fxpp->fx_pri = fxpp->fx_uprilim = 0;
463 		fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;
464 		fxpp->fx_nice =  NZERO;
465 	} else {
466 		/*
467 		 * Use supplied values.
468 		 */
469 
470 		if ((fxkparmsp->fx_cflags & FX_DOUPRILIM) == 0) {
471 			reqfxuprilim = 0;
472 		} else {
473 			if (fxkparmsp->fx_uprilim > FX_MAX_UNPRIV_PRI &&
474 			    secpolicy_setpriority(reqpcredp) != 0)
475 				return (EPERM);
476 			reqfxuprilim = fxkparmsp->fx_uprilim;
477 			FX_ADJUST_PRI(reqfxuprilim);
478 		}
479 
480 		if ((fxkparmsp->fx_cflags & FX_DOUPRI) == 0) {
481 			reqfxupri = reqfxuprilim;
482 		} else {
483 			if (fxkparmsp->fx_upri > FX_MAX_UNPRIV_PRI &&
484 			    secpolicy_setpriority(reqpcredp) != 0)
485 				return (EPERM);
486 			/*
487 			 * Set the user priority to the requested value
488 			 * or the upri limit, whichever is lower.
489 			 */
490 			reqfxupri = fxkparmsp->fx_upri;
491 			FX_ADJUST_PRI(reqfxupri);
492 
493 			if (reqfxupri > reqfxuprilim)
494 				reqfxupri = reqfxuprilim;
495 		}
496 
497 
498 		fxpp->fx_uprilim = reqfxuprilim;
499 		fxpp->fx_pri = reqfxupri;
500 
501 		fxpp->fx_nice = NZERO - (NZERO * reqfxupri) / fx_maxupri;
502 
503 		if (((fxkparmsp->fx_cflags & FX_DOTQ) == 0) ||
504 		    (fxkparmsp->fx_tqntm == FX_TQDEF)) {
505 			fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;
506 		} else {
507 			if (secpolicy_setpriority(reqpcredp) != 0)
508 				return (EPERM);
509 
510 			if (fxkparmsp->fx_tqntm == FX_TQINF)
511 				fxpp->fx_pquantum = FX_TQINF;
512 			else {
513 				fxpp->fx_pquantum = fxkparmsp->fx_tqntm;
514 			}
515 		}
516 
517 	}
518 
519 	fxpp->fx_timeleft = fxpp->fx_pquantum;
520 	cpucaps_sc_init(&fxpp->fx_caps);
521 	fxpp->fx_tp = t;
522 
523 	thread_lock(t);			/* get dispatcher lock on thread */
524 	t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
525 	t->t_cid = cid;
526 	t->t_cldata = (void *)fxpp;
527 	t->t_schedflag &= ~TS_RUNQMATCH;
528 	fx_change_priority(t, fxpp);
529 	thread_unlock(t);
530 
531 	return (0);
532 }
533 
534 /*
535  * The thread is exiting.
536  */
537 static void
fx_exit(kthread_t * t)538 fx_exit(kthread_t *t)
539 {
540 	fxproc_t *fxpp;
541 
542 	thread_lock(t);
543 	fxpp = (fxproc_t *)(t->t_cldata);
544 
545 	/*
546 	 * A thread could be exiting in between clock ticks, so we need to
547 	 * calculate how much CPU time it used since it was charged last time.
548 	 *
549 	 * CPU caps are not enforced on exiting processes - it is usually
550 	 * desirable to exit as soon as possible to free resources.
551 	 */
552 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ONLY);
553 
554 	if (FX_HAS_CB(fxpp)) {
555 		FX_CB_EXIT(FX_CALLB(fxpp), fxpp->fx_cookie);
556 		fxpp->fx_callback = NULL;
557 		fxpp->fx_cookie = 0;
558 		thread_unlock(t);
559 		FX_CB_LIST_DELETE(fxpp);
560 		return;
561 	}
562 
563 	thread_unlock(t);
564 }
565 
566 /*
567  * Exiting the class. Free fxproc structure of thread.
568  */
569 static void
fx_exitclass(void * procp)570 fx_exitclass(void *procp)
571 {
572 	fxproc_t *fxpp = (fxproc_t *)procp;
573 
574 	thread_lock(fxpp->fx_tp);
575 	if (FX_HAS_CB(fxpp)) {
576 
577 		FX_CB_EXIT(FX_CALLB(fxpp), fxpp->fx_cookie);
578 
579 		fxpp->fx_callback = NULL;
580 		fxpp->fx_cookie = 0;
581 		thread_unlock(fxpp->fx_tp);
582 		FX_CB_LIST_DELETE(fxpp);
583 	} else
584 		thread_unlock(fxpp->fx_tp);
585 
586 	kmem_free(fxpp, sizeof (fxproc_t));
587 }
588 
589 /* ARGSUSED */
590 static int
fx_canexit(kthread_t * t,cred_t * cred)591 fx_canexit(kthread_t *t, cred_t *cred)
592 {
593 	/*
594 	 * A thread can always leave the FX class
595 	 */
596 	return (0);
597 }
598 
599 /*
600  * Initialize fixed-priority class specific proc structure for a child.
601  * callbacks are not inherited upon fork.
602  */
603 static int
fx_fork(kthread_t * t,kthread_t * ct,void * bufp)604 fx_fork(kthread_t *t, kthread_t *ct, void *bufp)
605 {
606 	fxproc_t	*pfxpp;		/* ptr to parent's fxproc structure */
607 	fxproc_t	*cfxpp;		/* ptr to child's fxproc structure */
608 
609 	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
610 
611 	cfxpp = (fxproc_t *)bufp;
612 	ASSERT(cfxpp != NULL);
613 	thread_lock(t);
614 	pfxpp = (fxproc_t *)t->t_cldata;
615 	/*
616 	 * Initialize child's fxproc structure.
617 	 */
618 	cfxpp->fx_timeleft = cfxpp->fx_pquantum = pfxpp->fx_pquantum;
619 	cfxpp->fx_pri = pfxpp->fx_pri;
620 	cfxpp->fx_uprilim = pfxpp->fx_uprilim;
621 	cfxpp->fx_nice = pfxpp->fx_nice;
622 	cfxpp->fx_callback = NULL;
623 	cfxpp->fx_cookie = 0;
624 	cfxpp->fx_flags = pfxpp->fx_flags & ~(FXBACKQ);
625 	cpucaps_sc_init(&cfxpp->fx_caps);
626 
627 	cfxpp->fx_tp = ct;
628 	ct->t_cldata = (void *)cfxpp;
629 	thread_unlock(t);
630 
631 	/*
632 	 * Link new structure into fxproc list.
633 	 */
634 	return (0);
635 }
636 
637 
638 /*
639  * Child is placed at back of dispatcher queue and parent gives
640  * up processor so that the child runs first after the fork.
641  * This allows the child immediately execing to break the multiple
642  * use of copy on write pages with no disk home. The parent will
643  * get to steal them back rather than uselessly copying them.
644  */
645 static void
fx_forkret(kthread_t * t,kthread_t * ct)646 fx_forkret(kthread_t *t, kthread_t *ct)
647 {
648 	proc_t	*pp = ttoproc(t);
649 	proc_t	*cp = ttoproc(ct);
650 	fxproc_t *fxpp;
651 
652 	ASSERT(t == curthread);
653 	ASSERT(MUTEX_HELD(&pidlock));
654 
655 	/*
656 	 * Grab the child's p_lock before dropping pidlock to ensure
657 	 * the process does not disappear before we set it running.
658 	 */
659 	mutex_enter(&cp->p_lock);
660 	continuelwps(cp);
661 	mutex_exit(&cp->p_lock);
662 
663 	mutex_enter(&pp->p_lock);
664 	mutex_exit(&pidlock);
665 	continuelwps(pp);
666 
667 	thread_lock(t);
668 	fxpp = (fxproc_t *)(t->t_cldata);
669 	t->t_pri = fx_dptbl[fxpp->fx_pri].fx_globpri;
670 	ASSERT(t->t_pri >= 0 && t->t_pri <= fx_maxglobpri);
671 	THREAD_TRANSITION(t);
672 	fx_setrun(t);
673 	thread_unlock(t);
674 	/*
675 	 * Safe to drop p_lock now since it is safe to change
676 	 * the scheduling class after this point.
677 	 */
678 	mutex_exit(&pp->p_lock);
679 
680 	swtch();
681 }
682 
683 
684 /*
685  * Get information about the fixed-priority class into the buffer
686  * pointed to by fxinfop. The maximum configured user priority
687  * is the only information we supply.
688  */
689 static int
fx_getclinfo(void * infop)690 fx_getclinfo(void *infop)
691 {
692 	fxinfo_t *fxinfop = (fxinfo_t *)infop;
693 	fxinfop->fx_maxupri = fx_maxupri;
694 	return (0);
695 }
696 
697 
698 
699 /*
700  * Return the user mode scheduling priority range.
701  */
702 static int
fx_getclpri(pcpri_t * pcprip)703 fx_getclpri(pcpri_t *pcprip)
704 {
705 	pcprip->pc_clpmax = fx_maxupri;
706 	pcprip->pc_clpmin = 0;
707 	return (0);
708 }
709 
710 
711 static void
fx_nullsys()712 fx_nullsys()
713 {}
714 
715 
716 /*
717  * Get the fixed-priority parameters of the thread pointed to by
718  * fxprocp into the buffer pointed to by fxparmsp.
719  */
720 static void
fx_parmsget(kthread_t * t,void * parmsp)721 fx_parmsget(kthread_t *t, void *parmsp)
722 {
723 	fxproc_t *fxpp = (fxproc_t *)t->t_cldata;
724 	fxkparms_t *fxkparmsp = (fxkparms_t *)parmsp;
725 
726 	fxkparmsp->fx_upri = fxpp->fx_pri;
727 	fxkparmsp->fx_uprilim = fxpp->fx_uprilim;
728 	fxkparmsp->fx_tqntm = fxpp->fx_pquantum;
729 }
730 
731 
732 
733 /*
734  * Check the validity of the fixed-priority parameters in the buffer
735  * pointed to by fxparmsp.
736  */
737 static int
fx_parmsin(void * parmsp)738 fx_parmsin(void *parmsp)
739 {
740 	fxparms_t	*fxparmsp = (fxparms_t *)parmsp;
741 	uint_t		cflags;
742 	longlong_t	ticks;
743 	/*
744 	 * Check validity of parameters.
745 	 */
746 
747 	if ((fxparmsp->fx_uprilim > fx_maxupri ||
748 	    fxparmsp->fx_uprilim < 0) &&
749 	    fxparmsp->fx_uprilim != FX_NOCHANGE)
750 		return (EINVAL);
751 
752 	if ((fxparmsp->fx_upri > fx_maxupri ||
753 	    fxparmsp->fx_upri < 0) &&
754 	    fxparmsp->fx_upri != FX_NOCHANGE)
755 		return (EINVAL);
756 
757 	if ((fxparmsp->fx_tqsecs == 0 && fxparmsp->fx_tqnsecs == 0) ||
758 	    fxparmsp->fx_tqnsecs >= NANOSEC)
759 		return (EINVAL);
760 
761 	cflags = (fxparmsp->fx_upri != FX_NOCHANGE ? FX_DOUPRI : 0);
762 
763 	if (fxparmsp->fx_uprilim != FX_NOCHANGE) {
764 		cflags |= FX_DOUPRILIM;
765 	}
766 
767 	if (fxparmsp->fx_tqnsecs != FX_NOCHANGE)
768 		cflags |= FX_DOTQ;
769 
770 	/*
771 	 * convert the buffer to kernel format.
772 	 */
773 
774 	if (fxparmsp->fx_tqnsecs >= 0) {
775 		if ((ticks = SEC_TO_TICK((longlong_t)fxparmsp->fx_tqsecs) +
776 		    NSEC_TO_TICK_ROUNDUP(fxparmsp->fx_tqnsecs)) > INT_MAX)
777 			return (ERANGE);
778 
779 		((fxkparms_t *)fxparmsp)->fx_tqntm = (int)ticks;
780 	} else {
781 		if ((fxparmsp->fx_tqnsecs != FX_NOCHANGE) &&
782 		    (fxparmsp->fx_tqnsecs != FX_TQINF) &&
783 		    (fxparmsp->fx_tqnsecs != FX_TQDEF))
784 			return (EINVAL);
785 		((fxkparms_t *)fxparmsp)->fx_tqntm = fxparmsp->fx_tqnsecs;
786 	}
787 
788 	((fxkparms_t *)fxparmsp)->fx_cflags = cflags;
789 
790 	return (0);
791 }
792 
793 
794 /*
795  * Check the validity of the fixed-priority parameters in the pc_vaparms_t
796  * structure vaparmsp and put them in the buffer pointed to by fxprmsp.
797  * pc_vaparms_t contains (key, value) pairs of parameter.
798  */
799 static int
fx_vaparmsin(void * prmsp,pc_vaparms_t * vaparmsp)800 fx_vaparmsin(void *prmsp, pc_vaparms_t *vaparmsp)
801 {
802 	uint_t		secs = 0;
803 	uint_t		cnt;
804 	int		nsecs = 0;
805 	int		priflag, secflag, nsecflag, limflag;
806 	longlong_t	ticks;
807 	fxkparms_t	*fxprmsp = (fxkparms_t *)prmsp;
808 	pc_vaparm_t	*vpp = &vaparmsp->pc_parms[0];
809 
810 
811 	/*
812 	 * First check the validity of parameters and convert them
813 	 * from the user supplied format to the internal format.
814 	 */
815 	priflag = secflag = nsecflag = limflag = 0;
816 
817 	fxprmsp->fx_cflags = 0;
818 
819 	if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
820 		return (EINVAL);
821 
822 	for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
823 
824 		switch (vpp->pc_key) {
825 		case FX_KY_UPRILIM:
826 			if (limflag++)
827 				return (EINVAL);
828 			fxprmsp->fx_cflags |= FX_DOUPRILIM;
829 			fxprmsp->fx_uprilim = (pri_t)vpp->pc_parm;
830 			if (fxprmsp->fx_uprilim > fx_maxupri ||
831 			    fxprmsp->fx_uprilim < 0)
832 				return (EINVAL);
833 			break;
834 
835 		case FX_KY_UPRI:
836 			if (priflag++)
837 				return (EINVAL);
838 			fxprmsp->fx_cflags |= FX_DOUPRI;
839 			fxprmsp->fx_upri = (pri_t)vpp->pc_parm;
840 			if (fxprmsp->fx_upri > fx_maxupri ||
841 			    fxprmsp->fx_upri < 0)
842 				return (EINVAL);
843 			break;
844 
845 		case FX_KY_TQSECS:
846 			if (secflag++)
847 				return (EINVAL);
848 			fxprmsp->fx_cflags |= FX_DOTQ;
849 			secs = (uint_t)vpp->pc_parm;
850 			break;
851 
852 		case FX_KY_TQNSECS:
853 			if (nsecflag++)
854 				return (EINVAL);
855 			fxprmsp->fx_cflags |= FX_DOTQ;
856 			nsecs = (int)vpp->pc_parm;
857 			break;
858 
859 		default:
860 			return (EINVAL);
861 		}
862 	}
863 
864 	if (vaparmsp->pc_vaparmscnt == 0) {
865 		/*
866 		 * Use default parameters.
867 		 */
868 		fxprmsp->fx_upri = 0;
869 		fxprmsp->fx_uprilim = 0;
870 		fxprmsp->fx_tqntm = FX_TQDEF;
871 		fxprmsp->fx_cflags = FX_DOUPRI | FX_DOUPRILIM | FX_DOTQ;
872 	} else if ((fxprmsp->fx_cflags & FX_DOTQ) != 0) {
873 		if ((secs == 0 && nsecs == 0) || nsecs >= NANOSEC)
874 			return (EINVAL);
875 
876 		if (nsecs >= 0) {
877 			if ((ticks = SEC_TO_TICK((longlong_t)secs) +
878 			    NSEC_TO_TICK_ROUNDUP(nsecs)) > INT_MAX)
879 				return (ERANGE);
880 
881 			fxprmsp->fx_tqntm = (int)ticks;
882 		} else {
883 			if (nsecs != FX_TQINF && nsecs != FX_TQDEF)
884 				return (EINVAL);
885 			fxprmsp->fx_tqntm = nsecs;
886 		}
887 	}
888 
889 	return (0);
890 }
891 
892 
893 /*
894  * Nothing to do here but return success.
895  */
896 /* ARGSUSED */
897 static int
fx_parmsout(void * parmsp,pc_vaparms_t * vaparmsp)898 fx_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
899 {
900 	register fxkparms_t	*fxkprmsp = (fxkparms_t *)parmsp;
901 
902 	if (vaparmsp != NULL)
903 		return (0);
904 
905 	if (fxkprmsp->fx_tqntm < 0) {
906 		/*
907 		 * Quantum field set to special value (e.g. FX_TQINF)
908 		 */
909 		((fxparms_t *)fxkprmsp)->fx_tqnsecs = fxkprmsp->fx_tqntm;
910 		((fxparms_t *)fxkprmsp)->fx_tqsecs = 0;
911 
912 	} else {
913 		/* Convert quantum from ticks to seconds-nanoseconds */
914 
915 		timestruc_t ts;
916 		TICK_TO_TIMESTRUC(fxkprmsp->fx_tqntm, &ts);
917 		((fxparms_t *)fxkprmsp)->fx_tqsecs = ts.tv_sec;
918 		((fxparms_t *)fxkprmsp)->fx_tqnsecs = ts.tv_nsec;
919 	}
920 
921 	return (0);
922 }
923 
924 
925 /*
926  * Copy all selected fixed-priority class parameters to the user.
927  * The parameters are specified by a key.
928  */
929 static int
fx_vaparmsout(void * prmsp,pc_vaparms_t * vaparmsp)930 fx_vaparmsout(void *prmsp, pc_vaparms_t *vaparmsp)
931 {
932 	fxkparms_t	*fxkprmsp = (fxkparms_t *)prmsp;
933 	timestruc_t	ts;
934 	uint_t		cnt;
935 	uint_t		secs;
936 	int		nsecs;
937 	int		priflag, secflag, nsecflag, limflag;
938 	pc_vaparm_t	*vpp = &vaparmsp->pc_parms[0];
939 
940 	ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
941 
942 	priflag = secflag = nsecflag = limflag = 0;
943 
944 	if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
945 		return (EINVAL);
946 
947 	if (fxkprmsp->fx_tqntm < 0) {
948 		/*
949 		 * Quantum field set to special value (e.g. FX_TQINF).
950 		 */
951 		secs = 0;
952 		nsecs = fxkprmsp->fx_tqntm;
953 	} else {
954 		/*
955 		 * Convert quantum from ticks to seconds-nanoseconds.
956 		 */
957 		TICK_TO_TIMESTRUC(fxkprmsp->fx_tqntm, &ts);
958 		secs = ts.tv_sec;
959 		nsecs = ts.tv_nsec;
960 	}
961 
962 
963 	for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
964 
965 		switch (vpp->pc_key) {
966 		case FX_KY_UPRILIM:
967 			if (limflag++)
968 				return (EINVAL);
969 			if (copyout(&fxkprmsp->fx_uprilim,
970 			    (void *)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
971 				return (EFAULT);
972 			break;
973 
974 		case FX_KY_UPRI:
975 			if (priflag++)
976 				return (EINVAL);
977 			if (copyout(&fxkprmsp->fx_upri,
978 			    (void *)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
979 				return (EFAULT);
980 			break;
981 
982 		case FX_KY_TQSECS:
983 			if (secflag++)
984 				return (EINVAL);
985 			if (copyout(&secs,
986 			    (void *)(uintptr_t)vpp->pc_parm, sizeof (uint_t)))
987 				return (EFAULT);
988 			break;
989 
990 		case FX_KY_TQNSECS:
991 			if (nsecflag++)
992 				return (EINVAL);
993 			if (copyout(&nsecs,
994 			    (void *)(uintptr_t)vpp->pc_parm, sizeof (int)))
995 				return (EFAULT);
996 			break;
997 
998 		default:
999 			return (EINVAL);
1000 		}
1001 	}
1002 
1003 	return (0);
1004 }
1005 
1006 /*
1007  * Set the scheduling parameters of the thread pointed to by fxprocp
1008  * to those specified in the buffer pointed to by fxparmsp.
1009  */
1010 /* ARGSUSED */
1011 static int
fx_parmsset(kthread_t * tx,void * parmsp,id_t reqpcid,cred_t * reqpcredp)1012 fx_parmsset(kthread_t *tx, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1013 {
1014 	char		nice;
1015 	pri_t		reqfxuprilim;
1016 	pri_t		reqfxupri;
1017 	fxkparms_t	*fxkparmsp = (fxkparms_t *)parmsp;
1018 	fxproc_t	*fxpp;
1019 
1020 
1021 	ASSERT(MUTEX_HELD(&(ttoproc(tx))->p_lock));
1022 
1023 	thread_lock(tx);
1024 	fxpp = (fxproc_t *)tx->t_cldata;
1025 
1026 	if ((fxkparmsp->fx_cflags & FX_DOUPRILIM) == 0)
1027 		reqfxuprilim = fxpp->fx_uprilim;
1028 	else
1029 		reqfxuprilim = fxkparmsp->fx_uprilim;
1030 
1031 	/*
1032 	 * Basic permissions enforced by generic kernel code
1033 	 * for all classes require that a thread attempting
1034 	 * to change the scheduling parameters of a target
1035 	 * thread be privileged or have a real or effective
1036 	 * UID matching that of the target thread. We are not
1037 	 * called unless these basic permission checks have
1038 	 * already passed. The fixed priority class requires in
1039 	 * addition that the calling thread be privileged if it
1040 	 * is attempting to raise the pri above its current
1041 	 * value This may have been checked previously but if our
1042 	 * caller passed us a non-NULL credential pointer we assume
1043 	 * it hasn't and we check it here.
1044 	 */
1045 
1046 	if ((reqpcredp != NULL) &&
1047 	    (reqfxuprilim > fxpp->fx_uprilim ||
1048 	    ((fxkparmsp->fx_cflags & FX_DOTQ) != 0)) &&
1049 	    secpolicy_raisepriority(reqpcredp) != 0) {
1050 		thread_unlock(tx);
1051 		return (EPERM);
1052 	}
1053 
1054 	FX_ADJUST_PRI(reqfxuprilim);
1055 
1056 	if ((fxkparmsp->fx_cflags & FX_DOUPRI) == 0)
1057 		reqfxupri = fxpp->fx_pri;
1058 	else
1059 		reqfxupri = fxkparmsp->fx_upri;
1060 
1061 
1062 	/*
1063 	 * Make sure the user priority doesn't exceed the upri limit.
1064 	 */
1065 	if (reqfxupri > reqfxuprilim)
1066 		reqfxupri = reqfxuprilim;
1067 
1068 	/*
1069 	 * Set fx_nice to the nice value corresponding to the user
1070 	 * priority we are setting.  Note that setting the nice field
1071 	 * of the parameter struct won't affect upri or nice.
1072 	 */
1073 
1074 	nice = NZERO - (reqfxupri * NZERO) / fx_maxupri;
1075 
1076 	if (nice > NZERO)
1077 		nice = NZERO;
1078 
1079 	fxpp->fx_uprilim = reqfxuprilim;
1080 	fxpp->fx_pri = reqfxupri;
1081 
1082 	if (fxkparmsp->fx_tqntm == FX_TQINF)
1083 		fxpp->fx_pquantum = FX_TQINF;
1084 	else if (fxkparmsp->fx_tqntm == FX_TQDEF)
1085 		fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;
1086 	else if ((fxkparmsp->fx_cflags & FX_DOTQ) != 0)
1087 		fxpp->fx_pquantum = fxkparmsp->fx_tqntm;
1088 
1089 	fxpp->fx_nice = nice;
1090 
1091 	fx_change_priority(tx, fxpp);
1092 	thread_unlock(tx);
1093 	return (0);
1094 }
1095 
1096 
1097 /*
1098  * Return the global scheduling priority that would be assigned
1099  * to a thread entering the fixed-priority class with the fx_upri.
1100  */
1101 static pri_t
fx_globpri(kthread_t * t)1102 fx_globpri(kthread_t *t)
1103 {
1104 	fxproc_t *fxpp;
1105 
1106 	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1107 
1108 	fxpp = (fxproc_t *)t->t_cldata;
1109 	return (fx_dptbl[fxpp->fx_pri].fx_globpri);
1110 
1111 }
1112 
1113 /*
1114  * Arrange for thread to be placed in appropriate location
1115  * on dispatcher queue.
1116  *
1117  * This is called with the current thread in TS_ONPROC and locked.
1118  */
1119 static void
fx_preempt(kthread_t * t)1120 fx_preempt(kthread_t *t)
1121 {
1122 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);
1123 
1124 	ASSERT(t == curthread);
1125 	ASSERT(THREAD_LOCK_HELD(curthread));
1126 
1127 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ENFORCE);
1128 
1129 	/*
1130 	 * Check to see if we're doing "preemption control" here.  If
1131 	 * we are, and if the user has requested that this thread not
1132 	 * be preempted, and if preemptions haven't been put off for
1133 	 * too long, let the preemption happen here but try to make
1134 	 * sure the thread is rescheduled as soon as possible.  We do
1135 	 * this by putting it on the front of the highest priority run
1136 	 * queue in the FX class.  If the preemption has been put off
1137 	 * for too long, clear the "nopreempt" bit and let the thread
1138 	 * be preempted.
1139 	 */
1140 	if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1141 		if (fxpp->fx_pquantum == FX_TQINF ||
1142 		    fxpp->fx_timeleft > -SC_MAX_TICKS) {
1143 			DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
1144 			schedctl_set_yield(t, 1);
1145 			setfrontdq(t);
1146 			return;
1147 		} else {
1148 			schedctl_set_nopreempt(t, 0);
1149 			DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
1150 			/*
1151 			 * Fall through and be preempted below.
1152 			 */
1153 		}
1154 	}
1155 
1156 	if (FX_HAS_CB(fxpp)) {
1157 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;
1158 		pri_t	newpri = fxpp->fx_pri;
1159 		FX_CB_PREEMPT(FX_CALLB(fxpp), fxpp->fx_cookie,
1160 		    &new_quantum, &newpri);
1161 		FX_ADJUST_QUANTUM(new_quantum);
1162 		if ((int)new_quantum != fxpp->fx_pquantum) {
1163 			fxpp->fx_pquantum = (int)new_quantum;
1164 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1165 		}
1166 		FX_ADJUST_PRI(newpri);
1167 		fxpp->fx_pri = newpri;
1168 		THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);
1169 	}
1170 
1171 	/*
1172 	 * This thread may be placed on wait queue by CPU Caps. In this case we
1173 	 * do not need to do anything until it is removed from the wait queue.
1174 	 */
1175 	if (CPUCAPS_ENFORCE(t)) {
1176 		return;
1177 	}
1178 
1179 	if ((fxpp->fx_flags & (FXBACKQ)) == FXBACKQ) {
1180 		fxpp->fx_timeleft = fxpp->fx_pquantum;
1181 		fxpp->fx_flags &= ~FXBACKQ;
1182 		setbackdq(t);
1183 	} else {
1184 		setfrontdq(t);
1185 	}
1186 }
1187 
1188 static void
fx_setrun(kthread_t * t)1189 fx_setrun(kthread_t *t)
1190 {
1191 	fxproc_t *fxpp = (fxproc_t *)(t->t_cldata);
1192 
1193 	ASSERT(THREAD_LOCK_HELD(t));	/* t should be in transition */
1194 	fxpp->fx_flags &= ~FXBACKQ;
1195 
1196 	if (t->t_disp_time != ddi_get_lbolt())
1197 		setbackdq(t);
1198 	else
1199 		setfrontdq(t);
1200 }
1201 
1202 
1203 /*
1204  * Prepare thread for sleep. We reset the thread priority so it will
1205  * run at the kernel priority level when it wakes up.
1206  */
1207 static void
fx_sleep(kthread_t * t)1208 fx_sleep(kthread_t *t)
1209 {
1210 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);
1211 
1212 	ASSERT(t == curthread);
1213 	ASSERT(THREAD_LOCK_HELD(t));
1214 
1215 	/*
1216 	 * Account for time spent on CPU before going to sleep.
1217 	 */
1218 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ENFORCE);
1219 
1220 	if (FX_HAS_CB(fxpp)) {
1221 		FX_CB_SLEEP(FX_CALLB(fxpp), fxpp->fx_cookie);
1222 	}
1223 	t->t_stime = ddi_get_lbolt();		/* time stamp for the swapper */
1224 }
1225 
1226 
1227 /*
1228  * Return Values:
1229  *
1230  *	-1 if the thread is loaded or is not eligible to be swapped in.
1231  *
1232  * FX and RT threads are designed so that they don't swapout; however,
1233  * it is possible that while the thread is swapped out and in another class, it
1234  * can be changed to FX or RT.  Since these threads should be swapped in
1235  * as soon as they're runnable, rt_swapin returns SHRT_MAX, and fx_swapin
1236  * returns SHRT_MAX - 1, so that it gives deference to any swapped out
1237  * RT threads.
1238  */
1239 /* ARGSUSED */
1240 static pri_t
fx_swapin(kthread_t * t,int flags)1241 fx_swapin(kthread_t *t, int flags)
1242 {
1243 	pri_t	tpri = -1;
1244 
1245 	ASSERT(THREAD_LOCK_HELD(t));
1246 
1247 	if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1248 		tpri = (pri_t)SHRT_MAX - 1;
1249 	}
1250 
1251 	return (tpri);
1252 }
1253 
1254 /*
1255  * Return Values
1256  *	-1 if the thread isn't loaded or is not eligible to be swapped out.
1257  */
1258 /* ARGSUSED */
1259 static pri_t
fx_swapout(kthread_t * t,int flags)1260 fx_swapout(kthread_t *t, int flags)
1261 {
1262 	ASSERT(THREAD_LOCK_HELD(t));
1263 
1264 	return (-1);
1265 
1266 }
1267 
1268 /* ARGSUSED */
1269 static void
fx_stop(kthread_t * t,int why,int what)1270 fx_stop(kthread_t *t, int why, int what)
1271 {
1272 	fxproc_t *fxpp = (fxproc_t *)(t->t_cldata);
1273 
1274 	ASSERT(THREAD_LOCK_HELD(t));
1275 
1276 	if (FX_HAS_CB(fxpp)) {
1277 		FX_CB_STOP(FX_CALLB(fxpp), fxpp->fx_cookie);
1278 	}
1279 }
1280 
1281 /*
1282  * Check for time slice expiration.  If time slice has expired
1283  * set runrun to cause preemption.
1284  */
1285 static void
fx_tick(kthread_t * t)1286 fx_tick(kthread_t *t)
1287 {
1288 	boolean_t call_cpu_surrender = B_FALSE;
1289 	fxproc_t *fxpp;
1290 
1291 	ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1292 
1293 	thread_lock(t);
1294 
1295 	fxpp = (fxproc_t *)(t->t_cldata);
1296 
1297 	if (FX_HAS_CB(fxpp)) {
1298 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;
1299 		pri_t	newpri = fxpp->fx_pri;
1300 		FX_CB_TICK(FX_CALLB(fxpp), fxpp->fx_cookie,
1301 		    &new_quantum, &newpri);
1302 		FX_ADJUST_QUANTUM(new_quantum);
1303 		if ((int)new_quantum != fxpp->fx_pquantum) {
1304 			fxpp->fx_pquantum = (int)new_quantum;
1305 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1306 		}
1307 		FX_ADJUST_PRI(newpri);
1308 		if (newpri != fxpp->fx_pri) {
1309 			fxpp->fx_pri = newpri;
1310 			fx_change_priority(t, fxpp);
1311 		}
1312 	}
1313 
1314 	/*
1315 	 * Keep track of thread's project CPU usage.  Note that projects
1316 	 * get charged even when threads are running in the kernel.
1317 	 */
1318 	call_cpu_surrender =  CPUCAPS_CHARGE(t, &fxpp->fx_caps,
1319 	    CPUCAPS_CHARGE_ENFORCE);
1320 
1321 	if ((fxpp->fx_pquantum != FX_TQINF) &&
1322 	    (--fxpp->fx_timeleft <= 0)) {
1323 		pri_t	new_pri;
1324 
1325 		/*
1326 		 * If we're doing preemption control and trying to
1327 		 * avoid preempting this thread, just note that
1328 		 * the thread should yield soon and let it keep
1329 		 * running (unless it's been a while).
1330 		 */
1331 		if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1332 			if (fxpp->fx_timeleft > -SC_MAX_TICKS) {
1333 				DTRACE_SCHED1(schedctl__nopreempt,
1334 				    kthread_t *, t);
1335 				schedctl_set_yield(t, 1);
1336 				thread_unlock_nopreempt(t);
1337 				return;
1338 			}
1339 			DTRACE_SCHED1(schedctl__failsafe,
1340 			    kthread_t *, t);
1341 		}
1342 		new_pri = fx_dptbl[fxpp->fx_pri].fx_globpri;
1343 		ASSERT(new_pri >= 0 && new_pri <= fx_maxglobpri);
1344 		/*
1345 		 * When the priority of a thread is changed,
1346 		 * it may be necessary to adjust its position
1347 		 * on a sleep queue or dispatch queue. Even
1348 		 * when the priority is not changed, we need
1349 		 * to preserve round robin on dispatch queue.
1350 		 * The function thread_change_pri accomplishes
1351 		 * this.
1352 		 */
1353 		if (thread_change_pri(t, new_pri, 0)) {
1354 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1355 		} else {
1356 			call_cpu_surrender = B_TRUE;
1357 		}
1358 	} else if (t->t_state == TS_ONPROC &&
1359 	    t->t_pri < t->t_disp_queue->disp_maxrunpri) {
1360 		call_cpu_surrender = B_TRUE;
1361 	}
1362 
1363 	if (call_cpu_surrender) {
1364 		fxpp->fx_flags |= FXBACKQ;
1365 		cpu_surrender(t);
1366 	}
1367 	thread_unlock_nopreempt(t);	/* clock thread can't be preempted */
1368 }
1369 
1370 
1371 static void
fx_trapret(kthread_t * t)1372 fx_trapret(kthread_t *t)
1373 {
1374 	cpu_t		*cp = CPU;
1375 
1376 	ASSERT(THREAD_LOCK_HELD(t));
1377 	ASSERT(t == curthread);
1378 	ASSERT(cp->cpu_dispthread == t);
1379 	ASSERT(t->t_state == TS_ONPROC);
1380 }
1381 
1382 
1383 /*
1384  * Processes waking up go to the back of their queue.
1385  */
1386 static void
fx_wakeup(kthread_t * t)1387 fx_wakeup(kthread_t *t)
1388 {
1389 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);
1390 
1391 	ASSERT(THREAD_LOCK_HELD(t));
1392 
1393 	t->t_stime = ddi_get_lbolt();		/* time stamp for the swapper */
1394 	if (FX_HAS_CB(fxpp)) {
1395 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;
1396 		pri_t	newpri = fxpp->fx_pri;
1397 		FX_CB_WAKEUP(FX_CALLB(fxpp), fxpp->fx_cookie,
1398 		    &new_quantum, &newpri);
1399 		FX_ADJUST_QUANTUM(new_quantum);
1400 		if ((int)new_quantum != fxpp->fx_pquantum) {
1401 			fxpp->fx_pquantum = (int)new_quantum;
1402 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1403 		}
1404 
1405 		FX_ADJUST_PRI(newpri);
1406 		if (newpri != fxpp->fx_pri) {
1407 			fxpp->fx_pri = newpri;
1408 			THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);
1409 		}
1410 	}
1411 
1412 	fxpp->fx_flags &= ~FXBACKQ;
1413 
1414 	if (t->t_disp_time != ddi_get_lbolt())
1415 		setbackdq(t);
1416 	else
1417 		setfrontdq(t);
1418 }
1419 
1420 
1421 /*
1422  * When a thread yields, put it on the back of the run queue.
1423  */
1424 static void
fx_yield(kthread_t * t)1425 fx_yield(kthread_t *t)
1426 {
1427 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);
1428 
1429 	ASSERT(t == curthread);
1430 	ASSERT(THREAD_LOCK_HELD(t));
1431 
1432 	/*
1433 	 * Collect CPU usage spent before yielding CPU.
1434 	 */
1435 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ENFORCE);
1436 
1437 	if (FX_HAS_CB(fxpp))  {
1438 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;
1439 		pri_t	newpri = fxpp->fx_pri;
1440 		FX_CB_PREEMPT(FX_CALLB(fxpp), fxpp->fx_cookie,
1441 		    &new_quantum, &newpri);
1442 		FX_ADJUST_QUANTUM(new_quantum);
1443 		if ((int)new_quantum != fxpp->fx_pquantum) {
1444 			fxpp->fx_pquantum = (int)new_quantum;
1445 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1446 		}
1447 		FX_ADJUST_PRI(newpri);
1448 		fxpp->fx_pri = newpri;
1449 		THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);
1450 	}
1451 
1452 	/*
1453 	 * Clear the preemption control "yield" bit since the user is
1454 	 * doing a yield.
1455 	 */
1456 	if (t->t_schedctl)
1457 		schedctl_set_yield(t, 0);
1458 
1459 	if (fxpp->fx_timeleft <= 0) {
1460 		/*
1461 		 * Time slice was artificially extended to avoid
1462 		 * preemption, so pretend we're preempting it now.
1463 		 */
1464 		DTRACE_SCHED1(schedctl__yield, int, -fxpp->fx_timeleft);
1465 		fxpp->fx_timeleft = fxpp->fx_pquantum;
1466 		THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);
1467 		ASSERT(t->t_pri >= 0 && t->t_pri <= fx_maxglobpri);
1468 	}
1469 
1470 	fxpp->fx_flags &= ~FXBACKQ;
1471 	setbackdq(t);
1472 }
1473 
1474 /*
1475  * Increment the nice value of the specified thread by incr and
1476  * return the new value in *retvalp.
1477  */
1478 static int
fx_donice(kthread_t * t,cred_t * cr,int incr,int * retvalp)1479 fx_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
1480 {
1481 	int		newnice;
1482 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);
1483 	fxkparms_t	fxkparms;
1484 
1485 	ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1486 
1487 	/* If there's no change to priority, just return current setting */
1488 	if (incr == 0) {
1489 		if (retvalp) {
1490 			*retvalp = fxpp->fx_nice - NZERO;
1491 		}
1492 		return (0);
1493 	}
1494 
1495 	if ((incr < 0 || incr > 2 * NZERO) &&
1496 	    secpolicy_raisepriority(cr) != 0)
1497 		return (EPERM);
1498 
1499 	/*
1500 	 * Specifying a nice increment greater than the upper limit of
1501 	 * 2 * NZERO - 1 will result in the thread's nice value being
1502 	 * set to the upper limit.  We check for this before computing
1503 	 * the new value because otherwise we could get overflow
1504 	 * if a privileged user specified some ridiculous increment.
1505 	 */
1506 	if (incr > 2 * NZERO - 1)
1507 		incr = 2 * NZERO - 1;
1508 
1509 	newnice = fxpp->fx_nice + incr;
1510 	if (newnice > NZERO)
1511 		newnice = NZERO;
1512 	else if (newnice < 0)
1513 		newnice = 0;
1514 
1515 	fxkparms.fx_uprilim = fxkparms.fx_upri =
1516 	    -((newnice - NZERO) * fx_maxupri) / NZERO;
1517 
1518 	fxkparms.fx_cflags = FX_DOUPRILIM | FX_DOUPRI;
1519 
1520 	fxkparms.fx_tqntm = FX_TQDEF;
1521 
1522 	/*
1523 	 * Reset the uprilim and upri values of the thread. Adjust
1524 	 * time quantum accordingly.
1525 	 */
1526 
1527 	(void) fx_parmsset(t, (void *)&fxkparms, (id_t)0, (cred_t *)NULL);
1528 
1529 	/*
1530 	 * Although fx_parmsset already reset fx_nice it may
1531 	 * not have been set to precisely the value calculated above
1532 	 * because fx_parmsset determines the nice value from the
1533 	 * user priority and we may have truncated during the integer
1534 	 * conversion from nice value to user priority and back.
1535 	 * We reset fx_nice to the value we calculated above.
1536 	 */
1537 	fxpp->fx_nice = (char)newnice;
1538 
1539 	if (retvalp)
1540 		*retvalp = newnice - NZERO;
1541 
1542 	return (0);
1543 }
1544 
1545 /*
1546  * Increment the priority of the specified thread by incr and
1547  * return the new value in *retvalp.
1548  */
1549 static int
fx_doprio(kthread_t * t,cred_t * cr,int incr,int * retvalp)1550 fx_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
1551 {
1552 	int		newpri;
1553 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);
1554 	fxkparms_t	fxkparms;
1555 
1556 	ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1557 
1558 	/* If there's no change to priority, just return current setting */
1559 	if (incr == 0) {
1560 		*retvalp = fxpp->fx_pri;
1561 		return (0);
1562 	}
1563 
1564 	newpri = fxpp->fx_pri + incr;
1565 	if (newpri > fx_maxupri || newpri < 0)
1566 		return (EINVAL);
1567 
1568 	*retvalp = newpri;
1569 	fxkparms.fx_uprilim = fxkparms.fx_upri = newpri;
1570 	fxkparms.fx_tqntm = FX_NOCHANGE;
1571 	fxkparms.fx_cflags = FX_DOUPRILIM | FX_DOUPRI;
1572 
1573 	/*
1574 	 * Reset the uprilim and upri values of the thread.
1575 	 */
1576 	return (fx_parmsset(t, (void *)&fxkparms, (id_t)0, cr));
1577 }
1578 
1579 static void
fx_change_priority(kthread_t * t,fxproc_t * fxpp)1580 fx_change_priority(kthread_t *t, fxproc_t *fxpp)
1581 {
1582 	pri_t	new_pri;
1583 
1584 	ASSERT(THREAD_LOCK_HELD(t));
1585 	new_pri = fx_dptbl[fxpp->fx_pri].fx_globpri;
1586 	ASSERT(new_pri >= 0 && new_pri <= fx_maxglobpri);
1587 	t->t_cpri = fxpp->fx_pri;
1588 	if (t == curthread || t->t_state == TS_ONPROC) {
1589 		/* curthread is always onproc */
1590 		cpu_t	*cp = t->t_disp_queue->disp_cpu;
1591 		THREAD_CHANGE_PRI(t, new_pri);
1592 		if (t == cp->cpu_dispthread)
1593 			cp->cpu_dispatch_pri = DISP_PRIO(t);
1594 		if (DISP_MUST_SURRENDER(t)) {
1595 			fxpp->fx_flags |= FXBACKQ;
1596 			cpu_surrender(t);
1597 		} else {
1598 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1599 		}
1600 	} else {
1601 		/*
1602 		 * When the priority of a thread is changed,
1603 		 * it may be necessary to adjust its position
1604 		 * on a sleep queue or dispatch queue.
1605 		 * The function thread_change_pri accomplishes
1606 		 * this.
1607 		 */
1608 		if (thread_change_pri(t, new_pri, 0)) {
1609 			/*
1610 			 * The thread was on a run queue. Reset
1611 			 * its CPU timeleft from the quantum
1612 			 * associated with the new priority.
1613 			 */
1614 			fxpp->fx_timeleft = fxpp->fx_pquantum;
1615 		} else {
1616 			fxpp->fx_flags |= FXBACKQ;
1617 		}
1618 	}
1619 }
1620 
1621 static int
fx_alloc(void ** p,int flag)1622 fx_alloc(void **p, int flag)
1623 {
1624 	void *bufp;
1625 
1626 	bufp = kmem_alloc(sizeof (fxproc_t), flag);
1627 	if (bufp == NULL) {
1628 		return (ENOMEM);
1629 	} else {
1630 		*p = bufp;
1631 		return (0);
1632 	}
1633 }
1634 
1635 static void
fx_free(void * bufp)1636 fx_free(void *bufp)
1637 {
1638 	if (bufp)
1639 		kmem_free(bufp, sizeof (fxproc_t));
1640 }
1641 
1642 /*
1643  * Release the callback list mutex after successful lookup
1644  */
1645 void
fx_list_release(fxproc_t * fxpp)1646 fx_list_release(fxproc_t *fxpp)
1647 {
1648 	int index = FX_CB_LIST_HASH(fxpp->fx_ktid);
1649 	kmutex_t *lockp = &fx_cb_list_lock[index];
1650 	mutex_exit(lockp);
1651 }
1652 
1653 fxproc_t *
fx_list_lookup(kt_did_t ktid)1654 fx_list_lookup(kt_did_t ktid)
1655 {
1656 	int index = FX_CB_LIST_HASH(ktid);
1657 	kmutex_t *lockp = &fx_cb_list_lock[index];
1658 	fxproc_t *fxpp;
1659 
1660 	mutex_enter(lockp);
1661 
1662 	for (fxpp = fx_cb_plisthead[index].fx_cb_next;
1663 	    fxpp != &fx_cb_plisthead[index]; fxpp = fxpp->fx_cb_next) {
1664 		if (fxpp->fx_tp->t_cid == fx_cid && fxpp->fx_ktid == ktid &&
1665 		    fxpp->fx_callback != NULL) {
1666 			/*
1667 			 * The caller is responsible for calling
1668 			 * fx_list_release to drop the lock upon
1669 			 * successful lookup
1670 			 */
1671 			return (fxpp);
1672 		}
1673 	}
1674 	mutex_exit(lockp);
1675 	return ((fxproc_t *)NULL);
1676 }
1677 
1678 
1679 /*
1680  * register a callback set of routines for current thread
1681  * thread should already be in FX class
1682  */
1683 int
fx_register_callbacks(fx_callbacks_t * fx_callback,fx_cookie_t cookie,pri_t pri,clock_t quantum)1684 fx_register_callbacks(fx_callbacks_t *fx_callback, fx_cookie_t cookie,
1685     pri_t pri, clock_t quantum)
1686 {
1687 
1688 	fxproc_t	*fxpp;
1689 
1690 	if (fx_callback == NULL)
1691 		return (EINVAL);
1692 
1693 	if (secpolicy_dispadm(CRED()) != 0)
1694 		return (EPERM);
1695 
1696 	if (FX_CB_VERSION(fx_callback) != FX_CALLB_REV)
1697 		return (EINVAL);
1698 
1699 	if (!FX_ISVALID(pri, quantum))
1700 		return (EINVAL);
1701 
1702 	thread_lock(curthread);		/* get dispatcher lock on thread */
1703 
1704 	if (curthread->t_cid != fx_cid) {
1705 		thread_unlock(curthread);
1706 		return (EINVAL);
1707 	}
1708 
1709 	fxpp = (fxproc_t *)(curthread->t_cldata);
1710 	ASSERT(fxpp != NULL);
1711 	if (FX_HAS_CB(fxpp)) {
1712 		thread_unlock(curthread);
1713 		return (EINVAL);
1714 	}
1715 
1716 	fxpp->fx_callback = fx_callback;
1717 	fxpp->fx_cookie = cookie;
1718 
1719 	if (pri != FX_CB_NOCHANGE) {
1720 		fxpp->fx_pri = pri;
1721 		FX_ADJUST_PRI(fxpp->fx_pri);
1722 		if (quantum == FX_TQDEF) {
1723 			fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;
1724 		} else if (quantum == FX_TQINF) {
1725 			fxpp->fx_pquantum = FX_TQINF;
1726 		} else if (quantum != FX_NOCHANGE) {
1727 			FX_ADJUST_QUANTUM(quantum);
1728 			fxpp->fx_pquantum = quantum;
1729 		}
1730 	} else if (quantum != FX_NOCHANGE && quantum != FX_TQDEF) {
1731 		if (quantum == FX_TQINF)
1732 			fxpp->fx_pquantum = FX_TQINF;
1733 		else {
1734 			FX_ADJUST_QUANTUM(quantum);
1735 			fxpp->fx_pquantum = quantum;
1736 		}
1737 	}
1738 
1739 	fxpp->fx_ktid = ddi_get_kt_did();
1740 
1741 	fx_change_priority(curthread, fxpp);
1742 
1743 	thread_unlock(curthread);
1744 
1745 	/*
1746 	 * Link new structure into fxproc list.
1747 	 */
1748 	FX_CB_LIST_INSERT(fxpp);
1749 	return (0);
1750 }
1751 
1752 /* unregister a callback set of routines for current thread */
1753 int
fx_unregister_callbacks()1754 fx_unregister_callbacks()
1755 {
1756 	fxproc_t	*fxpp;
1757 
1758 	if ((fxpp = fx_list_lookup(ddi_get_kt_did())) == NULL) {
1759 		/*
1760 		 * did not have a registered callback;
1761 		 */
1762 		return (EINVAL);
1763 	}
1764 
1765 	thread_lock(fxpp->fx_tp);
1766 	fxpp->fx_callback = NULL;
1767 	fxpp->fx_cookie = 0;
1768 	thread_unlock(fxpp->fx_tp);
1769 	fx_list_release(fxpp);
1770 
1771 	FX_CB_LIST_DELETE(fxpp);
1772 	return (0);
1773 }
1774 
1775 /*
1776  * modify priority and/or quantum value of a thread with callback
1777  */
1778 int
fx_modify_priority(kt_did_t ktid,clock_t quantum,pri_t pri)1779 fx_modify_priority(kt_did_t ktid, clock_t quantum, pri_t pri)
1780 {
1781 	fxproc_t	*fxpp;
1782 
1783 	if (!FX_ISVALID(pri, quantum))
1784 		return (EINVAL);
1785 
1786 	if ((fxpp = fx_list_lookup(ktid)) == NULL) {
1787 		/*
1788 		 * either thread had exited or did not have a registered
1789 		 * callback;
1790 		 */
1791 		return (ESRCH);
1792 	}
1793 
1794 	thread_lock(fxpp->fx_tp);
1795 
1796 	if (pri != FX_CB_NOCHANGE) {
1797 		fxpp->fx_pri = pri;
1798 		FX_ADJUST_PRI(fxpp->fx_pri);
1799 		if (quantum == FX_TQDEF) {
1800 			fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;
1801 		} else if (quantum == FX_TQINF) {
1802 			fxpp->fx_pquantum = FX_TQINF;
1803 		} else if (quantum != FX_NOCHANGE) {
1804 			FX_ADJUST_QUANTUM(quantum);
1805 			fxpp->fx_pquantum = quantum;
1806 		}
1807 	} else if (quantum != FX_NOCHANGE && quantum != FX_TQDEF) {
1808 		if (quantum == FX_TQINF) {
1809 			fxpp->fx_pquantum = FX_TQINF;
1810 		} else {
1811 			FX_ADJUST_QUANTUM(quantum);
1812 			fxpp->fx_pquantum = quantum;
1813 		}
1814 	}
1815 
1816 	fx_change_priority(fxpp->fx_tp, fxpp);
1817 
1818 	thread_unlock(fxpp->fx_tp);
1819 	fx_list_release(fxpp);
1820 	return (0);
1821 }
1822 
1823 
1824 /*
1825  * return an iblock cookie for mutex initialization to be used in callbacks
1826  */
1827 void *
fx_get_mutex_cookie()1828 fx_get_mutex_cookie()
1829 {
1830 	return ((void *)(uintptr_t)__ipltospl(DISP_LEVEL));
1831 }
1832 
1833 /*
1834  * return maximum relative priority
1835  */
1836 pri_t
fx_get_maxpri()1837 fx_get_maxpri()
1838 {
1839 	return (fx_maxumdpri);
1840 }
1841