1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2010 The FreeBSD Foundation
5 *
6 * This software was developed by Edward Tomasz Napierala under sponsorship
7 * from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_sched.h"
33
34 #include <sys/param.h>
35 #include <sys/buf.h>
36 #include <sys/systm.h>
37 #include <sys/eventhandler.h>
38 #include <sys/jail.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/lock.h>
42 #include <sys/loginclass.h>
43 #include <sys/malloc.h>
44 #include <sys/mutex.h>
45 #include <sys/proc.h>
46 #include <sys/racct.h>
47 #include <sys/resourcevar.h>
48 #include <sys/sbuf.h>
49 #include <sys/sched.h>
50 #include <sys/sdt.h>
51 #include <sys/smp.h>
52 #include <sys/sx.h>
53 #include <sys/sysctl.h>
54 #include <sys/sysproto.h>
55 #include <sys/umtxvar.h>
56 #include <machine/smp.h>
57
58 #ifdef RCTL
59 #include <sys/rctl.h>
60 #endif
61
62 FEATURE(racct, "Resource Accounting");
63
64 /*
65 * Do not block processes that have their %cpu usage <= pcpu_threshold.
66 */
67 static int pcpu_threshold = 1;
68 #ifdef RACCT_DEFAULT_TO_DISABLED
69 bool __read_frequently racct_enable = false;
70 #else
71 bool __read_frequently racct_enable = true;
72 #endif
73
74 SYSCTL_NODE(_kern, OID_AUTO, racct, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
75 "Resource Accounting");
76 SYSCTL_BOOL(_kern_racct, OID_AUTO, enable, CTLFLAG_RDTUN, &racct_enable,
77 0, "Enable RACCT/RCTL");
78 SYSCTL_UINT(_kern_racct, OID_AUTO, pcpu_threshold, CTLFLAG_RW, &pcpu_threshold,
79 0, "Processes with higher %cpu usage than this value can be throttled.");
80
81 /*
82 * How many seconds it takes to use the scheduler %cpu calculations. When a
83 * process starts, we compute its %cpu usage by dividing its runtime by the
84 * process wall clock time. After RACCT_PCPU_SECS pass, we use the value
85 * provided by the scheduler.
86 */
87 #define RACCT_PCPU_SECS 3
88
89 struct mtx racct_lock;
90 MTX_SYSINIT(racct_lock, &racct_lock, "racct lock", MTX_DEF);
91
92 static uma_zone_t racct_zone;
93
94 static void racct_sub_racct(struct racct *dest, const struct racct *src);
95 static void racct_sub_cred_locked(struct ucred *cred, int resource,
96 uint64_t amount);
97 static void racct_add_cred_locked(struct ucred *cred, int resource,
98 uint64_t amount);
99 static int racct_set_locked(struct proc *p, int resource, uint64_t amount,
100 int force);
101 static void racct_updatepcpu_locked(struct proc *p);
102 static void racct_updatepcpu_racct_locked(struct racct *racct);
103 static void racct_updatepcpu_containers(void);
104 static void racct_settime_locked(struct proc *p, bool exit);
105 static void racct_zeropcpu_locked(struct proc *p);
106
107 SDT_PROVIDER_DEFINE(racct);
108 SDT_PROBE_DEFINE3(racct, , rusage, add,
109 "struct proc *", "int", "uint64_t");
110 SDT_PROBE_DEFINE3(racct, , rusage, add__failure,
111 "struct proc *", "int", "uint64_t");
112 SDT_PROBE_DEFINE3(racct, , rusage, add__buf,
113 "struct proc *", "const struct buf *", "int");
114 SDT_PROBE_DEFINE3(racct, , rusage, add__cred,
115 "struct ucred *", "int", "uint64_t");
116 SDT_PROBE_DEFINE3(racct, , rusage, add__force,
117 "struct proc *", "int", "uint64_t");
118 SDT_PROBE_DEFINE3(racct, , rusage, set,
119 "struct proc *", "int", "uint64_t");
120 SDT_PROBE_DEFINE3(racct, , rusage, set__failure,
121 "struct proc *", "int", "uint64_t");
122 SDT_PROBE_DEFINE3(racct, , rusage, set__force,
123 "struct proc *", "int", "uint64_t");
124 SDT_PROBE_DEFINE3(racct, , rusage, sub,
125 "struct proc *", "int", "uint64_t");
126 SDT_PROBE_DEFINE3(racct, , rusage, sub__cred,
127 "struct ucred *", "int", "uint64_t");
128 SDT_PROBE_DEFINE1(racct, , racct, create,
129 "struct racct *");
130 SDT_PROBE_DEFINE1(racct, , racct, destroy,
131 "struct racct *");
132 SDT_PROBE_DEFINE2(racct, , racct, join,
133 "struct racct *", "struct racct *");
134 SDT_PROBE_DEFINE2(racct, , racct, join__failure,
135 "struct racct *", "struct racct *");
136 SDT_PROBE_DEFINE2(racct, , racct, leave,
137 "struct racct *", "struct racct *");
138
139 int racct_types[] = {
140 [RACCT_CPU] =
141 RACCT_IN_MILLIONS,
142 [RACCT_DATA] =
143 RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
144 [RACCT_STACK] =
145 RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
146 [RACCT_CORE] =
147 RACCT_DENIABLE,
148 [RACCT_RSS] =
149 RACCT_RECLAIMABLE,
150 [RACCT_MEMLOCK] =
151 RACCT_RECLAIMABLE | RACCT_DENIABLE,
152 [RACCT_NPROC] =
153 RACCT_RECLAIMABLE | RACCT_DENIABLE,
154 [RACCT_NOFILE] =
155 RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
156 [RACCT_VMEM] =
157 RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
158 [RACCT_NPTS] =
159 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
160 [RACCT_SWAP] =
161 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
162 [RACCT_NTHR] =
163 RACCT_RECLAIMABLE | RACCT_DENIABLE,
164 [RACCT_MSGQQUEUED] =
165 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
166 [RACCT_MSGQSIZE] =
167 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
168 [RACCT_NMSGQ] =
169 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
170 [RACCT_NSEM] =
171 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
172 [RACCT_NSEMOP] =
173 RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
174 [RACCT_NSHM] =
175 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
176 [RACCT_SHMSIZE] =
177 RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
178 [RACCT_WALLCLOCK] =
179 RACCT_IN_MILLIONS,
180 [RACCT_PCTCPU] =
181 RACCT_DECAYING | RACCT_DENIABLE | RACCT_IN_MILLIONS,
182 [RACCT_READBPS] =
183 RACCT_DECAYING,
184 [RACCT_WRITEBPS] =
185 RACCT_DECAYING,
186 [RACCT_READIOPS] =
187 RACCT_DECAYING,
188 [RACCT_WRITEIOPS] =
189 RACCT_DECAYING };
190
191 static const fixpt_t RACCT_DECAY_FACTOR = 0.3 * FSCALE;
192
193 #ifdef SCHED_4BSD
194 /*
195 * Contains intermediate values for %cpu calculations to avoid using floating
196 * point in the kernel.
197 * ccpu_exp[k] = FSCALE * (ccpu/FSCALE)^k = FSCALE * exp(-k/20)
198 * It is needed only for the 4BSD scheduler, because in ULE, the ccpu equals to
199 * zero so the calculations are more straightforward.
200 */
201 fixpt_t ccpu_exp[] = {
202 [0] = FSCALE * 1,
203 [1] = FSCALE * 0.95122942450071400909,
204 [2] = FSCALE * 0.90483741803595957316,
205 [3] = FSCALE * 0.86070797642505780722,
206 [4] = FSCALE * 0.81873075307798185866,
207 [5] = FSCALE * 0.77880078307140486824,
208 [6] = FSCALE * 0.74081822068171786606,
209 [7] = FSCALE * 0.70468808971871343435,
210 [8] = FSCALE * 0.67032004603563930074,
211 [9] = FSCALE * 0.63762815162177329314,
212 [10] = FSCALE * 0.60653065971263342360,
213 [11] = FSCALE * 0.57694981038048669531,
214 [12] = FSCALE * 0.54881163609402643262,
215 [13] = FSCALE * 0.52204577676101604789,
216 [14] = FSCALE * 0.49658530379140951470,
217 [15] = FSCALE * 0.47236655274101470713,
218 [16] = FSCALE * 0.44932896411722159143,
219 [17] = FSCALE * 0.42741493194872666992,
220 [18] = FSCALE * 0.40656965974059911188,
221 [19] = FSCALE * 0.38674102345450120691,
222 [20] = FSCALE * 0.36787944117144232159,
223 [21] = FSCALE * 0.34993774911115535467,
224 [22] = FSCALE * 0.33287108369807955328,
225 [23] = FSCALE * 0.31663676937905321821,
226 [24] = FSCALE * 0.30119421191220209664,
227 [25] = FSCALE * 0.28650479686019010032,
228 [26] = FSCALE * 0.27253179303401260312,
229 [27] = FSCALE * 0.25924026064589150757,
230 [28] = FSCALE * 0.24659696394160647693,
231 [29] = FSCALE * 0.23457028809379765313,
232 [30] = FSCALE * 0.22313016014842982893,
233 [31] = FSCALE * 0.21224797382674305771,
234 [32] = FSCALE * 0.20189651799465540848,
235 [33] = FSCALE * 0.19204990862075411423,
236 [34] = FSCALE * 0.18268352405273465022,
237 [35] = FSCALE * 0.17377394345044512668,
238 [36] = FSCALE * 0.16529888822158653829,
239 [37] = FSCALE * 0.15723716631362761621,
240 [38] = FSCALE * 0.14956861922263505264,
241 [39] = FSCALE * 0.14227407158651357185,
242 [40] = FSCALE * 0.13533528323661269189,
243 [41] = FSCALE * 0.12873490358780421886,
244 [42] = FSCALE * 0.12245642825298191021,
245 [43] = FSCALE * 0.11648415777349695786,
246 [44] = FSCALE * 0.11080315836233388333,
247 [45] = FSCALE * 0.10539922456186433678,
248 [46] = FSCALE * 0.10025884372280373372,
249 [47] = FSCALE * 0.09536916221554961888,
250 [48] = FSCALE * 0.09071795328941250337,
251 [49] = FSCALE * 0.08629358649937051097,
252 [50] = FSCALE * 0.08208499862389879516,
253 [51] = FSCALE * 0.07808166600115315231,
254 [52] = FSCALE * 0.07427357821433388042,
255 [53] = FSCALE * 0.07065121306042958674,
256 [54] = FSCALE * 0.06720551273974976512,
257 [55] = FSCALE * 0.06392786120670757270,
258 [56] = FSCALE * 0.06081006262521796499,
259 [57] = FSCALE * 0.05784432087483846296,
260 [58] = FSCALE * 0.05502322005640722902,
261 [59] = FSCALE * 0.05233970594843239308,
262 [60] = FSCALE * 0.04978706836786394297,
263 [61] = FSCALE * 0.04735892439114092119,
264 [62] = FSCALE * 0.04504920239355780606,
265 [63] = FSCALE * 0.04285212686704017991,
266 [64] = FSCALE * 0.04076220397836621516,
267 [65] = FSCALE * 0.03877420783172200988,
268 [66] = FSCALE * 0.03688316740124000544,
269 [67] = FSCALE * 0.03508435410084502588,
270 [68] = FSCALE * 0.03337326996032607948,
271 [69] = FSCALE * 0.03174563637806794323,
272 [70] = FSCALE * 0.03019738342231850073,
273 [71] = FSCALE * 0.02872463965423942912,
274 [72] = FSCALE * 0.02732372244729256080,
275 [73] = FSCALE * 0.02599112877875534358,
276 [74] = FSCALE * 0.02472352647033939120,
277 [75] = FSCALE * 0.02351774585600910823,
278 [76] = FSCALE * 0.02237077185616559577,
279 [77] = FSCALE * 0.02127973643837716938,
280 [78] = FSCALE * 0.02024191144580438847,
281 [79] = FSCALE * 0.01925470177538692429,
282 [80] = FSCALE * 0.01831563888873418029,
283 [81] = FSCALE * 0.01742237463949351138,
284 [82] = FSCALE * 0.01657267540176124754,
285 [83] = FSCALE * 0.01576441648485449082,
286 [84] = FSCALE * 0.01499557682047770621,
287 [85] = FSCALE * 0.01426423390899925527,
288 [86] = FSCALE * 0.01356855901220093175,
289 [87] = FSCALE * 0.01290681258047986886,
290 [88] = FSCALE * 0.01227733990306844117,
291 [89] = FSCALE * 0.01167856697039544521,
292 [90] = FSCALE * 0.01110899653824230649,
293 [91] = FSCALE * 0.01056720438385265337,
294 [92] = FSCALE * 0.01005183574463358164,
295 [93] = FSCALE * 0.00956160193054350793,
296 [94] = FSCALE * 0.00909527710169581709,
297 [95] = FSCALE * 0.00865169520312063417,
298 [96] = FSCALE * 0.00822974704902002884,
299 [97] = FSCALE * 0.00782837754922577143,
300 [98] = FSCALE * 0.00744658307092434051,
301 [99] = FSCALE * 0.00708340892905212004,
302 [100] = FSCALE * 0.00673794699908546709,
303 [101] = FSCALE * 0.00640933344625638184,
304 [102] = FSCALE * 0.00609674656551563610,
305 [103] = FSCALE * 0.00579940472684214321,
306 [104] = FSCALE * 0.00551656442076077241,
307 [105] = FSCALE * 0.00524751839918138427,
308 [106] = FSCALE * 0.00499159390691021621,
309 [107] = FSCALE * 0.00474815099941147558,
310 [108] = FSCALE * 0.00451658094261266798,
311 [109] = FSCALE * 0.00429630469075234057,
312 [110] = FSCALE * 0.00408677143846406699,
313 };
314 #endif
315
316 #define CCPU_EXP_MAX 110
317
318 static void
racct_add_racct(struct racct * dest,const struct racct * src)319 racct_add_racct(struct racct *dest, const struct racct *src)
320 {
321 int i;
322
323 ASSERT_RACCT_ENABLED();
324 RACCT_LOCK_ASSERT();
325
326 /*
327 * Update resource usage in dest.
328 */
329 for (i = 0; i <= RACCT_MAX; i++) {
330 KASSERT(dest->r_resources[i] >= 0,
331 ("%s: resource %d propagation meltdown: dest < 0",
332 __func__, i));
333 KASSERT(src->r_resources[i] >= 0,
334 ("%s: resource %d propagation meltdown: src < 0",
335 __func__, i));
336 dest->r_resources[i] += src->r_resources[i];
337 }
338 }
339
340 static void
racct_sub_racct(struct racct * dest,const struct racct * src)341 racct_sub_racct(struct racct *dest, const struct racct *src)
342 {
343 int i;
344
345 ASSERT_RACCT_ENABLED();
346 RACCT_LOCK_ASSERT();
347
348 /*
349 * Update resource usage in dest.
350 */
351 for (i = 0; i <= RACCT_MAX; i++) {
352 if (!RACCT_IS_SLOPPY(i) && !RACCT_IS_DECAYING(i)) {
353 KASSERT(dest->r_resources[i] >= 0,
354 ("%s: resource %d propagation meltdown: dest < 0",
355 __func__, i));
356 KASSERT(src->r_resources[i] >= 0,
357 ("%s: resource %d propagation meltdown: src < 0",
358 __func__, i));
359 KASSERT(src->r_resources[i] <= dest->r_resources[i],
360 ("%s: resource %d propagation meltdown: src > dest",
361 __func__, i));
362 }
363 if (RACCT_CAN_DROP(i)) {
364 dest->r_resources[i] -= src->r_resources[i];
365 if (dest->r_resources[i] < 0)
366 dest->r_resources[i] = 0;
367 }
368 }
369 }
370
371 void
racct_create(struct racct ** racctp)372 racct_create(struct racct **racctp)
373 {
374
375 if (!racct_enable)
376 return;
377
378 SDT_PROBE1(racct, , racct, create, racctp);
379
380 KASSERT(*racctp == NULL, ("racct already allocated"));
381
382 *racctp = uma_zalloc(racct_zone, M_WAITOK | M_ZERO);
383 }
384
385 static void
racct_destroy_locked(struct racct ** racctp)386 racct_destroy_locked(struct racct **racctp)
387 {
388 struct racct *racct;
389 int i;
390
391 ASSERT_RACCT_ENABLED();
392
393 SDT_PROBE1(racct, , racct, destroy, racctp);
394
395 RACCT_LOCK_ASSERT();
396 KASSERT(racctp != NULL, ("NULL racctp"));
397 KASSERT(*racctp != NULL, ("NULL racct"));
398
399 racct = *racctp;
400
401 for (i = 0; i <= RACCT_MAX; i++) {
402 if (RACCT_IS_SLOPPY(i))
403 continue;
404 if (!RACCT_IS_RECLAIMABLE(i))
405 continue;
406 KASSERT(racct->r_resources[i] == 0,
407 ("destroying non-empty racct: "
408 "%ju allocated for resource %d\n",
409 racct->r_resources[i], i));
410 }
411 uma_zfree(racct_zone, racct);
412 *racctp = NULL;
413 }
414
415 void
racct_destroy(struct racct ** racct)416 racct_destroy(struct racct **racct)
417 {
418
419 if (!racct_enable)
420 return;
421
422 RACCT_LOCK();
423 racct_destroy_locked(racct);
424 RACCT_UNLOCK();
425 }
426
427 /*
428 * Increase consumption of 'resource' by 'amount' for 'racct',
429 * but not its parents. Differently from other cases, 'amount' here
430 * may be less than zero.
431 */
432 static void
racct_adjust_resource(struct racct * racct,int resource,int64_t amount)433 racct_adjust_resource(struct racct *racct, int resource,
434 int64_t amount)
435 {
436
437 ASSERT_RACCT_ENABLED();
438 RACCT_LOCK_ASSERT();
439 KASSERT(racct != NULL, ("NULL racct"));
440
441 racct->r_resources[resource] += amount;
442 if (racct->r_resources[resource] < 0) {
443 KASSERT(RACCT_IS_SLOPPY(resource) || RACCT_IS_DECAYING(resource),
444 ("%s: resource %d usage < 0", __func__, resource));
445 racct->r_resources[resource] = 0;
446 }
447 }
448
449 static int
racct_add_locked(struct proc * p,int resource,uint64_t amount,int force)450 racct_add_locked(struct proc *p, int resource, uint64_t amount, int force)
451 {
452 #ifdef RCTL
453 int error;
454 #endif
455
456 ASSERT_RACCT_ENABLED();
457
458 /*
459 * We need proc lock to dereference p->p_ucred.
460 */
461 PROC_LOCK_ASSERT(p, MA_OWNED);
462
463 #ifdef RCTL
464 error = rctl_enforce(p, resource, amount);
465 if (error && !force && RACCT_IS_DENIABLE(resource)) {
466 SDT_PROBE3(racct, , rusage, add__failure, p, resource, amount);
467 return (error);
468 }
469 #endif
470 racct_adjust_resource(p->p_racct, resource, amount);
471 racct_add_cred_locked(p->p_ucred, resource, amount);
472
473 return (0);
474 }
475
476 /*
477 * Increase allocation of 'resource' by 'amount' for process 'p'.
478 * Return 0 if it's below limits, or errno, if it's not.
479 */
480 int
racct_add(struct proc * p,int resource,uint64_t amount)481 racct_add(struct proc *p, int resource, uint64_t amount)
482 {
483 int error;
484
485 if (!racct_enable)
486 return (0);
487
488 SDT_PROBE3(racct, , rusage, add, p, resource, amount);
489
490 RACCT_LOCK();
491 error = racct_add_locked(p, resource, amount, 0);
492 RACCT_UNLOCK();
493 return (error);
494 }
495
496 /*
497 * Increase allocation of 'resource' by 'amount' for process 'p'.
498 * Doesn't check for limits and never fails.
499 */
500 void
racct_add_force(struct proc * p,int resource,uint64_t amount)501 racct_add_force(struct proc *p, int resource, uint64_t amount)
502 {
503
504 if (!racct_enable)
505 return;
506
507 SDT_PROBE3(racct, , rusage, add__force, p, resource, amount);
508
509 RACCT_LOCK();
510 racct_add_locked(p, resource, amount, 1);
511 RACCT_UNLOCK();
512 }
513
514 static void
racct_add_cred_locked(struct ucred * cred,int resource,uint64_t amount)515 racct_add_cred_locked(struct ucred *cred, int resource, uint64_t amount)
516 {
517 struct prison *pr;
518
519 ASSERT_RACCT_ENABLED();
520
521 racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, amount);
522 for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent)
523 racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource,
524 amount);
525 racct_adjust_resource(cred->cr_loginclass->lc_racct, resource, amount);
526 }
527
528 /*
529 * Increase allocation of 'resource' by 'amount' for credential 'cred'.
530 * Doesn't check for limits and never fails.
531 */
532 void
racct_add_cred(struct ucred * cred,int resource,uint64_t amount)533 racct_add_cred(struct ucred *cred, int resource, uint64_t amount)
534 {
535
536 if (!racct_enable)
537 return;
538
539 SDT_PROBE3(racct, , rusage, add__cred, cred, resource, amount);
540
541 RACCT_LOCK();
542 racct_add_cred_locked(cred, resource, amount);
543 RACCT_UNLOCK();
544 }
545
546 /*
547 * Account for disk IO resource consumption. Checks for limits,
548 * but never fails, due to disk limits being undeniable.
549 */
550 void
racct_add_buf(struct proc * p,const struct buf * bp,int is_write)551 racct_add_buf(struct proc *p, const struct buf *bp, int is_write)
552 {
553
554 ASSERT_RACCT_ENABLED();
555 PROC_LOCK_ASSERT(p, MA_OWNED);
556
557 SDT_PROBE3(racct, , rusage, add__buf, p, bp, is_write);
558
559 RACCT_LOCK();
560 if (is_write) {
561 racct_add_locked(curproc, RACCT_WRITEBPS, bp->b_bcount, 1);
562 racct_add_locked(curproc, RACCT_WRITEIOPS, 1, 1);
563 } else {
564 racct_add_locked(curproc, RACCT_READBPS, bp->b_bcount, 1);
565 racct_add_locked(curproc, RACCT_READIOPS, 1, 1);
566 }
567 RACCT_UNLOCK();
568 }
569
570 static void
racct_settime_locked(struct proc * p,bool exit)571 racct_settime_locked(struct proc *p, bool exit)
572 {
573 struct thread *td;
574 struct timeval wallclock;
575 uint64_t runtime;
576
577 ASSERT_RACCT_ENABLED();
578 RACCT_LOCK_ASSERT();
579 PROC_LOCK_ASSERT(p, MA_OWNED);
580
581 if (exit) {
582 /*
583 * proc_reap() has already calculated rux
584 * and added crux to rux.
585 */
586 runtime = cputick2usec(p->p_rux.rux_runtime -
587 p->p_crux.rux_runtime);
588 } else {
589 PROC_STATLOCK(p);
590 FOREACH_THREAD_IN_PROC(p, td)
591 ruxagg(p, td);
592 PROC_STATUNLOCK(p);
593 runtime = cputick2usec(p->p_rux.rux_runtime);
594 }
595 microuptime(&wallclock);
596 timevalsub(&wallclock, &p->p_stats->p_start);
597
598 racct_set_locked(p, RACCT_CPU, runtime, 0);
599 racct_set_locked(p, RACCT_WALLCLOCK,
600 (uint64_t)wallclock.tv_sec * 1000000 +
601 wallclock.tv_usec, 0);
602 }
603
604 static int
racct_set_locked(struct proc * p,int resource,uint64_t amount,int force)605 racct_set_locked(struct proc *p, int resource, uint64_t amount, int force)
606 {
607 int64_t old_amount, diff_proc, diff_cred;
608 #ifdef RCTL
609 int error;
610 #endif
611
612 ASSERT_RACCT_ENABLED();
613
614 /*
615 * We need proc lock to dereference p->p_ucred.
616 */
617 PROC_LOCK_ASSERT(p, MA_OWNED);
618
619 old_amount = p->p_racct->r_resources[resource];
620 /*
621 * The diffs may be negative.
622 */
623 diff_proc = amount - old_amount;
624 diff_cred = diff_proc;
625 #ifdef notyet
626 KASSERT(diff_proc >= 0 || RACCT_CAN_DROP(resource),
627 ("%s: usage of non-droppable resource %d dropping", __func__,
628 resource));
629 #endif
630 #ifdef RCTL
631 if (diff_proc > 0) {
632 error = rctl_enforce(p, resource, diff_proc);
633 if (error && !force && RACCT_IS_DENIABLE(resource)) {
634 SDT_PROBE3(racct, , rusage, set__failure, p, resource,
635 amount);
636 return (error);
637 }
638 }
639 #endif
640 racct_adjust_resource(p->p_racct, resource, diff_proc);
641 if (diff_cred > 0)
642 racct_add_cred_locked(p->p_ucred, resource, diff_cred);
643 else if (diff_cred < 0)
644 racct_sub_cred_locked(p->p_ucred, resource, -diff_cred);
645
646 return (0);
647 }
648
649 /*
650 * Set allocation of 'resource' to 'amount' for process 'p'.
651 * Return 0 if it's below limits, or errno, if it's not.
652 *
653 * Note that decreasing the allocation always returns 0,
654 * even if it's above the limit.
655 */
656 int
racct_set_unlocked(struct proc * p,int resource,uint64_t amount)657 racct_set_unlocked(struct proc *p, int resource, uint64_t amount)
658 {
659 int error;
660
661 ASSERT_RACCT_ENABLED();
662 PROC_LOCK(p);
663 error = racct_set(p, resource, amount);
664 PROC_UNLOCK(p);
665 return (error);
666 }
667
668 int
racct_set(struct proc * p,int resource,uint64_t amount)669 racct_set(struct proc *p, int resource, uint64_t amount)
670 {
671 int error;
672
673 if (!racct_enable)
674 return (0);
675
676 SDT_PROBE3(racct, , rusage, set__force, p, resource, amount);
677
678 RACCT_LOCK();
679 error = racct_set_locked(p, resource, amount, 0);
680 RACCT_UNLOCK();
681 return (error);
682 }
683
684 void
racct_set_force(struct proc * p,int resource,uint64_t amount)685 racct_set_force(struct proc *p, int resource, uint64_t amount)
686 {
687
688 if (!racct_enable)
689 return;
690
691 SDT_PROBE3(racct, , rusage, set, p, resource, amount);
692
693 RACCT_LOCK();
694 racct_set_locked(p, resource, amount, 1);
695 RACCT_UNLOCK();
696 }
697
698 /*
699 * Returns amount of 'resource' the process 'p' can keep allocated.
700 * Allocating more than that would be denied, unless the resource
701 * is marked undeniable. Amount of already allocated resource does
702 * not matter.
703 */
704 uint64_t
racct_get_limit(struct proc * p,int resource)705 racct_get_limit(struct proc *p, int resource)
706 {
707 #ifdef RCTL
708 uint64_t available;
709
710 if (!racct_enable)
711 return (UINT64_MAX);
712
713 RACCT_LOCK();
714 available = rctl_get_limit(p, resource);
715 RACCT_UNLOCK();
716
717 return (available);
718 #else
719
720 return (UINT64_MAX);
721 #endif
722 }
723
724 /*
725 * Returns amount of 'resource' the process 'p' can keep allocated.
726 * Allocating more than that would be denied, unless the resource
727 * is marked undeniable. Amount of already allocated resource does
728 * matter.
729 */
730 uint64_t
racct_get_available(struct proc * p,int resource)731 racct_get_available(struct proc *p, int resource)
732 {
733 #ifdef RCTL
734 uint64_t available;
735
736 if (!racct_enable)
737 return (UINT64_MAX);
738
739 RACCT_LOCK();
740 available = rctl_get_available(p, resource);
741 RACCT_UNLOCK();
742
743 return (available);
744 #else
745
746 return (UINT64_MAX);
747 #endif
748 }
749
750 /*
751 * Returns amount of the %cpu resource that process 'p' can add to its %cpu
752 * utilization. Adding more than that would lead to the process being
753 * throttled.
754 */
755 static int64_t
racct_pcpu_available(struct proc * p)756 racct_pcpu_available(struct proc *p)
757 {
758 #ifdef RCTL
759 uint64_t available;
760
761 ASSERT_RACCT_ENABLED();
762
763 RACCT_LOCK();
764 available = rctl_pcpu_available(p);
765 RACCT_UNLOCK();
766
767 return (available);
768 #else
769
770 return (INT64_MAX);
771 #endif
772 }
773
774 /*
775 * Decrease allocation of 'resource' by 'amount' for process 'p'.
776 */
777 void
racct_sub(struct proc * p,int resource,uint64_t amount)778 racct_sub(struct proc *p, int resource, uint64_t amount)
779 {
780
781 if (!racct_enable)
782 return;
783
784 SDT_PROBE3(racct, , rusage, sub, p, resource, amount);
785
786 /*
787 * We need proc lock to dereference p->p_ucred.
788 */
789 PROC_LOCK_ASSERT(p, MA_OWNED);
790 KASSERT(RACCT_CAN_DROP(resource),
791 ("%s: called for non-droppable resource %d", __func__, resource));
792
793 RACCT_LOCK();
794 KASSERT(amount <= p->p_racct->r_resources[resource],
795 ("%s: freeing %ju of resource %d, which is more "
796 "than allocated %jd for %s (pid %d)", __func__, amount, resource,
797 (intmax_t)p->p_racct->r_resources[resource], p->p_comm, p->p_pid));
798
799 racct_adjust_resource(p->p_racct, resource, -amount);
800 racct_sub_cred_locked(p->p_ucred, resource, amount);
801 RACCT_UNLOCK();
802 }
803
804 static void
racct_sub_cred_locked(struct ucred * cred,int resource,uint64_t amount)805 racct_sub_cred_locked(struct ucred *cred, int resource, uint64_t amount)
806 {
807 struct prison *pr;
808
809 ASSERT_RACCT_ENABLED();
810
811 racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, -amount);
812 for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent)
813 racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource,
814 -amount);
815 racct_adjust_resource(cred->cr_loginclass->lc_racct, resource, -amount);
816 }
817
818 /*
819 * Decrease allocation of 'resource' by 'amount' for credential 'cred'.
820 */
821 void
racct_sub_cred(struct ucred * cred,int resource,uint64_t amount)822 racct_sub_cred(struct ucred *cred, int resource, uint64_t amount)
823 {
824
825 if (!racct_enable)
826 return;
827
828 SDT_PROBE3(racct, , rusage, sub__cred, cred, resource, amount);
829
830 #ifdef notyet
831 KASSERT(RACCT_CAN_DROP(resource),
832 ("%s: called for resource %d which can not drop", __func__,
833 resource));
834 #endif
835
836 RACCT_LOCK();
837 racct_sub_cred_locked(cred, resource, amount);
838 RACCT_UNLOCK();
839 }
840
841 /*
842 * Inherit resource usage information from the parent process.
843 */
844 int
racct_proc_fork(struct proc * parent,struct proc * child)845 racct_proc_fork(struct proc *parent, struct proc *child)
846 {
847 int i, error = 0;
848
849 if (!racct_enable)
850 return (0);
851
852 /*
853 * Create racct for the child process.
854 */
855 racct_create(&child->p_racct);
856
857 PROC_LOCK(parent);
858 PROC_LOCK(child);
859 RACCT_LOCK();
860
861 #ifdef RCTL
862 error = rctl_proc_fork(parent, child);
863 if (error != 0)
864 goto out;
865 #endif
866
867 child->p_throttled = 0;
868
869 /*
870 * Inherit resource usage.
871 */
872 for (i = 0; i <= RACCT_MAX; i++) {
873 if (parent->p_racct->r_resources[i] == 0 ||
874 !RACCT_IS_INHERITABLE(i))
875 continue;
876
877 error = racct_set_locked(child, i,
878 parent->p_racct->r_resources[i], 0);
879 if (error != 0)
880 goto out;
881 }
882
883 error = racct_add_locked(child, RACCT_NPROC, 1, 0);
884 error += racct_add_locked(child, RACCT_NTHR, 1, 0);
885
886 out:
887 RACCT_UNLOCK();
888 PROC_UNLOCK(child);
889 PROC_UNLOCK(parent);
890
891 if (error != 0)
892 racct_proc_exit(child);
893
894 return (error);
895 }
896
897 /*
898 * Called at the end of fork1(), to handle rules that require the process
899 * to be fully initialized.
900 */
901 void
racct_proc_fork_done(struct proc * child)902 racct_proc_fork_done(struct proc *child)
903 {
904
905 if (!racct_enable)
906 return;
907
908 #ifdef RCTL
909 PROC_LOCK(child);
910 RACCT_LOCK();
911 rctl_enforce(child, RACCT_NPROC, 0);
912 rctl_enforce(child, RACCT_NTHR, 0);
913 RACCT_UNLOCK();
914 PROC_UNLOCK(child);
915 #endif
916 }
917
918 void
racct_proc_exit(struct proc * p)919 racct_proc_exit(struct proc *p)
920 {
921 int i;
922
923 if (!racct_enable)
924 return;
925
926 PROC_LOCK(p);
927 RACCT_LOCK();
928
929 racct_settime_locked(p, true);
930 racct_zeropcpu_locked(p);
931
932 KASSERT(p->p_racct->r_resources[RACCT_RSS] == 0,
933 ("process reaped with %ju allocated for RSS\n",
934 p->p_racct->r_resources[RACCT_RSS]));
935 for (i = 0; i <= RACCT_MAX; i++) {
936 if (p->p_racct->r_resources[i] == 0)
937 continue;
938 if (!RACCT_IS_RECLAIMABLE(i))
939 continue;
940 racct_set_locked(p, i, 0, 0);
941 }
942
943 #ifdef RCTL
944 rctl_racct_release(p->p_racct);
945 #endif
946 racct_destroy_locked(&p->p_racct);
947 RACCT_UNLOCK();
948 PROC_UNLOCK(p);
949 }
950
951 /*
952 * Called after credentials change, to move resource utilisation
953 * between raccts.
954 */
955 void
racct_proc_ucred_changed(struct proc * p,struct ucred * oldcred,struct ucred * newcred)956 racct_proc_ucred_changed(struct proc *p, struct ucred *oldcred,
957 struct ucred *newcred)
958 {
959 struct uidinfo *olduip, *newuip;
960 struct loginclass *oldlc, *newlc;
961 struct prison *oldpr, *newpr, *pr;
962
963 if (!racct_enable)
964 return;
965
966 PROC_LOCK_ASSERT(p, MA_OWNED);
967
968 newuip = newcred->cr_ruidinfo;
969 olduip = oldcred->cr_ruidinfo;
970 newlc = newcred->cr_loginclass;
971 oldlc = oldcred->cr_loginclass;
972 newpr = newcred->cr_prison;
973 oldpr = oldcred->cr_prison;
974
975 RACCT_LOCK();
976 if (newuip != olduip) {
977 racct_sub_racct(olduip->ui_racct, p->p_racct);
978 racct_add_racct(newuip->ui_racct, p->p_racct);
979 }
980 if (newlc != oldlc) {
981 racct_sub_racct(oldlc->lc_racct, p->p_racct);
982 racct_add_racct(newlc->lc_racct, p->p_racct);
983 }
984 if (newpr != oldpr) {
985 for (pr = oldpr; pr != NULL; pr = pr->pr_parent)
986 racct_sub_racct(pr->pr_prison_racct->prr_racct,
987 p->p_racct);
988 for (pr = newpr; pr != NULL; pr = pr->pr_parent)
989 racct_add_racct(pr->pr_prison_racct->prr_racct,
990 p->p_racct);
991 }
992 RACCT_UNLOCK();
993 }
994
995 void
racct_move(struct racct * dest,struct racct * src)996 racct_move(struct racct *dest, struct racct *src)
997 {
998
999 ASSERT_RACCT_ENABLED();
1000
1001 RACCT_LOCK();
1002 racct_add_racct(dest, src);
1003 racct_sub_racct(src, src);
1004 dest->r_runtime = src->r_runtime;
1005 dest->r_time = src->r_time;
1006 src->r_runtime = 0;
1007 timevalsub(&src->r_time, &src->r_time);
1008 RACCT_UNLOCK();
1009 }
1010
1011 static void
ast_racct(struct thread * td,int tda __unused)1012 ast_racct(struct thread *td, int tda __unused)
1013 {
1014 struct proc *p;
1015
1016 ASSERT_RACCT_ENABLED();
1017
1018 p = td->td_proc;
1019 if (p->p_throttled == 0)
1020 return;
1021
1022 PROC_LOCK(p);
1023 while (p->p_throttled != 0) {
1024 msleep(p->p_racct, &p->p_mtx, 0, "racct",
1025 p->p_throttled < 0 ? 0 : p->p_throttled);
1026 if (p->p_throttled > 0)
1027 p->p_throttled = 0;
1028 }
1029 PROC_UNLOCK(p);
1030 }
1031
1032 /*
1033 * Make the process sleep in userret() for 'timeout' ticks. Setting
1034 * timeout to -1 makes it sleep until woken up by racct_proc_wakeup().
1035 */
1036 void
racct_proc_throttle(struct proc * p,int timeout)1037 racct_proc_throttle(struct proc *p, int timeout)
1038 {
1039 struct thread *td;
1040 #ifdef SMP
1041 int cpuid;
1042 #endif
1043
1044 KASSERT(timeout != 0, ("timeout %d", timeout));
1045 ASSERT_RACCT_ENABLED();
1046 PROC_LOCK_ASSERT(p, MA_OWNED);
1047
1048 /*
1049 * Do not block kernel processes. Also do not block processes with
1050 * low %cpu utilization to improve interactivity.
1051 */
1052 if ((p->p_flag & (P_SYSTEM | P_KPROC)) != 0)
1053 return;
1054
1055 if (p->p_throttled < 0 || (timeout > 0 && p->p_throttled > timeout))
1056 return;
1057
1058 p->p_throttled = timeout;
1059
1060 FOREACH_THREAD_IN_PROC(p, td) {
1061 thread_lock(td);
1062 ast_sched_locked(td, TDA_RACCT);
1063
1064 switch (TD_GET_STATE(td)) {
1065 case TDS_RUNQ:
1066 /*
1067 * If the thread is on the scheduler run-queue, we can
1068 * not just remove it from there. So we set the flag
1069 * TDA_SCHED for the thread, so that once it is
1070 * running, it is taken off the cpu as soon as possible.
1071 */
1072 ast_sched_locked(td, TDA_SCHED);
1073 break;
1074 case TDS_RUNNING:
1075 /*
1076 * If the thread is running, we request a context
1077 * switch for it by setting the TDA_SCHED flag.
1078 */
1079 ast_sched_locked(td, TDA_SCHED);
1080 #ifdef SMP
1081 cpuid = td->td_oncpu;
1082 if ((cpuid != NOCPU) && (td != curthread))
1083 ipi_cpu(cpuid, IPI_AST);
1084 #endif
1085 break;
1086 default:
1087 break;
1088 }
1089 thread_unlock(td);
1090 }
1091 }
1092
1093 static void
racct_proc_wakeup(struct proc * p)1094 racct_proc_wakeup(struct proc *p)
1095 {
1096
1097 ASSERT_RACCT_ENABLED();
1098
1099 PROC_LOCK_ASSERT(p, MA_OWNED);
1100
1101 if (p->p_throttled != 0) {
1102 p->p_throttled = 0;
1103 wakeup(p->p_racct);
1104 }
1105 }
1106
1107 static void
racct_decay_callback(struct racct * racct,void * dummy1,void * dummy2)1108 racct_decay_callback(struct racct *racct, void *dummy1, void *dummy2)
1109 {
1110 ASSERT_RACCT_ENABLED();
1111 RACCT_LOCK_ASSERT();
1112
1113 #ifdef RCTL
1114 rctl_throttle_decay(racct, RACCT_READBPS);
1115 rctl_throttle_decay(racct, RACCT_WRITEBPS);
1116 rctl_throttle_decay(racct, RACCT_READIOPS);
1117 rctl_throttle_decay(racct, RACCT_WRITEIOPS);
1118 #endif
1119 }
1120
1121 static void
racct_decay_pre(void)1122 racct_decay_pre(void)
1123 {
1124
1125 RACCT_LOCK();
1126 }
1127
1128 static void
racct_decay_post(void)1129 racct_decay_post(void)
1130 {
1131
1132 RACCT_UNLOCK();
1133 }
1134
1135 static void
racct_decay(void)1136 racct_decay(void)
1137 {
1138
1139 ASSERT_RACCT_ENABLED();
1140
1141 ui_racct_foreach(racct_decay_callback, racct_decay_pre,
1142 racct_decay_post, NULL, NULL);
1143 loginclass_racct_foreach(racct_decay_callback, racct_decay_pre,
1144 racct_decay_post, NULL, NULL);
1145 prison_racct_foreach(racct_decay_callback, racct_decay_pre,
1146 racct_decay_post, NULL, NULL);
1147 }
1148
1149 static void
racct_updatepcpu_racct_locked(struct racct * racct)1150 racct_updatepcpu_racct_locked(struct racct *racct)
1151 {
1152 struct timeval diff;
1153 uint64_t elapsed;
1154 uint64_t runtime;
1155 uint64_t newpcpu;
1156 uint64_t oldpcpu;
1157
1158 ASSERT_RACCT_ENABLED();
1159 RACCT_LOCK_ASSERT();
1160
1161 /* Difference between now and previously-recorded time. */
1162 microuptime(&diff);
1163 timevalsub(&diff, &racct->r_time);
1164 elapsed = (uint64_t)diff.tv_sec * 1000000 + diff.tv_usec;
1165
1166 /* Difference between current and previously-recorded runtime. */
1167 runtime = racct->r_resources[RACCT_CPU] - racct->r_runtime;
1168
1169 newpcpu = runtime * 100 * 1000000 / elapsed;
1170 oldpcpu = racct->r_resources[RACCT_PCTCPU];
1171 /*
1172 * This calculation is equivalent to
1173 * (1 - 0.3) * newpcpu + 0.3 * oldpcpu
1174 * where RACCT_DECAY_FACTOR = 0.3 * FSCALE.
1175 */
1176 racct->r_resources[RACCT_PCTCPU] = ((FSCALE - RACCT_DECAY_FACTOR) *
1177 newpcpu + RACCT_DECAY_FACTOR * oldpcpu) / FSCALE;
1178 if (racct->r_resources[RACCT_PCTCPU] >
1179 100 * 1000000 * (uint64_t)mp_ncpus)
1180 racct->r_resources[RACCT_PCTCPU] = 100 * 1000000 *
1181 (uint64_t)mp_ncpus;
1182
1183 /* Record current times. */
1184 racct->r_runtime = racct->r_resources[RACCT_CPU];
1185 timevaladd(&racct->r_time, &diff);
1186 }
1187
1188 static void
racct_zeropcpu_locked(struct proc * p)1189 racct_zeropcpu_locked(struct proc *p)
1190 {
1191 ASSERT_RACCT_ENABLED();
1192 PROC_LOCK_ASSERT(p, MA_OWNED);
1193
1194 p->p_racct->r_resources[RACCT_PCTCPU] = 0;
1195 }
1196
1197 static void
racct_updatepcpu_locked(struct proc * p)1198 racct_updatepcpu_locked(struct proc *p)
1199 {
1200 ASSERT_RACCT_ENABLED();
1201 PROC_LOCK_ASSERT(p, MA_OWNED);
1202
1203 racct_updatepcpu_racct_locked(p->p_racct);
1204 }
1205
1206 static void
racct_updatepcpu_pre(void)1207 racct_updatepcpu_pre(void)
1208 {
1209
1210 RACCT_LOCK();
1211 }
1212
1213 static void
racct_updatepcpu_post(void)1214 racct_updatepcpu_post(void)
1215 {
1216
1217 RACCT_UNLOCK();
1218 }
1219
1220 static void
racct_updatepcpu_racct_callback(struct racct * racct,void * dummy1,void * dummy2)1221 racct_updatepcpu_racct_callback(struct racct *racct, void *dummy1, void *dummy2)
1222 {
1223 racct_updatepcpu_racct_locked(racct);
1224 }
1225
1226 static void
racct_updatepcpu_containers(void)1227 racct_updatepcpu_containers(void)
1228 {
1229 ASSERT_RACCT_ENABLED();
1230
1231 ui_racct_foreach(racct_updatepcpu_racct_callback, racct_updatepcpu_pre,
1232 racct_updatepcpu_post, NULL, NULL);
1233 loginclass_racct_foreach(racct_updatepcpu_racct_callback, racct_updatepcpu_pre,
1234 racct_updatepcpu_post, NULL, NULL);
1235 prison_racct_foreach(racct_updatepcpu_racct_callback, racct_updatepcpu_pre,
1236 racct_updatepcpu_post, NULL, NULL);
1237 }
1238
1239 static void
racctd(void)1240 racctd(void)
1241 {
1242 struct proc *p;
1243 struct proc *idle;
1244
1245 ASSERT_RACCT_ENABLED();
1246
1247 idle = STAILQ_FIRST(&cpuhead)->pc_idlethread->td_proc;
1248
1249 for (;;) {
1250 racct_decay();
1251
1252 sx_slock(&allproc_lock);
1253
1254 FOREACH_PROC_IN_SYSTEM(p) {
1255 PROC_LOCK(p);
1256 if (p == idle) {
1257 PROC_UNLOCK(p);
1258 continue;
1259 }
1260 if (p->p_state != PRS_NORMAL ||
1261 (p->p_flag & P_IDLEPROC) != 0) {
1262 PROC_UNLOCK(p);
1263 continue;
1264 }
1265
1266 RACCT_LOCK();
1267 #ifdef RCTL
1268 rctl_throttle_decay(p->p_racct, RACCT_READBPS);
1269 rctl_throttle_decay(p->p_racct, RACCT_WRITEBPS);
1270 rctl_throttle_decay(p->p_racct, RACCT_READIOPS);
1271 rctl_throttle_decay(p->p_racct, RACCT_WRITEIOPS);
1272 #endif
1273 racct_settime_locked(p, false);
1274 racct_updatepcpu_locked(p);
1275 RACCT_UNLOCK();
1276 PROC_UNLOCK(p);
1277 }
1278
1279 /*
1280 * To ensure that processes are throttled in a fair way, we need
1281 * to iterate over all processes again and check the limits
1282 * for %cpu resource only after ucred racct containers have been
1283 * properly filled.
1284 */
1285 FOREACH_PROC_IN_SYSTEM(p) {
1286 PROC_LOCK(p);
1287 if (p->p_state != PRS_NORMAL) {
1288 PROC_UNLOCK(p);
1289 continue;
1290 }
1291
1292 if (racct_pcpu_available(p) <= 0) {
1293 if (p->p_racct->r_resources[RACCT_PCTCPU] >
1294 pcpu_threshold)
1295 racct_proc_throttle(p, -1);
1296 } else if (p->p_throttled == -1) {
1297 racct_proc_wakeup(p);
1298 }
1299 PROC_UNLOCK(p);
1300 }
1301 sx_sunlock(&allproc_lock);
1302
1303 racct_updatepcpu_containers();
1304 pause("-", hz);
1305 }
1306 }
1307
1308 static struct kproc_desc racctd_kp = {
1309 "racctd",
1310 racctd,
1311 NULL
1312 };
1313
1314 static void
racctd_init(void)1315 racctd_init(void)
1316 {
1317 if (!racct_enable)
1318 return;
1319
1320 kproc_start(&racctd_kp);
1321 }
1322 SYSINIT(racctd, SI_SUB_RACCTD, SI_ORDER_FIRST, racctd_init, NULL);
1323
1324 static void
racct_init(void)1325 racct_init(void)
1326 {
1327 if (!racct_enable)
1328 return;
1329
1330 racct_zone = uma_zcreate("racct", sizeof(struct racct),
1331 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1332 ast_register(TDA_RACCT, ASTR_ASTF_REQUIRED, 0, ast_racct);
1333
1334 /*
1335 * XXX: Move this somewhere.
1336 */
1337 prison0.pr_prison_racct = prison_racct_find("0");
1338 }
1339 SYSINIT(racct, SI_SUB_RACCT, SI_ORDER_FIRST, racct_init, NULL);
1340