1 2 #ifdef CONFIG_SCHEDSTATS 3 4 /* 5 * Expects runqueue lock to be held for atomicity of update 6 */ 7 static inline void 8 rq_sched_info_arrive(struct rq *rq, unsigned long long delta) 9 { 10 if (rq) { 11 rq->rq_sched_info.run_delay += delta; 12 rq->rq_sched_info.pcount++; 13 } 14 } 15 16 /* 17 * Expects runqueue lock to be held for atomicity of update 18 */ 19 static inline void 20 rq_sched_info_depart(struct rq *rq, unsigned long long delta) 21 { 22 if (rq) 23 rq->rq_cpu_time += delta; 24 } 25 26 static inline void 27 rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) 28 { 29 if (rq) 30 rq->rq_sched_info.run_delay += delta; 31 } 32 # define schedstat_inc(rq, field) do { (rq)->field++; } while (0) 33 # define schedstat_add(rq, field, amt) do { (rq)->field += (amt); } while (0) 34 # define schedstat_set(var, val) do { var = (val); } while (0) 35 #else /* !CONFIG_SCHEDSTATS */ 36 static inline void 37 rq_sched_info_arrive(struct rq *rq, unsigned long long delta) 38 {} 39 static inline void 40 rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) 41 {} 42 static inline void 43 rq_sched_info_depart(struct rq *rq, unsigned long long delta) 44 {} 45 # define schedstat_inc(rq, field) do { } while (0) 46 # define schedstat_add(rq, field, amt) do { } while (0) 47 # define schedstat_set(var, val) do { } while (0) 48 #endif 49 50 #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) 51 static inline void sched_info_reset_dequeued(struct task_struct *t) 52 { 53 t->sched_info.last_queued = 0; 54 } 55 56 /* 57 * We are interested in knowing how long it was from the *first* time a 58 * task was queued to the time that it finally hit a cpu, we call this routine 59 * from dequeue_task() to account for possible rq->clock skew across cpus. The 60 * delta taken on each cpu would annul the skew. 61 */ 62 static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t) 63 { 64 unsigned long long now = rq_clock(rq), delta = 0; 65 66 if (unlikely(sched_info_on())) 67 if (t->sched_info.last_queued) 68 delta = now - t->sched_info.last_queued; 69 sched_info_reset_dequeued(t); 70 t->sched_info.run_delay += delta; 71 72 rq_sched_info_dequeued(rq, delta); 73 } 74 75 /* 76 * Called when a task finally hits the cpu. We can now calculate how 77 * long it was waiting to run. We also note when it began so that we 78 * can keep stats on how long its timeslice is. 79 */ 80 static void sched_info_arrive(struct rq *rq, struct task_struct *t) 81 { 82 unsigned long long now = rq_clock(rq), delta = 0; 83 84 if (t->sched_info.last_queued) 85 delta = now - t->sched_info.last_queued; 86 sched_info_reset_dequeued(t); 87 t->sched_info.run_delay += delta; 88 t->sched_info.last_arrival = now; 89 t->sched_info.pcount++; 90 91 rq_sched_info_arrive(rq, delta); 92 } 93 94 /* 95 * This function is only called from enqueue_task(), but also only updates 96 * the timestamp if it is already not set. It's assumed that 97 * sched_info_dequeued() will clear that stamp when appropriate. 98 */ 99 static inline void sched_info_queued(struct rq *rq, struct task_struct *t) 100 { 101 if (unlikely(sched_info_on())) 102 if (!t->sched_info.last_queued) 103 t->sched_info.last_queued = rq_clock(rq); 104 } 105 106 /* 107 * Called when a process ceases being the active-running process involuntarily 108 * due, typically, to expiring its time slice (this may also be called when 109 * switching to the idle task). Now we can calculate how long we ran. 110 * Also, if the process is still in the TASK_RUNNING state, call 111 * sched_info_queued() to mark that it has now again started waiting on 112 * the runqueue. 113 */ 114 static inline void sched_info_depart(struct rq *rq, struct task_struct *t) 115 { 116 unsigned long long delta = rq_clock(rq) - 117 t->sched_info.last_arrival; 118 119 rq_sched_info_depart(rq, delta); 120 121 if (t->state == TASK_RUNNING) 122 sched_info_queued(rq, t); 123 } 124 125 /* 126 * Called when tasks are switched involuntarily due, typically, to expiring 127 * their time slice. (This may also be called when switching to or from 128 * the idle task.) We are only called when prev != next. 129 */ 130 static inline void 131 __sched_info_switch(struct rq *rq, 132 struct task_struct *prev, struct task_struct *next) 133 { 134 /* 135 * prev now departs the cpu. It's not interesting to record 136 * stats about how efficient we were at scheduling the idle 137 * process, however. 138 */ 139 if (prev != rq->idle) 140 sched_info_depart(rq, prev); 141 142 if (next != rq->idle) 143 sched_info_arrive(rq, next); 144 } 145 static inline void 146 sched_info_switch(struct rq *rq, 147 struct task_struct *prev, struct task_struct *next) 148 { 149 if (unlikely(sched_info_on())) 150 __sched_info_switch(rq, prev, next); 151 } 152 #else 153 #define sched_info_queued(rq, t) do { } while (0) 154 #define sched_info_reset_dequeued(t) do { } while (0) 155 #define sched_info_dequeued(rq, t) do { } while (0) 156 #define sched_info_depart(rq, t) do { } while (0) 157 #define sched_info_arrive(rq, next) do { } while (0) 158 #define sched_info_switch(rq, t, next) do { } while (0) 159 #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */ 160 161 /* 162 * The following are functions that support scheduler-internal time accounting. 163 * These functions are generally called at the timer tick. None of this depends 164 * on CONFIG_SCHEDSTATS. 165 */ 166 167 /** 168 * cputimer_running - return true if cputimer is running 169 * 170 * @tsk: Pointer to target task. 171 */ 172 static inline bool cputimer_running(struct task_struct *tsk) 173 174 { 175 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 176 177 if (!cputimer->running) 178 return false; 179 180 /* 181 * After we flush the task's sum_exec_runtime to sig->sum_sched_runtime 182 * in __exit_signal(), we won't account to the signal struct further 183 * cputime consumed by that task, even though the task can still be 184 * ticking after __exit_signal(). 185 * 186 * In order to keep a consistent behaviour between thread group cputime 187 * and thread group cputimer accounting, lets also ignore the cputime 188 * elapsing after __exit_signal() in any thread group timer running. 189 * 190 * This makes sure that POSIX CPU clocks and timers are synchronized, so 191 * that a POSIX CPU timer won't expire while the corresponding POSIX CPU 192 * clock delta is behind the expiring timer value. 193 */ 194 if (unlikely(!tsk->sighand)) 195 return false; 196 197 return true; 198 } 199 200 /** 201 * account_group_user_time - Maintain utime for a thread group. 202 * 203 * @tsk: Pointer to task structure. 204 * @cputime: Time value by which to increment the utime field of the 205 * thread_group_cputime structure. 206 * 207 * If thread group time is being maintained, get the structure for the 208 * running CPU and update the utime field there. 209 */ 210 static inline void account_group_user_time(struct task_struct *tsk, 211 cputime_t cputime) 212 { 213 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 214 215 if (!cputimer_running(tsk)) 216 return; 217 218 raw_spin_lock(&cputimer->lock); 219 cputimer->cputime.utime += cputime; 220 raw_spin_unlock(&cputimer->lock); 221 } 222 223 /** 224 * account_group_system_time - Maintain stime for a thread group. 225 * 226 * @tsk: Pointer to task structure. 227 * @cputime: Time value by which to increment the stime field of the 228 * thread_group_cputime structure. 229 * 230 * If thread group time is being maintained, get the structure for the 231 * running CPU and update the stime field there. 232 */ 233 static inline void account_group_system_time(struct task_struct *tsk, 234 cputime_t cputime) 235 { 236 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 237 238 if (!cputimer_running(tsk)) 239 return; 240 241 raw_spin_lock(&cputimer->lock); 242 cputimer->cputime.stime += cputime; 243 raw_spin_unlock(&cputimer->lock); 244 } 245 246 /** 247 * account_group_exec_runtime - Maintain exec runtime for a thread group. 248 * 249 * @tsk: Pointer to task structure. 250 * @ns: Time value by which to increment the sum_exec_runtime field 251 * of the thread_group_cputime structure. 252 * 253 * If thread group time is being maintained, get the structure for the 254 * running CPU and update the sum_exec_runtime field there. 255 */ 256 static inline void account_group_exec_runtime(struct task_struct *tsk, 257 unsigned long long ns) 258 { 259 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 260 261 if (!cputimer_running(tsk)) 262 return; 263 264 raw_spin_lock(&cputimer->lock); 265 cputimer->cputime.sum_exec_runtime += ns; 266 raw_spin_unlock(&cputimer->lock); 267 } 268