1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11 /*
12 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
13 */
14
15 #include <sys/cmn_err.h>
16 #include <sys/ddi_periodic.h>
17 #include <sys/id_space.h>
18 #include <sys/kobj.h>
19 #include <sys/sysmacros.h>
20 #include <sys/systm.h>
21 #include <sys/taskq.h>
22 #include <sys/taskq_impl.h>
23 #include <sys/time.h>
24 #include <sys/types.h>
25 #include <sys/sdt.h>
26
27 extern void sir_on(int);
28
29 /*
30 * The ddi_periodic_add(9F) Implementation
31 *
32 * This file contains the implementation of the ddi_periodic_add(9F) interface.
33 * It is a thin wrapper around the cyclic subsystem (see documentation in
34 * uts/common/os/cyclic.c), providing a DDI interface for registering
35 * (and unregistering) callbacks for periodic invocation at arbitrary
36 * interrupt levels, or in kernel context.
37 *
38 * Each call to ddi_periodic_add will result in a new opaque handle, as
39 * allocated from an id_space, a new "periodic" object (ddi_periodic_impl_t)
40 * and a registered cyclic.
41 *
42 * Operation
43 *
44 * Whenever the cyclic fires, our cyclic handler checks that the particular
45 * periodic is not dispatched already (we do not support overlapping execution
46 * of the consumer's handler function), and not yet cancelled. If both of
47 * these conditions hold, we mark the periodic as DPF_DISPATCHED and enqueue it
48 * to either the taskq (for DDI_IPL_0) or to one of the soft interrupt queues
49 * (DDI_IPL_1 to DDI_IPL_10).
50 *
51 * While the taskq (or soft interrupt handler) is handling a particular
52 * periodic, we mark it as DPF_EXECUTING. When complete, we reset both
53 * DPF_DISPATCHED and DPF_EXECUTING.
54 *
55 * Cancellation
56 *
57 * ddi_periodic_delete(9F) historically had spectacularly loose semantics with
58 * respect to cancellation concurrent with handler execution. These semantics
59 * are now tighter:
60 *
61 * 1. At most one invocation of ddi_periodic_delete(9F) will actually
62 * perform the deletion, all others will return immediately.
63 * 2. The invocation that performs the deletion will _block_ until
64 * the handler is no longer running, and all resources have been
65 * released.
66 *
67 * We affect this model by removing the cancelling periodic from the
68 * global list and marking it DPF_CANCELLED. This will prevent further
69 * execution of the handler. We then wait on a CV until the DPF_EXECUTING
70 * and DPF_DISPATCHED flags are clear, which means the periodic is removed
71 * from all request queues, is no longer executing, and may be freed. At this
72 * point we return the opaque ID to the id_space and free the memory.
73 *
74 * NOTE:
75 * The ddi_periodic_add(9F) interface is presently limited to a minimum period
76 * of 10ms between firings.
77 */
78
79 /*
80 * Tuneables:
81 */
82 int ddi_periodic_max_id = 1024;
83 int ddi_periodic_taskq_threadcount = 4;
84 hrtime_t ddi_periodic_resolution = 10000000;
85
86 /*
87 * Globals:
88 */
89 static kmem_cache_t *periodic_cache;
90 static id_space_t *periodic_id_space;
91 static taskq_t *periodic_taskq;
92
93 /*
94 * periodics_lock protects the list of all periodics (periodics), and
95 * each of the soft interrupt request queues (periodic_softint_queue).
96 *
97 * Do not hold an individual periodic's lock while obtaining periodics_lock.
98 * While in the periodic_softint_queue list, the periodic will be marked
99 * DPF_DISPATCHED, and thus safe from frees. Only the invocation of
100 * i_untimeout() that removes the periodic from the global list is allowed
101 * to free it.
102 */
103 static kmutex_t periodics_lock;
104 static list_t periodics;
105 static list_t periodic_softint_queue[10]; /* for IPL1 up to IPL10 */
106
107 typedef enum periodic_ipl {
108 PERI_IPL_0 = 0,
109 PERI_IPL_1,
110 PERI_IPL_2,
111 PERI_IPL_3,
112 PERI_IPL_4,
113 PERI_IPL_5,
114 PERI_IPL_6,
115 PERI_IPL_7,
116 PERI_IPL_8,
117 PERI_IPL_9,
118 PERI_IPL_10
119 } periodic_ipl_t;
120
121 static char *
periodic_handler_symbol(ddi_periodic_impl_t * dpr)122 periodic_handler_symbol(ddi_periodic_impl_t *dpr)
123 {
124 ulong_t off;
125
126 return (kobj_getsymname((uintptr_t)dpr->dpr_handler, &off));
127 }
128
129 /*
130 * This function may be called either from a soft interrupt handler
131 * (ddi_periodic_softintr), or as a taskq worker function.
132 */
133 static void
periodic_execute(void * arg)134 periodic_execute(void *arg)
135 {
136 ddi_periodic_impl_t *dpr = arg;
137 mutex_enter(&dpr->dpr_lock);
138
139 /*
140 * We must be DISPATCHED, but not yet EXECUTING:
141 */
142 VERIFY((dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) ==
143 DPF_DISPATCHED);
144 VERIFY(dpr->dpr_thread == NULL);
145
146 if (!(dpr->dpr_flags & DPF_CANCELLED)) {
147 int level = dpr->dpr_level;
148 uint64_t count = dpr->dpr_fire_count;
149 /*
150 * If we have not yet been cancelled, then
151 * mark us executing:
152 */
153 dpr->dpr_flags |= DPF_EXECUTING;
154 dpr->dpr_thread = curthread;
155 mutex_exit(&dpr->dpr_lock);
156
157 /*
158 * Execute the handler, without holding locks:
159 */
160 DTRACE_PROBE4(ddi__periodic__execute, void *, dpr->dpr_handler,
161 void *, dpr->dpr_arg, int, level, uint64_t, count);
162 (*dpr->dpr_handler)(dpr->dpr_arg);
163 DTRACE_PROBE4(ddi__periodic__done, void *, dpr->dpr_handler,
164 void *, dpr->dpr_arg, int, level, uint64_t, count);
165
166 mutex_enter(&dpr->dpr_lock);
167 dpr->dpr_thread = NULL;
168 dpr->dpr_fire_count++;
169 }
170
171 /*
172 * We're done with this periodic for now, so release it and
173 * wake anybody that was waiting for us to be finished:
174 */
175 dpr->dpr_flags &= ~(DPF_DISPATCHED | DPF_EXECUTING);
176 cv_broadcast(&dpr->dpr_cv);
177 mutex_exit(&dpr->dpr_lock);
178 }
179
180 void
ddi_periodic_softintr(int level)181 ddi_periodic_softintr(int level)
182 {
183 ddi_periodic_impl_t *dpr;
184 VERIFY(level >= PERI_IPL_1 && level <= PERI_IPL_10);
185
186 mutex_enter(&periodics_lock);
187 /*
188 * Pull the first scheduled periodic off the queue for this priority
189 * level:
190 */
191 while ((dpr = list_remove_head(&periodic_softint_queue[level - 1])) !=
192 NULL) {
193 mutex_exit(&periodics_lock);
194 /*
195 * And execute it:
196 */
197 periodic_execute(dpr);
198 mutex_enter(&periodics_lock);
199 }
200 mutex_exit(&periodics_lock);
201 }
202
203 void
ddi_periodic_init(void)204 ddi_periodic_init(void)
205 {
206 int i;
207
208 /*
209 * Create a kmem_cache for request tracking objects, and a list
210 * to store them in so we can later delete based on opaque handles:
211 */
212 periodic_cache = kmem_cache_create("ddi_periodic",
213 sizeof (ddi_periodic_impl_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
214 list_create(&periodics, sizeof (ddi_periodic_impl_t),
215 offsetof(ddi_periodic_impl_t, dpr_link));
216
217 /*
218 * Initialise the identifier space for ddi_periodic_add(9F):
219 */
220 periodic_id_space = id_space_create("ddi_periodic", 1,
221 ddi_periodic_max_id);
222
223 /*
224 * Initialise the request queue for each soft interrupt level:
225 */
226 for (i = PERI_IPL_1; i <= PERI_IPL_10; i++) {
227 list_create(&periodic_softint_queue[i - 1],
228 sizeof (ddi_periodic_impl_t), offsetof(ddi_periodic_impl_t,
229 dpr_softint_link));
230 }
231
232 /*
233 * Create the taskq for running PERI_IPL_0 handlers. This taskq will
234 * _only_ be used with taskq_dispatch_ent(), and a taskq_ent_t
235 * pre-allocated with the ddi_periodic_impl_t.
236 */
237 periodic_taskq = taskq_create_instance("ddi_periodic_taskq", -1,
238 ddi_periodic_taskq_threadcount, maxclsyspri, 0, 0, 0);
239
240 /*
241 * Initialize the mutex lock used for the soft interrupt request
242 * queues.
243 */
244 mutex_init(&periodics_lock, NULL, MUTEX_ADAPTIVE, NULL);
245 }
246
247 void
ddi_periodic_fini(void)248 ddi_periodic_fini(void)
249 {
250 int i;
251 ddi_periodic_impl_t *dpr;
252
253 /*
254 * Find all periodics that have not yet been unregistered and,
255 * on DEBUG bits, print a warning about this resource leak.
256 */
257 mutex_enter(&periodics_lock);
258 while ((dpr = list_head(&periodics)) != NULL) {
259 #ifdef DEBUG
260 printf("DDI periodic handler not deleted (id=%lx, hdlr=%s)\n",
261 (unsigned long)dpr->dpr_id, periodic_handler_symbol(dpr));
262 #endif
263
264 mutex_exit(&periodics_lock);
265 /*
266 * Delete the periodic ourselves:
267 */
268 i_untimeout((timeout_t)(uintptr_t)dpr->dpr_id);
269 mutex_enter(&periodics_lock);
270 }
271 mutex_exit(&periodics_lock);
272
273 /*
274 * At this point there are no remaining cyclics, so clean up the
275 * remaining resources:
276 */
277 taskq_destroy(periodic_taskq);
278 periodic_taskq = NULL;
279
280 id_space_destroy(periodic_id_space);
281 periodic_id_space = NULL;
282
283 kmem_cache_destroy(periodic_cache);
284 periodic_cache = NULL;
285
286 list_destroy(&periodics);
287 for (i = PERI_IPL_1; i <= PERI_IPL_10; i++)
288 list_destroy(&periodic_softint_queue[i - 1]);
289
290 mutex_destroy(&periodics_lock);
291 }
292
293 static void
periodic_cyclic_handler(void * arg)294 periodic_cyclic_handler(void *arg)
295 {
296 ddi_periodic_impl_t *dpr = arg;
297
298 mutex_enter(&dpr->dpr_lock);
299 /*
300 * If we've been cancelled, or we're already dispatched, then exit
301 * immediately:
302 */
303 if (dpr->dpr_flags & (DPF_CANCELLED | DPF_DISPATCHED)) {
304 mutex_exit(&dpr->dpr_lock);
305 return;
306 }
307 VERIFY(!(dpr->dpr_flags & DPF_EXECUTING));
308
309 /*
310 * This periodic is not presently dispatched, so dispatch it now:
311 */
312 dpr->dpr_flags |= DPF_DISPATCHED;
313 mutex_exit(&dpr->dpr_lock);
314
315 if (dpr->dpr_level == PERI_IPL_0) {
316 /*
317 * DDI_IPL_0 periodics are dispatched onto the taskq:
318 */
319 taskq_dispatch_ent(periodic_taskq, periodic_execute,
320 dpr, 0, &dpr->dpr_taskq_ent);
321 } else {
322 /*
323 * Higher priority periodics are handled by a soft interrupt
324 * handler. Enqueue us for processing by the handler:
325 */
326 mutex_enter(&periodics_lock);
327 list_insert_tail(&periodic_softint_queue[dpr->dpr_level - 1],
328 dpr);
329 mutex_exit(&periodics_lock);
330
331 /*
332 * Request the execution of the soft interrupt handler for this
333 * periodic's priority level.
334 */
335 sir_on(dpr->dpr_level);
336 }
337 }
338
339 static void
periodic_destroy(ddi_periodic_impl_t * dpr)340 periodic_destroy(ddi_periodic_impl_t *dpr)
341 {
342 if (dpr == NULL)
343 return;
344
345 /*
346 * By now, we should have a periodic that is not busy, and has been
347 * cancelled:
348 */
349 VERIFY(dpr->dpr_flags == DPF_CANCELLED);
350 VERIFY(dpr->dpr_thread == NULL);
351
352 id_free(periodic_id_space, dpr->dpr_id);
353 cv_destroy(&dpr->dpr_cv);
354 mutex_destroy(&dpr->dpr_lock);
355 kmem_cache_free(periodic_cache, dpr);
356 }
357
358 static ddi_periodic_impl_t *
periodic_create(void)359 periodic_create(void)
360 {
361 ddi_periodic_impl_t *dpr;
362
363 dpr = kmem_cache_alloc(periodic_cache, KM_SLEEP);
364 bzero(dpr, sizeof (*dpr));
365 dpr->dpr_id = id_alloc(periodic_id_space);
366 mutex_init(&dpr->dpr_lock, NULL, MUTEX_ADAPTIVE, NULL);
367 cv_init(&dpr->dpr_cv, NULL, CV_DEFAULT, NULL);
368
369 return (dpr);
370 }
371
372 /*
373 * This function provides the implementation for the ddi_periodic_add(9F)
374 * interface. It registers a periodic handler and returns an opaque identifier
375 * that can be unregistered via ddi_periodic_delete(9F)/i_untimeout().
376 *
377 * It may be called in user or kernel context, provided cpu_lock is not held.
378 */
379 timeout_t
i_timeout(void (* func)(void *),void * arg,hrtime_t interval,int level)380 i_timeout(void (*func)(void *), void *arg, hrtime_t interval, int level)
381 {
382 cyc_handler_t cyh;
383 cyc_time_t cyt;
384 ddi_periodic_impl_t *dpr;
385
386 VERIFY(func != NULL);
387 VERIFY(level >= 0 && level <= 10);
388
389 /*
390 * Allocate object to track this periodic:
391 */
392 dpr = periodic_create();
393 dpr->dpr_level = level;
394 dpr->dpr_handler = func;
395 dpr->dpr_arg = arg;
396
397 /*
398 * The minimum supported interval between firings of the periodic
399 * handler is 10ms; see ddi_periodic_add(9F) for more details. If a
400 * shorter interval is requested, round up.
401 */
402 if (ddi_periodic_resolution > interval) {
403 cmn_err(CE_WARN,
404 "The periodic timeout (handler=%s, interval=%lld) "
405 "requests a finer interval than the supported resolution. "
406 "It rounds up to %lld\n", periodic_handler_symbol(dpr),
407 interval, ddi_periodic_resolution);
408 interval = ddi_periodic_resolution;
409 }
410
411 /*
412 * Ensure that the interval is an even multiple of the base resolution
413 * that is at least as long as the requested interval.
414 */
415 dpr->dpr_interval = roundup(interval, ddi_periodic_resolution);
416
417 /*
418 * Create the underlying cyclic:
419 */
420 cyh.cyh_func = periodic_cyclic_handler;
421 cyh.cyh_arg = dpr;
422 cyh.cyh_level = CY_LOCK_LEVEL;
423
424 cyt.cyt_when = 0;
425 cyt.cyt_interval = dpr->dpr_interval;
426
427 mutex_enter(&cpu_lock);
428 dpr->dpr_cyclic_id = cyclic_add(&cyh, &cyt);
429 mutex_exit(&cpu_lock);
430
431 /*
432 * Make the id visible to ddi_periodic_delete(9F) before we
433 * return it:
434 */
435 mutex_enter(&periodics_lock);
436 list_insert_tail(&periodics, dpr);
437 mutex_exit(&periodics_lock);
438
439 return ((timeout_t)(uintptr_t)dpr->dpr_id);
440 }
441
442 /*
443 * This function provides the implementation for the ddi_periodic_delete(9F)
444 * interface. It cancels a periodic handler previously registered through
445 * ddi_periodic_add(9F)/i_timeout().
446 *
447 * It may be called in user or kernel context, provided cpu_lock is not held.
448 * It may NOT be called from within a periodic handler.
449 */
450 void
i_untimeout(timeout_t id)451 i_untimeout(timeout_t id)
452 {
453 ddi_periodic_impl_t *dpr;
454
455 /*
456 * Find the periodic in the list of all periodics and remove it.
457 * If we find in (and remove it from) the global list, we have
458 * license to free it once it is no longer busy.
459 */
460 mutex_enter(&periodics_lock);
461 for (dpr = list_head(&periodics); dpr != NULL; dpr =
462 list_next(&periodics, dpr)) {
463 if (dpr->dpr_id == (id_t)(uintptr_t)id) {
464 list_remove(&periodics, dpr);
465 break;
466 }
467 }
468 mutex_exit(&periodics_lock);
469
470 /*
471 * We could not find a periodic for this id, so bail out:
472 */
473 if (dpr == NULL)
474 return;
475
476 mutex_enter(&dpr->dpr_lock);
477 /*
478 * We should be the only one trying to cancel this periodic:
479 */
480 VERIFY(!(dpr->dpr_flags & DPF_CANCELLED));
481 /*
482 * Removing a periodic from within its own handler function will
483 * cause a deadlock, so panic explicitly.
484 */
485 if (dpr->dpr_thread == curthread) {
486 panic("ddi_periodic_delete(%lx) called from its own handler\n",
487 (unsigned long)dpr->dpr_id);
488 }
489 /*
490 * Mark the periodic as cancelled:
491 */
492 dpr->dpr_flags |= DPF_CANCELLED;
493 mutex_exit(&dpr->dpr_lock);
494
495 /*
496 * Cancel our cyclic. cyclic_remove() guarantees that the cyclic
497 * handler will not run again after it returns. Note that the cyclic
498 * handler merely _dispatches_ the periodic, so this does _not_ mean
499 * the periodic handler is also finished running.
500 */
501 mutex_enter(&cpu_lock);
502 cyclic_remove(dpr->dpr_cyclic_id);
503 mutex_exit(&cpu_lock);
504
505 /*
506 * Wait until the periodic handler is no longer running:
507 */
508 mutex_enter(&dpr->dpr_lock);
509 while (dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) {
510 cv_wait(&dpr->dpr_cv, &dpr->dpr_lock);
511 }
512 mutex_exit(&dpr->dpr_lock);
513
514 periodic_destroy(dpr);
515 }
516