xref: /illumos-gate/usr/src/uts/common/os/ddi_periodic.c (revision a288e5a9793fdffe5e842d7e61ab45263e75eaca)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 /*
12  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
13  */
14 
15 #include <sys/cmn_err.h>
16 #include <sys/ddi_periodic.h>
17 #include <sys/id_space.h>
18 #include <sys/kobj.h>
19 #include <sys/sysmacros.h>
20 #include <sys/systm.h>
21 #include <sys/taskq.h>
22 #include <sys/taskq_impl.h>
23 #include <sys/time.h>
24 #include <sys/types.h>
25 #include <sys/sdt.h>
26 
27 extern void sir_on(int);
28 
29 /*
30  * The ddi_periodic_add(9F) Implementation
31  *
32  * This file contains the implementation of the ddi_periodic_add(9F) interface.
33  * It is a thin wrapper around the cyclic subsystem (see documentation in
34  * uts/common/os/cyclic.c), providing a DDI interface for registering
35  * (and unregistering) callbacks for periodic invocation at arbitrary
36  * interrupt levels, or in kernel context.
37  *
38  * Each call to ddi_periodic_add will result in a new opaque handle, as
39  * allocated from an id_space, a new "periodic" object (ddi_periodic_impl_t)
40  * and a registered cyclic.
41  *
42  * Operation
43  *
44  * Whenever the cyclic fires, our cyclic handler checks that the particular
45  * periodic is not dispatched already (we do not support overlapping execution
46  * of the consumer's handler function), and not yet cancelled.  If both of
47  * these conditions hold, we mark the periodic as DPF_DISPATCHED and enqueue it
48  * to either the taskq (for DDI_IPL_0) or to one of the soft interrupt queues
49  * (DDI_IPL_1 to DDI_IPL_10).
50  *
51  * While the taskq (or soft interrupt handler) is handling a particular
52  * periodic, we mark it as DPF_EXECUTING.  When complete, we reset both
53  * DPF_DISPATCHED and DPF_EXECUTING.
54  *
55  * Cancellation
56  *
57  * ddi_periodic_delete(9F) historically had spectacularly loose semantics with
58  * respect to cancellation concurrent with handler execution.  These semantics
59  * are now tighter:
60  *
61  *   1. At most one invocation of ddi_periodic_delete(9F) will actually
62  *      perform the deletion, all others will return immediately.
63  *   2. The invocation that performs the deletion will _block_ until
64  *      the handler is no longer running, and all resources have been
65  *      released.
66  *
67  * We affect this model by removing the cancelling periodic from the
68  * global list and marking it DPF_CANCELLED.  This will prevent further
69  * execution of the handler.  We then wait on a CV until the DPF_EXECUTING
70  * and DPF_DISPATCHED flags are clear, which means the periodic is removed
71  * from all request queues, is no longer executing, and may be freed.  At this
72  * point we return the opaque ID to the id_space and free the memory.
73  *
74  * NOTE:
75  * The ddi_periodic_add(9F) interface is presently limited to a minimum period
76  * of 10ms between firings.
77  */
78 
79 /*
80  * Tuneables:
81  */
82 int ddi_periodic_max_id = 1024;
83 int ddi_periodic_taskq_threadcount = 4;
84 hrtime_t ddi_periodic_resolution = 10000000;
85 
86 /*
87  * Globals:
88  */
89 static kmem_cache_t *periodic_cache;
90 static id_space_t *periodic_id_space;
91 static taskq_t *periodic_taskq;
92 
93 /*
94  * periodics_lock protects the list of all periodics (periodics), and
95  * each of the soft interrupt request queues (periodic_softint_queue).
96  *
97  * Do not hold an individual periodic's lock while obtaining periodics_lock.
98  * While in the periodic_softint_queue list, the periodic will be marked
99  * DPF_DISPATCHED, and thus safe from frees.  Only the invocation of
100  * i_untimeout() that removes the periodic from the global list is allowed
101  * to free it.
102  */
103 static kmutex_t periodics_lock;
104 static list_t periodics;
105 static list_t periodic_softint_queue[10]; /* for IPL1 up to IPL10 */
106 
107 typedef enum periodic_ipl {
108 	PERI_IPL_0 = 0,
109 	PERI_IPL_1,
110 	PERI_IPL_2,
111 	PERI_IPL_3,
112 	PERI_IPL_4,
113 	PERI_IPL_5,
114 	PERI_IPL_6,
115 	PERI_IPL_7,
116 	PERI_IPL_8,
117 	PERI_IPL_9,
118 	PERI_IPL_10
119 } periodic_ipl_t;
120 
121 static char *
periodic_handler_symbol(ddi_periodic_impl_t * dpr)122 periodic_handler_symbol(ddi_periodic_impl_t *dpr)
123 {
124 	ulong_t off;
125 
126 	return (kobj_getsymname((uintptr_t)dpr->dpr_handler, &off));
127 }
128 
129 /*
130  * This function may be called either from a soft interrupt handler
131  * (ddi_periodic_softintr), or as a taskq worker function.
132  */
133 static void
periodic_execute(void * arg)134 periodic_execute(void *arg)
135 {
136 	ddi_periodic_impl_t *dpr = arg;
137 	mutex_enter(&dpr->dpr_lock);
138 
139 	/*
140 	 * We must be DISPATCHED, but not yet EXECUTING:
141 	 */
142 	VERIFY((dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) ==
143 	    DPF_DISPATCHED);
144 	VERIFY(dpr->dpr_thread == NULL);
145 
146 	if (!(dpr->dpr_flags & DPF_CANCELLED)) {
147 		int level = dpr->dpr_level;
148 		uint64_t count = dpr->dpr_fire_count;
149 		/*
150 		 * If we have not yet been cancelled, then
151 		 * mark us executing:
152 		 */
153 		dpr->dpr_flags |= DPF_EXECUTING;
154 		dpr->dpr_thread = curthread;
155 		mutex_exit(&dpr->dpr_lock);
156 
157 		/*
158 		 * Execute the handler, without holding locks:
159 		 */
160 		DTRACE_PROBE4(ddi__periodic__execute, void *, dpr->dpr_handler,
161 		    void *, dpr->dpr_arg, int, level, uint64_t, count);
162 		(*dpr->dpr_handler)(dpr->dpr_arg);
163 		DTRACE_PROBE4(ddi__periodic__done, void *, dpr->dpr_handler,
164 		    void *, dpr->dpr_arg, int, level, uint64_t, count);
165 
166 		mutex_enter(&dpr->dpr_lock);
167 		dpr->dpr_thread = NULL;
168 		dpr->dpr_fire_count++;
169 	}
170 
171 	/*
172 	 * We're done with this periodic for now, so release it and
173 	 * wake anybody that was waiting for us to be finished:
174 	 */
175 	dpr->dpr_flags &= ~(DPF_DISPATCHED | DPF_EXECUTING);
176 	cv_broadcast(&dpr->dpr_cv);
177 	mutex_exit(&dpr->dpr_lock);
178 }
179 
180 void
ddi_periodic_softintr(int level)181 ddi_periodic_softintr(int level)
182 {
183 	ddi_periodic_impl_t *dpr;
184 	VERIFY(level >= PERI_IPL_1 && level <= PERI_IPL_10);
185 
186 	mutex_enter(&periodics_lock);
187 	/*
188 	 * Pull the first scheduled periodic off the queue for this priority
189 	 * level:
190 	 */
191 	while ((dpr = list_remove_head(&periodic_softint_queue[level - 1])) !=
192 	    NULL) {
193 		mutex_exit(&periodics_lock);
194 		/*
195 		 * And execute it:
196 		 */
197 		periodic_execute(dpr);
198 		mutex_enter(&periodics_lock);
199 	}
200 	mutex_exit(&periodics_lock);
201 }
202 
203 void
ddi_periodic_init(void)204 ddi_periodic_init(void)
205 {
206 	int i;
207 
208 	/*
209 	 * Create a kmem_cache for request tracking objects, and a list
210 	 * to store them in so we can later delete based on opaque handles:
211 	 */
212 	periodic_cache = kmem_cache_create("ddi_periodic",
213 	    sizeof (ddi_periodic_impl_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
214 	list_create(&periodics, sizeof (ddi_periodic_impl_t),
215 	    offsetof(ddi_periodic_impl_t, dpr_link));
216 
217 	/*
218 	 * Initialise the identifier space for ddi_periodic_add(9F):
219 	 */
220 	periodic_id_space = id_space_create("ddi_periodic", 1,
221 	    ddi_periodic_max_id);
222 
223 	/*
224 	 * Initialise the request queue for each soft interrupt level:
225 	 */
226 	for (i = PERI_IPL_1; i <= PERI_IPL_10; i++) {
227 		list_create(&periodic_softint_queue[i - 1],
228 		    sizeof (ddi_periodic_impl_t), offsetof(ddi_periodic_impl_t,
229 		    dpr_softint_link));
230 	}
231 
232 	/*
233 	 * Create the taskq for running PERI_IPL_0 handlers.  This taskq will
234 	 * _only_ be used with taskq_dispatch_ent(), and a taskq_ent_t
235 	 * pre-allocated with the ddi_periodic_impl_t.
236 	 */
237 	periodic_taskq = taskq_create_instance("ddi_periodic_taskq", -1,
238 	    ddi_periodic_taskq_threadcount, maxclsyspri, 0, 0, 0);
239 
240 	/*
241 	 * Initialize the mutex lock used for the soft interrupt request
242 	 * queues.
243 	 */
244 	mutex_init(&periodics_lock, NULL, MUTEX_ADAPTIVE, NULL);
245 }
246 
247 void
ddi_periodic_fini(void)248 ddi_periodic_fini(void)
249 {
250 	int i;
251 	ddi_periodic_impl_t *dpr;
252 
253 	/*
254 	 * Find all periodics that have not yet been unregistered and,
255 	 * on DEBUG bits, print a warning about this resource leak.
256 	 */
257 	mutex_enter(&periodics_lock);
258 	while ((dpr = list_head(&periodics)) != NULL) {
259 #ifdef	DEBUG
260 		printf("DDI periodic handler not deleted (id=%lx, hdlr=%s)\n",
261 		    (unsigned long)dpr->dpr_id, periodic_handler_symbol(dpr));
262 #endif
263 
264 		mutex_exit(&periodics_lock);
265 		/*
266 		 * Delete the periodic ourselves:
267 		 */
268 		i_untimeout((timeout_t)(uintptr_t)dpr->dpr_id);
269 		mutex_enter(&periodics_lock);
270 	}
271 	mutex_exit(&periodics_lock);
272 
273 	/*
274 	 * At this point there are no remaining cyclics, so clean up the
275 	 * remaining resources:
276 	 */
277 	taskq_destroy(periodic_taskq);
278 	periodic_taskq = NULL;
279 
280 	id_space_destroy(periodic_id_space);
281 	periodic_id_space = NULL;
282 
283 	kmem_cache_destroy(periodic_cache);
284 	periodic_cache = NULL;
285 
286 	list_destroy(&periodics);
287 	for (i = PERI_IPL_1; i <= PERI_IPL_10; i++)
288 		list_destroy(&periodic_softint_queue[i - 1]);
289 
290 	mutex_destroy(&periodics_lock);
291 }
292 
293 static void
periodic_cyclic_handler(void * arg)294 periodic_cyclic_handler(void *arg)
295 {
296 	ddi_periodic_impl_t *dpr = arg;
297 
298 	mutex_enter(&dpr->dpr_lock);
299 	/*
300 	 * If we've been cancelled, or we're already dispatched, then exit
301 	 * immediately:
302 	 */
303 	if (dpr->dpr_flags & (DPF_CANCELLED | DPF_DISPATCHED)) {
304 		mutex_exit(&dpr->dpr_lock);
305 		return;
306 	}
307 	VERIFY(!(dpr->dpr_flags & DPF_EXECUTING));
308 
309 	/*
310 	 * This periodic is not presently dispatched, so dispatch it now:
311 	 */
312 	dpr->dpr_flags |= DPF_DISPATCHED;
313 	mutex_exit(&dpr->dpr_lock);
314 
315 	if (dpr->dpr_level == PERI_IPL_0) {
316 		/*
317 		 * DDI_IPL_0 periodics are dispatched onto the taskq:
318 		 */
319 		taskq_dispatch_ent(periodic_taskq, periodic_execute,
320 		    dpr, 0, &dpr->dpr_taskq_ent);
321 	} else {
322 		/*
323 		 * Higher priority periodics are handled by a soft interrupt
324 		 * handler.  Enqueue us for processing by the handler:
325 		 */
326 		mutex_enter(&periodics_lock);
327 		list_insert_tail(&periodic_softint_queue[dpr->dpr_level - 1],
328 		    dpr);
329 		mutex_exit(&periodics_lock);
330 
331 		/*
332 		 * Request the execution of the soft interrupt handler for this
333 		 * periodic's priority level.
334 		 */
335 		sir_on(dpr->dpr_level);
336 	}
337 }
338 
339 static void
periodic_destroy(ddi_periodic_impl_t * dpr)340 periodic_destroy(ddi_periodic_impl_t *dpr)
341 {
342 	if (dpr == NULL)
343 		return;
344 
345 	/*
346 	 * By now, we should have a periodic that is not busy, and has been
347 	 * cancelled:
348 	 */
349 	VERIFY(dpr->dpr_flags == DPF_CANCELLED);
350 	VERIFY(dpr->dpr_thread == NULL);
351 
352 	id_free(periodic_id_space, dpr->dpr_id);
353 	cv_destroy(&dpr->dpr_cv);
354 	mutex_destroy(&dpr->dpr_lock);
355 	kmem_cache_free(periodic_cache, dpr);
356 }
357 
358 static ddi_periodic_impl_t *
periodic_create(void)359 periodic_create(void)
360 {
361 	ddi_periodic_impl_t *dpr;
362 
363 	dpr = kmem_cache_alloc(periodic_cache, KM_SLEEP);
364 	bzero(dpr, sizeof (*dpr));
365 	dpr->dpr_id = id_alloc(periodic_id_space);
366 	mutex_init(&dpr->dpr_lock, NULL, MUTEX_ADAPTIVE, NULL);
367 	cv_init(&dpr->dpr_cv, NULL, CV_DEFAULT, NULL);
368 
369 	return (dpr);
370 }
371 
372 /*
373  * This function provides the implementation for the ddi_periodic_add(9F)
374  * interface.  It registers a periodic handler and returns an opaque identifier
375  * that can be unregistered via ddi_periodic_delete(9F)/i_untimeout().
376  *
377  * It may be called in user or kernel context, provided cpu_lock is not held.
378  */
379 timeout_t
i_timeout(void (* func)(void *),void * arg,hrtime_t interval,int level)380 i_timeout(void (*func)(void *), void *arg, hrtime_t interval, int level)
381 {
382 	cyc_handler_t cyh;
383 	cyc_time_t cyt;
384 	ddi_periodic_impl_t *dpr;
385 
386 	VERIFY(func != NULL);
387 	VERIFY(level >= 0 && level <= 10);
388 
389 	/*
390 	 * Allocate object to track this periodic:
391 	 */
392 	dpr = periodic_create();
393 	dpr->dpr_level = level;
394 	dpr->dpr_handler = func;
395 	dpr->dpr_arg = arg;
396 
397 	/*
398 	 * The minimum supported interval between firings of the periodic
399 	 * handler is 10ms; see ddi_periodic_add(9F) for more details.  If a
400 	 * shorter interval is requested, round up.
401 	 */
402 	if (ddi_periodic_resolution > interval) {
403 		cmn_err(CE_WARN,
404 		    "The periodic timeout (handler=%s, interval=%lld) "
405 		    "requests a finer interval than the supported resolution. "
406 		    "It rounds up to %lld\n", periodic_handler_symbol(dpr),
407 		    interval, ddi_periodic_resolution);
408 		interval = ddi_periodic_resolution;
409 	}
410 
411 	/*
412 	 * Ensure that the interval is an even multiple of the base resolution
413 	 * that is at least as long as the requested interval.
414 	 */
415 	dpr->dpr_interval = roundup(interval, ddi_periodic_resolution);
416 
417 	/*
418 	 * Create the underlying cyclic:
419 	 */
420 	cyh.cyh_func = periodic_cyclic_handler;
421 	cyh.cyh_arg = dpr;
422 	cyh.cyh_level = CY_LOCK_LEVEL;
423 
424 	cyt.cyt_when = 0;
425 	cyt.cyt_interval = dpr->dpr_interval;
426 
427 	mutex_enter(&cpu_lock);
428 	dpr->dpr_cyclic_id = cyclic_add(&cyh, &cyt);
429 	mutex_exit(&cpu_lock);
430 
431 	/*
432 	 * Make the id visible to ddi_periodic_delete(9F) before we
433 	 * return it:
434 	 */
435 	mutex_enter(&periodics_lock);
436 	list_insert_tail(&periodics, dpr);
437 	mutex_exit(&periodics_lock);
438 
439 	return ((timeout_t)(uintptr_t)dpr->dpr_id);
440 }
441 
442 /*
443  * This function provides the implementation for the ddi_periodic_delete(9F)
444  * interface.  It cancels a periodic handler previously registered through
445  * ddi_periodic_add(9F)/i_timeout().
446  *
447  * It may be called in user or kernel context, provided cpu_lock is not held.
448  * It may NOT be called from within a periodic handler.
449  */
450 void
i_untimeout(timeout_t id)451 i_untimeout(timeout_t id)
452 {
453 	ddi_periodic_impl_t *dpr;
454 
455 	/*
456 	 * Find the periodic in the list of all periodics and remove it.
457 	 * If we find in (and remove it from) the global list, we have
458 	 * license to free it once it is no longer busy.
459 	 */
460 	mutex_enter(&periodics_lock);
461 	for (dpr = list_head(&periodics); dpr != NULL; dpr =
462 	    list_next(&periodics, dpr)) {
463 		if (dpr->dpr_id == (id_t)(uintptr_t)id) {
464 			list_remove(&periodics, dpr);
465 			break;
466 		}
467 	}
468 	mutex_exit(&periodics_lock);
469 
470 	/*
471 	 * We could not find a periodic for this id, so bail out:
472 	 */
473 	if (dpr == NULL)
474 		return;
475 
476 	mutex_enter(&dpr->dpr_lock);
477 	/*
478 	 * We should be the only one trying to cancel this periodic:
479 	 */
480 	VERIFY(!(dpr->dpr_flags & DPF_CANCELLED));
481 	/*
482 	 * Removing a periodic from within its own handler function will
483 	 * cause a deadlock, so panic explicitly.
484 	 */
485 	if (dpr->dpr_thread == curthread) {
486 		panic("ddi_periodic_delete(%lx) called from its own handler\n",
487 		    (unsigned long)dpr->dpr_id);
488 	}
489 	/*
490 	 * Mark the periodic as cancelled:
491 	 */
492 	dpr->dpr_flags |= DPF_CANCELLED;
493 	mutex_exit(&dpr->dpr_lock);
494 
495 	/*
496 	 * Cancel our cyclic.  cyclic_remove() guarantees that the cyclic
497 	 * handler will not run again after it returns.  Note that the cyclic
498 	 * handler merely _dispatches_ the periodic, so this does _not_ mean
499 	 * the periodic handler is also finished running.
500 	 */
501 	mutex_enter(&cpu_lock);
502 	cyclic_remove(dpr->dpr_cyclic_id);
503 	mutex_exit(&cpu_lock);
504 
505 	/*
506 	 * Wait until the periodic handler is no longer running:
507 	 */
508 	mutex_enter(&dpr->dpr_lock);
509 	while (dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) {
510 		cv_wait(&dpr->dpr_cv, &dpr->dpr_lock);
511 	}
512 	mutex_exit(&dpr->dpr_lock);
513 
514 	periodic_destroy(dpr);
515 }
516