1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 /* 12 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 13 */ 14 15 #include <sys/cmn_err.h> 16 #include <sys/ddi_periodic.h> 17 #include <sys/id_space.h> 18 #include <sys/kobj.h> 19 #include <sys/sysmacros.h> 20 #include <sys/systm.h> 21 #include <sys/taskq.h> 22 #include <sys/taskq_impl.h> 23 #include <sys/time.h> 24 #include <sys/types.h> 25 #include <sys/sdt.h> 26 27 extern void sir_on(int); 28 29 /* 30 * The ddi_periodic_add(9F) Implementation 31 * 32 * This file contains the implementation of the ddi_periodic_add(9F) interface. 33 * It is a thin wrapper around the cyclic subsystem (see documentation in 34 * uts/common/os/cyclic.c), providing a DDI interface for registering 35 * (and unregistering) callbacks for periodic invocation at arbitrary 36 * interrupt levels, or in kernel context. 37 * 38 * Each call to ddi_periodic_add will result in a new opaque handle, as 39 * allocated from an id_space, a new "periodic" object (ddi_periodic_impl_t) 40 * and a registered cyclic. 41 * 42 * Operation 43 * 44 * Whenever the cyclic fires, our cyclic handler checks that the particular 45 * periodic is not dispatched already (we do not support overlapping execution 46 * of the consumer's handler function), and not yet cancelled. If both of 47 * these conditions hold, we mark the periodic as DPF_DISPATCHED and enqueue it 48 * to either the taskq (for DDI_IPL_0) or to one of the soft interrupt queues 49 * (DDI_IPL_1 to DDI_IPL_10). 50 * 51 * While the taskq (or soft interrupt handler) is handling a particular 52 * periodic, we mark it as DPF_EXECUTING. When complete, we reset both 53 * DPF_DISPATCHED and DPF_EXECUTING. 54 * 55 * Cancellation 56 * 57 * ddi_periodic_delete(9F) historically had spectacularly loose semantics with 58 * respect to cancellation concurrent with handler execution. These semantics 59 * are now tighter: 60 * 61 * 1. At most one invocation of ddi_periodic_delete(9F) will actually 62 * perform the deletion, all others will return immediately. 63 * 2. The invocation that performs the deletion will _block_ until 64 * the handler is no longer running, and all resources have been 65 * released. 66 * 67 * We affect this model by removing the cancelling periodic from the 68 * global list and marking it DPF_CANCELLED. This will prevent further 69 * execution of the handler. We then wait on a CV until the DPF_EXECUTING 70 * and DPF_DISPATCHED flags are clear, which means the periodic is removed 71 * from all request queues, is no longer executing, and may be freed. At this 72 * point we return the opaque ID to the id_space and free the memory. 73 * 74 * NOTE: 75 * The ddi_periodic_add(9F) interface is presently limited to a minimum period 76 * of 10ms between firings. 77 */ 78 79 /* 80 * Tuneables: 81 */ 82 int ddi_periodic_max_id = 1024; 83 int ddi_periodic_taskq_threadcount = 4; 84 hrtime_t ddi_periodic_resolution = 10000000; 85 86 /* 87 * Globals: 88 */ 89 static kmem_cache_t *periodic_cache; 90 static id_space_t *periodic_id_space; 91 static taskq_t *periodic_taskq; 92 93 /* 94 * periodics_lock protects the list of all periodics (periodics), and 95 * each of the soft interrupt request queues (periodic_softint_queue). 96 * 97 * Do not hold an individual periodic's lock while obtaining periodics_lock. 98 * While in the periodic_softint_queue list, the periodic will be marked 99 * DPF_DISPATCHED, and thus safe from frees. Only the invocation of 100 * i_untimeout() that removes the periodic from the global list is allowed 101 * to free it. 102 */ 103 static kmutex_t periodics_lock; 104 static list_t periodics; 105 static list_t periodic_softint_queue[10]; /* for IPL1 up to IPL10 */ 106 107 typedef enum periodic_ipl { 108 PERI_IPL_0 = 0, 109 PERI_IPL_1, 110 PERI_IPL_2, 111 PERI_IPL_3, 112 PERI_IPL_4, 113 PERI_IPL_5, 114 PERI_IPL_6, 115 PERI_IPL_7, 116 PERI_IPL_8, 117 PERI_IPL_9, 118 PERI_IPL_10 119 } periodic_ipl_t; 120 121 static char * 122 periodic_handler_symbol(ddi_periodic_impl_t *dpr) 123 { 124 ulong_t off; 125 126 return (kobj_getsymname((uintptr_t)dpr->dpr_handler, &off)); 127 } 128 129 /* 130 * This function may be called either from a soft interrupt handler 131 * (ddi_periodic_softintr), or as a taskq worker function. 132 */ 133 static void 134 periodic_execute(void *arg) 135 { 136 ddi_periodic_impl_t *dpr = arg; 137 mutex_enter(&dpr->dpr_lock); 138 139 /* 140 * We must be DISPATCHED, but not yet EXECUTING: 141 */ 142 VERIFY((dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) == 143 DPF_DISPATCHED); 144 VERIFY(dpr->dpr_thread == NULL); 145 146 if (!(dpr->dpr_flags & DPF_CANCELLED)) { 147 int level = dpr->dpr_level; 148 uint64_t count = dpr->dpr_fire_count; 149 /* 150 * If we have not yet been cancelled, then 151 * mark us executing: 152 */ 153 dpr->dpr_flags |= DPF_EXECUTING; 154 dpr->dpr_thread = curthread; 155 mutex_exit(&dpr->dpr_lock); 156 157 /* 158 * Execute the handler, without holding locks: 159 */ 160 DTRACE_PROBE4(ddi__periodic__execute, void *, dpr->dpr_handler, 161 void *, dpr->dpr_arg, int, level, uint64_t, count); 162 (*dpr->dpr_handler)(dpr->dpr_arg); 163 DTRACE_PROBE4(ddi__periodic__done, void *, dpr->dpr_handler, 164 void *, dpr->dpr_arg, int, level, uint64_t, count); 165 166 mutex_enter(&dpr->dpr_lock); 167 dpr->dpr_thread = NULL; 168 dpr->dpr_fire_count++; 169 } 170 171 /* 172 * We're done with this periodic for now, so release it and 173 * wake anybody that was waiting for us to be finished: 174 */ 175 dpr->dpr_flags &= ~(DPF_DISPATCHED | DPF_EXECUTING); 176 cv_broadcast(&dpr->dpr_cv); 177 mutex_exit(&dpr->dpr_lock); 178 } 179 180 void 181 ddi_periodic_softintr(int level) 182 { 183 ddi_periodic_impl_t *dpr; 184 VERIFY(level >= PERI_IPL_1 && level <= PERI_IPL_10); 185 186 mutex_enter(&periodics_lock); 187 /* 188 * Pull the first scheduled periodic off the queue for this priority 189 * level: 190 */ 191 while ((dpr = list_remove_head(&periodic_softint_queue[level - 1])) != 192 NULL) { 193 mutex_exit(&periodics_lock); 194 /* 195 * And execute it: 196 */ 197 periodic_execute(dpr); 198 mutex_enter(&periodics_lock); 199 } 200 mutex_exit(&periodics_lock); 201 } 202 203 void 204 ddi_periodic_init(void) 205 { 206 int i; 207 208 /* 209 * Create a kmem_cache for request tracking objects, and a list 210 * to store them in so we can later delete based on opaque handles: 211 */ 212 periodic_cache = kmem_cache_create("ddi_periodic", 213 sizeof (ddi_periodic_impl_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 214 list_create(&periodics, sizeof (ddi_periodic_impl_t), 215 offsetof(ddi_periodic_impl_t, dpr_link)); 216 217 /* 218 * Initialise the identifier space for ddi_periodic_add(9F): 219 */ 220 periodic_id_space = id_space_create("ddi_periodic", 1, 221 ddi_periodic_max_id); 222 223 /* 224 * Initialise the request queue for each soft interrupt level: 225 */ 226 for (i = PERI_IPL_1; i <= PERI_IPL_10; i++) { 227 list_create(&periodic_softint_queue[i - 1], 228 sizeof (ddi_periodic_impl_t), offsetof(ddi_periodic_impl_t, 229 dpr_softint_link)); 230 } 231 232 /* 233 * Create the taskq for running PERI_IPL_0 handlers. This taskq will 234 * _only_ be used with taskq_dispatch_ent(), and a taskq_ent_t 235 * pre-allocated with the ddi_periodic_impl_t. 236 */ 237 periodic_taskq = taskq_create_instance("ddi_periodic_taskq", -1, 238 ddi_periodic_taskq_threadcount, maxclsyspri, 0, 0, 0); 239 240 /* 241 * Initialize the mutex lock used for the soft interrupt request 242 * queues. 243 */ 244 mutex_init(&periodics_lock, NULL, MUTEX_ADAPTIVE, NULL); 245 } 246 247 void 248 ddi_periodic_fini(void) 249 { 250 int i; 251 ddi_periodic_impl_t *dpr; 252 253 /* 254 * Find all periodics that have not yet been unregistered and, 255 * on DEBUG bits, print a warning about this resource leak. 256 */ 257 mutex_enter(&periodics_lock); 258 while ((dpr = list_head(&periodics)) != NULL) { 259 #ifdef DEBUG 260 printf("DDI periodic handler not deleted (id=%lx, hdlr=%s)\n", 261 (unsigned long)dpr->dpr_id, periodic_handler_symbol(dpr)); 262 #endif 263 264 mutex_exit(&periodics_lock); 265 /* 266 * Delete the periodic ourselves: 267 */ 268 i_untimeout((timeout_t)(uintptr_t)dpr->dpr_id); 269 mutex_enter(&periodics_lock); 270 } 271 mutex_exit(&periodics_lock); 272 273 /* 274 * At this point there are no remaining cyclics, so clean up the 275 * remaining resources: 276 */ 277 taskq_destroy(periodic_taskq); 278 periodic_taskq = NULL; 279 280 id_space_destroy(periodic_id_space); 281 periodic_id_space = NULL; 282 283 kmem_cache_destroy(periodic_cache); 284 periodic_cache = NULL; 285 286 list_destroy(&periodics); 287 for (i = PERI_IPL_1; i <= PERI_IPL_10; i++) 288 list_destroy(&periodic_softint_queue[i - 1]); 289 290 mutex_destroy(&periodics_lock); 291 } 292 293 static void 294 periodic_cyclic_handler(void *arg) 295 { 296 ddi_periodic_impl_t *dpr = arg; 297 298 mutex_enter(&dpr->dpr_lock); 299 /* 300 * If we've been cancelled, or we're already dispatched, then exit 301 * immediately: 302 */ 303 if (dpr->dpr_flags & (DPF_CANCELLED | DPF_DISPATCHED)) { 304 mutex_exit(&dpr->dpr_lock); 305 return; 306 } 307 VERIFY(!(dpr->dpr_flags & DPF_EXECUTING)); 308 309 /* 310 * This periodic is not presently dispatched, so dispatch it now: 311 */ 312 dpr->dpr_flags |= DPF_DISPATCHED; 313 mutex_exit(&dpr->dpr_lock); 314 315 if (dpr->dpr_level == PERI_IPL_0) { 316 /* 317 * DDI_IPL_0 periodics are dispatched onto the taskq: 318 */ 319 taskq_dispatch_ent(periodic_taskq, periodic_execute, 320 dpr, 0, &dpr->dpr_taskq_ent); 321 } else { 322 /* 323 * Higher priority periodics are handled by a soft interrupt 324 * handler. Enqueue us for processing by the handler: 325 */ 326 mutex_enter(&periodics_lock); 327 list_insert_tail(&periodic_softint_queue[dpr->dpr_level - 1], 328 dpr); 329 mutex_exit(&periodics_lock); 330 331 /* 332 * Request the execution of the soft interrupt handler for this 333 * periodic's priority level. 334 */ 335 sir_on(dpr->dpr_level); 336 } 337 } 338 339 static void 340 periodic_destroy(ddi_periodic_impl_t *dpr) 341 { 342 if (dpr == NULL) 343 return; 344 345 /* 346 * By now, we should have a periodic that is not busy, and has been 347 * cancelled: 348 */ 349 VERIFY(dpr->dpr_flags == DPF_CANCELLED); 350 VERIFY(dpr->dpr_thread == NULL); 351 352 id_free(periodic_id_space, dpr->dpr_id); 353 cv_destroy(&dpr->dpr_cv); 354 mutex_destroy(&dpr->dpr_lock); 355 kmem_cache_free(periodic_cache, dpr); 356 } 357 358 static ddi_periodic_impl_t * 359 periodic_create(void) 360 { 361 ddi_periodic_impl_t *dpr; 362 363 dpr = kmem_cache_alloc(periodic_cache, KM_SLEEP); 364 bzero(dpr, sizeof (*dpr)); 365 dpr->dpr_id = id_alloc(periodic_id_space); 366 mutex_init(&dpr->dpr_lock, NULL, MUTEX_ADAPTIVE, NULL); 367 cv_init(&dpr->dpr_cv, NULL, CV_DEFAULT, NULL); 368 369 return (dpr); 370 } 371 372 /* 373 * This function provides the implementation for the ddi_periodic_add(9F) 374 * interface. It registers a periodic handler and returns an opaque identifier 375 * that can be unregistered via ddi_periodic_delete(9F)/i_untimeout(). 376 * 377 * It may be called in user or kernel context, provided cpu_lock is not held. 378 */ 379 timeout_t 380 i_timeout(void (*func)(void *), void *arg, hrtime_t interval, int level) 381 { 382 cyc_handler_t cyh; 383 cyc_time_t cyt; 384 ddi_periodic_impl_t *dpr; 385 386 VERIFY(func != NULL); 387 VERIFY(level >= 0 && level <= 10); 388 389 /* 390 * Allocate object to track this periodic: 391 */ 392 dpr = periodic_create(); 393 dpr->dpr_level = level; 394 dpr->dpr_handler = func; 395 dpr->dpr_arg = arg; 396 397 /* 398 * The minimum supported interval between firings of the periodic 399 * handler is 10ms; see ddi_periodic_add(9F) for more details. If a 400 * shorter interval is requested, round up. 401 */ 402 if (ddi_periodic_resolution > interval) { 403 cmn_err(CE_WARN, 404 "The periodic timeout (handler=%s, interval=%lld) " 405 "requests a finer interval than the supported resolution. " 406 "It rounds up to %lld\n", periodic_handler_symbol(dpr), 407 interval, ddi_periodic_resolution); 408 interval = ddi_periodic_resolution; 409 } 410 411 /* 412 * Ensure that the interval is an even multiple of the base resolution 413 * that is at least as long as the requested interval. 414 */ 415 dpr->dpr_interval = roundup(interval, ddi_periodic_resolution); 416 417 /* 418 * Create the underlying cyclic: 419 */ 420 cyh.cyh_func = periodic_cyclic_handler; 421 cyh.cyh_arg = dpr; 422 cyh.cyh_level = CY_LOCK_LEVEL; 423 424 cyt.cyt_when = 0; 425 cyt.cyt_interval = dpr->dpr_interval; 426 427 mutex_enter(&cpu_lock); 428 dpr->dpr_cyclic_id = cyclic_add(&cyh, &cyt); 429 mutex_exit(&cpu_lock); 430 431 /* 432 * Make the id visible to ddi_periodic_delete(9F) before we 433 * return it: 434 */ 435 mutex_enter(&periodics_lock); 436 list_insert_tail(&periodics, dpr); 437 mutex_exit(&periodics_lock); 438 439 return ((timeout_t)(uintptr_t)dpr->dpr_id); 440 } 441 442 /* 443 * This function provides the implementation for the ddi_periodic_delete(9F) 444 * interface. It cancels a periodic handler previously registered through 445 * ddi_periodic_add(9F)/i_timeout(). 446 * 447 * It may be called in user or kernel context, provided cpu_lock is not held. 448 * It may NOT be called from within a periodic handler. 449 */ 450 void 451 i_untimeout(timeout_t id) 452 { 453 ddi_periodic_impl_t *dpr; 454 455 /* 456 * Find the periodic in the list of all periodics and remove it. 457 * If we find in (and remove it from) the global list, we have 458 * license to free it once it is no longer busy. 459 */ 460 mutex_enter(&periodics_lock); 461 for (dpr = list_head(&periodics); dpr != NULL; dpr = 462 list_next(&periodics, dpr)) { 463 if (dpr->dpr_id == (id_t)(uintptr_t)id) { 464 list_remove(&periodics, dpr); 465 break; 466 } 467 } 468 mutex_exit(&periodics_lock); 469 470 /* 471 * We could not find a periodic for this id, so bail out: 472 */ 473 if (dpr == NULL) 474 return; 475 476 mutex_enter(&dpr->dpr_lock); 477 /* 478 * We should be the only one trying to cancel this periodic: 479 */ 480 VERIFY(!(dpr->dpr_flags & DPF_CANCELLED)); 481 /* 482 * Removing a periodic from within its own handler function will 483 * cause a deadlock, so panic explicitly. 484 */ 485 if (dpr->dpr_thread == curthread) { 486 panic("ddi_periodic_delete(%lx) called from its own handler\n", 487 (unsigned long)dpr->dpr_id); 488 } 489 /* 490 * Mark the periodic as cancelled: 491 */ 492 dpr->dpr_flags |= DPF_CANCELLED; 493 mutex_exit(&dpr->dpr_lock); 494 495 /* 496 * Cancel our cyclic. cyclic_remove() guarantees that the cyclic 497 * handler will not run again after it returns. Note that the cyclic 498 * handler merely _dispatches_ the periodic, so this does _not_ mean 499 * the periodic handler is also finished running. 500 */ 501 mutex_enter(&cpu_lock); 502 cyclic_remove(dpr->dpr_cyclic_id); 503 mutex_exit(&cpu_lock); 504 505 /* 506 * Wait until the periodic handler is no longer running: 507 */ 508 mutex_enter(&dpr->dpr_lock); 509 while (dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) { 510 cv_wait(&dpr->dpr_cv, &dpr->dpr_lock); 511 } 512 mutex_exit(&dpr->dpr_lock); 513 514 periodic_destroy(dpr); 515 } 516