1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/kmem.h> 29 #include <sys/note.h> 30 31 #include "ghd.h" 32 33 34 35 /*ARGSUSED*/ 36 gtgt_t * 37 ghd_target_init(dev_info_t *hba_dip, 38 dev_info_t *tgt_dip, 39 ccc_t *cccp, 40 size_t tgt_private_size, 41 void *hba_private, 42 ushort_t target, 43 uchar_t lun) 44 { 45 _NOTE(ARGUNUSED(hba_dip)) 46 gtgt_t *gtgtp; 47 size_t size = sizeof (*gtgtp) + tgt_private_size; 48 gdev_t *gdevp; 49 ulong_t maxactive; 50 51 gtgtp = kmem_zalloc(size, KM_SLEEP); 52 53 /* 54 * initialize the per instance structure 55 */ 56 57 gtgtp->gt_tgt_private = (void *)(gtgtp + 1); 58 gtgtp->gt_size = size; 59 gtgtp->gt_hba_private = hba_private; 60 gtgtp->gt_target = target; 61 gtgtp->gt_lun = lun; 62 gtgtp->gt_ccc = cccp; 63 64 /* 65 * set the queue's maxactive to 1 if 66 * property not specified on target or hba devinfo node 67 */ 68 maxactive = ddi_getprop(DDI_DEV_T_ANY, tgt_dip, 0, "ghd-maxactive", 1); 69 gtgtp->gt_maxactive = maxactive; 70 71 /* initialize the linked list pointers */ 72 GTGT_INIT(gtgtp); 73 74 /* 75 * grab both mutexes so the queue structures 76 * stay stable while adding this instance to the linked lists 77 */ 78 mutex_enter(&cccp->ccc_hba_mutex); 79 mutex_enter(&cccp->ccc_waitq_mutex); 80 81 /* 82 * Search the HBA's linked list of device structures. 83 * 84 * If this device is already attached then link this instance 85 * to the existing per-device-structure on the ccc_devs list. 86 * 87 */ 88 gdevp = CCCP2GDEVP(cccp); 89 while (gdevp != NULL) { 90 if (gdevp->gd_target == target && gdevp->gd_lun == lun) { 91 GDBG_WAITQ(("ghd_target_init(%d,%d) found gdevp 0x%p" 92 " gtgtp 0x%p max %lu\n", target, lun, 93 (void *)gdevp, (void *)gtgtp, maxactive)); 94 95 goto foundit; 96 } 97 gdevp = GDEV_NEXTP(gdevp); 98 } 99 100 /* 101 * Not found. This is the first instance for this device. 102 */ 103 104 105 /* allocate the per-device-structure */ 106 107 gdevp = kmem_zalloc(sizeof (*gdevp), KM_SLEEP); 108 gdevp->gd_target = target; 109 gdevp->gd_lun = lun; 110 111 /* 112 * link this second level queue to the HBA's first 113 * level queue 114 */ 115 GDEV_QATTACH(gdevp, cccp, maxactive); 116 117 GDBG_WAITQ(("ghd_target_init(%d,%d) new gdevp 0x%p gtgtp 0x%p" 118 " max %lu\n", target, lun, (void *)gdevp, (void *)gtgtp, 119 maxactive)); 120 121 foundit: 122 123 /* save the ptr to the per device structure */ 124 gtgtp->gt_gdevp = gdevp; 125 126 /* Add the per instance structure to the per device list */ 127 GTGT_ATTACH(gtgtp, gdevp); 128 129 ghd_waitq_process_and_mutex_exit(cccp); 130 131 return (gtgtp); 132 } 133 134 /*ARGSUSED*/ 135 void 136 ghd_target_free(dev_info_t *hba_dip, 137 dev_info_t *tgt_dip, 138 ccc_t *cccp, 139 gtgt_t *gtgtp) 140 { 141 _NOTE(ARGUNUSED(hba_dip,tgt_dip)) 142 143 gdev_t *gdevp = gtgtp->gt_gdevp; 144 145 GDBG_WAITQ(("ghd_target_free(%d,%d) gdevp-0x%p gtgtp 0x%p\n", 146 gtgtp->gt_target, gtgtp->gt_lun, (void *)gdevp, (void *)gtgtp)); 147 148 /* 149 * grab both mutexes so the queue structures 150 * stay stable while deleting this instance 151 */ 152 mutex_enter(&cccp->ccc_hba_mutex); 153 mutex_enter(&cccp->ccc_waitq_mutex); 154 155 ASSERT(gdevp->gd_ninstances > 0); 156 157 /* 158 * remove this per-instance structure from the device list and 159 * free the memory 160 */ 161 GTGT_DEATTACH(gtgtp, gdevp); 162 kmem_free((caddr_t)gtgtp, gtgtp->gt_size); 163 164 if (gdevp->gd_ninstances == 1) { 165 GDBG_WAITQ(("ghd_target_free: N=1 gdevp 0x%p\n", 166 (void *)gdevp)); 167 /* 168 * If there's now just one instance left attached to this 169 * device then reset the queue's max active value 170 * from that instance's saved value. 171 */ 172 gtgtp = GDEVP2GTGTP(gdevp); 173 GDEV_MAXACTIVE(gdevp) = gtgtp->gt_maxactive; 174 175 } else if (gdevp->gd_ninstances == 0) { 176 /* else no instances left */ 177 GDBG_WAITQ(("ghd_target_free: N=0 gdevp 0x%p\n", 178 (void *)gdevp)); 179 180 /* detach this per-dev-structure from the HBA's dev list */ 181 GDEV_QDETACH(gdevp, cccp); 182 kmem_free(gdevp, sizeof (*gdevp)); 183 184 } 185 #if defined(GHD_DEBUG) || defined(__lint) 186 else { 187 /* leave maxactive set to 1 */ 188 GDBG_WAITQ(("ghd_target_free: N>1 gdevp 0x%p\n", 189 (void *)gdevp)); 190 } 191 #endif 192 193 ghd_waitq_process_and_mutex_exit(cccp); 194 } 195 196 void 197 ghd_waitq_shuffle_up(ccc_t *cccp, gdev_t *gdevp) 198 { 199 gcmd_t *gcmdp; 200 201 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex)); 202 203 GDBG_WAITQ(("ghd_waitq_shuffle_up: cccp 0x%p gdevp 0x%p N %ld " 204 "max %ld\n", (void *)cccp, (void *)gdevp, GDEV_NACTIVE(gdevp), 205 GDEV_MAXACTIVE(gdevp))); 206 for (;;) { 207 /* 208 * Now check the device wait queue throttle to see if I can 209 * shuffle up a request to the HBA wait queue. 210 */ 211 if (GDEV_NACTIVE(gdevp) >= GDEV_MAXACTIVE(gdevp)) { 212 GDBG_WAITQ(("ghd_waitq_shuffle_up: N>MAX gdevp 0x%p\n", 213 (void *)gdevp)); 214 return; 215 } 216 217 /* 218 * single thread requests while multiple instances 219 * because the different target drives might have 220 * conflicting maxactive throttles. 221 */ 222 if (gdevp->gd_ninstances > 1 && GDEV_NACTIVE(gdevp) > 0) { 223 GDBG_WAITQ(("ghd_waitq_shuffle_up: multi gdevp 0x%p\n", 224 (void *)gdevp)); 225 return; 226 } 227 228 /* 229 * promote the topmost request from the device queue to 230 * the HBA queue. 231 */ 232 if ((gcmdp = L2_remove_head(&GDEV_QHEAD(gdevp))) == NULL) { 233 /* the device is empty so we're done */ 234 GDBG_WAITQ(("ghd_waitq_shuffle_up: MT gdevp 0x%p\n", 235 (void *)gdevp)); 236 return; 237 } 238 L2_add(&GHBA_QHEAD(cccp), &gcmdp->cmd_q, gcmdp); 239 GDEV_NACTIVE(gdevp)++; 240 gcmdp->cmd_waitq_level++; 241 GDBG_WAITQ(("ghd_waitq_shuffle_up: gdevp 0x%p gcmdp 0x%p\n", 242 (void *)gdevp, (void *)gcmdp)); 243 } 244 } 245 246 247 void 248 ghd_waitq_delete(ccc_t *cccp, gcmd_t *gcmdp) 249 { 250 gtgt_t *gtgtp = GCMDP2GTGTP(gcmdp); 251 gdev_t *gdevp = gtgtp->gt_gdevp; 252 #if defined(GHD_DEBUG) || defined(__lint) 253 Q_t *qp = &gdevp->gd_waitq; 254 #endif 255 256 ASSERT(mutex_owned(&cccp->ccc_hba_mutex)); 257 mutex_enter(&cccp->ccc_waitq_mutex); 258 259 /* 260 * Adjust all queue counters. If this request is being aborted 261 * it might only have made it to the target queue. Otherwise, 262 * both the target and hba queue have to be adjusted when a 263 * request is completed normally. The cmd_waitq_level value 264 * indicates which queue counters need to be adjusted. It's 265 * incremented as the request progresses up the queues. 266 */ 267 switch (gcmdp->cmd_waitq_level) { 268 case 0: 269 break; 270 case 1: 271 /* 272 * If this is an early-timeout, or early-abort, the request 273 * is still linked onto a waitq. Remove it now. If it's 274 * an active request and no longer on the waitq then calling 275 * L2_delete a second time does no harm. 276 */ 277 L2_delete(&gcmdp->cmd_q); 278 break; 279 280 case 2: 281 L2_delete(&gcmdp->cmd_q); 282 #if defined(GHD_DEBUG) || defined(__lint) 283 if (GDEV_NACTIVE(gdevp) == 0) 284 debug_enter("\n\nGHD WAITQ DELETE\n\n"); 285 #endif 286 GDEV_NACTIVE(gdevp)--; 287 break; 288 289 case 3: 290 /* it's an active or completed command */ 291 #if defined(GHD_DEBUG) || defined(__lint) 292 if (GDEV_NACTIVE(gdevp) == 0 || GHBA_NACTIVE(cccp) == 0) 293 debug_enter("\n\nGHD WAITQ DELETE\n\n"); 294 #endif 295 GDEV_NACTIVE(gdevp)--; 296 GHBA_NACTIVE(cccp)--; 297 break; 298 299 default: 300 /* this shouldn't happen */ 301 #if defined(GHD_DEBUG) || defined(__lint) 302 debug_enter("\n\nGHD WAITQ LEVEL > 3\n\n"); 303 #endif 304 break; 305 } 306 307 GDBG_WAITQ(("ghd_waitq_delete: gcmdp 0x%p qp 0x%p level %ld\n", 308 (void *)gcmdp, (void *)qp, gcmdp->cmd_waitq_level)); 309 310 311 /* 312 * There's probably now more room in the HBA queue. Move 313 * up as many requests as possible. 314 */ 315 ghd_waitq_shuffle_up(cccp, gdevp); 316 317 mutex_exit(&cccp->ccc_waitq_mutex); 318 } 319 320 321 int 322 ghd_waitq_process_and_mutex_hold(ccc_t *cccp) 323 { 324 gcmd_t *gcmdp; 325 int rc = FALSE; 326 327 ASSERT(mutex_owned(&cccp->ccc_hba_mutex)); 328 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex)); 329 330 for (;;) { 331 if (L2_EMPTY(&GHBA_QHEAD(cccp))) { 332 /* return if the list is empty */ 333 GDBG_WAITQ(("ghd_waitq_proc: MT cccp 0x%p qp 0x%p\n", 334 (void *)cccp, (void *)&cccp->ccc_waitq)); 335 break; 336 } 337 if (GHBA_NACTIVE(cccp) >= GHBA_MAXACTIVE(cccp)) { 338 /* return if the HBA is too active */ 339 GDBG_WAITQ(("ghd_waitq_proc: N>M cccp 0x%p qp 0x%p" 340 " N %ld max %ld\n", (void *)cccp, 341 (void *)&cccp->ccc_waitq, 342 GHBA_NACTIVE(cccp), 343 GHBA_MAXACTIVE(cccp))); 344 break; 345 } 346 347 /* 348 * bail out if the wait queue has been 349 * "held" by the HBA driver 350 */ 351 if (cccp->ccc_waitq_held) { 352 GDBG_WAITQ(("ghd_waitq_proc: held")); 353 return (rc); 354 } 355 356 if (cccp->ccc_waitq_frozen) { 357 358 clock_t lbolt, delay_in_hz, time_to_wait; 359 360 delay_in_hz = 361 drv_usectohz(cccp->ccc_waitq_freezedelay * 1000); 362 363 lbolt = ddi_get_lbolt(); 364 time_to_wait = delay_in_hz - 365 (lbolt - cccp->ccc_waitq_freezetime); 366 367 if (time_to_wait > 0) { 368 /* 369 * stay frozen; we'll be called again 370 * by ghd_timeout_softintr() 371 */ 372 GDBG_WAITQ(("ghd_waitq_proc: frozen")); 373 return (rc); 374 } else { 375 /* unfreeze and continue */ 376 GDBG_WAITQ(("ghd_waitq_proc: unfreezing")); 377 cccp->ccc_waitq_freezetime = 0; 378 cccp->ccc_waitq_freezedelay = 0; 379 cccp->ccc_waitq_frozen = 0; 380 } 381 } 382 383 gcmdp = (gcmd_t *)L2_remove_head(&GHBA_QHEAD(cccp)); 384 GHBA_NACTIVE(cccp)++; 385 gcmdp->cmd_waitq_level++; 386 mutex_exit(&cccp->ccc_waitq_mutex); 387 388 /* 389 * Start up the next I/O request 390 */ 391 ASSERT(gcmdp != NULL); 392 gcmdp->cmd_state = GCMD_STATE_ACTIVE; 393 if (!(*cccp->ccc_hba_start)(cccp->ccc_hba_handle, gcmdp)) { 394 /* if the HBA rejected the request, requeue it */ 395 gcmdp->cmd_state = GCMD_STATE_WAITQ; 396 mutex_enter(&cccp->ccc_waitq_mutex); 397 GHBA_NACTIVE(cccp)--; 398 gcmdp->cmd_waitq_level--; 399 L2_add_head(&GHBA_QHEAD(cccp), &gcmdp->cmd_q, gcmdp); 400 GDBG_WAITQ(("ghd_waitq_proc: busy cccp 0x%p gcmdp 0x%p" 401 " handle 0x%p\n", (void *)cccp, (void *)gcmdp, 402 cccp->ccc_hba_handle)); 403 break; 404 } 405 rc = TRUE; 406 mutex_enter(&cccp->ccc_waitq_mutex); 407 GDBG_WAITQ(("ghd_waitq_proc: ++ cccp 0x%p gcmdp 0x%p N %ld\n", 408 (void *)cccp, (void *)gcmdp, GHBA_NACTIVE(cccp))); 409 } 410 ASSERT(mutex_owned(&cccp->ccc_hba_mutex)); 411 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex)); 412 return (rc); 413 } 414 415 void 416 ghd_waitq_process_and_mutex_exit(ccc_t *cccp) 417 { 418 ASSERT(mutex_owned(&cccp->ccc_hba_mutex)); 419 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex)); 420 421 GDBG_WAITQ(("ghd_waitq_process_and_mutex_exit: cccp 0x%p\n", 422 (void *)cccp)); 423 424 (void) ghd_waitq_process_and_mutex_hold(cccp); 425 426 /* 427 * Release the mutexes in the opposite order that they 428 * were acquired to prevent requests queued by 429 * ghd_transport() from getting hung up in the wait queue. 430 */ 431 mutex_exit(&cccp->ccc_hba_mutex); 432 mutex_exit(&cccp->ccc_waitq_mutex); 433 } 434