1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/conf.h> 28 #include <sys/ddi.h> 29 #include <sys/sunddi.h> 30 #include <sys/ddi_impldefs.h> 31 #include <sys/cmn_err.h> 32 #include <vm/hat_sfmmu.h> 33 34 #include <sys/iommu.h> 35 #include <sys/iocache.h> 36 #include <sys/sysiosbus.h> 37 38 #include <sys/nexusdebug.h> 39 #include <sys/debug.h> 40 41 #define IOCACHE_REGISTERS_DEBUG 0x1 42 #define IOCACHE_SYNC_DEBUG 0x2 43 #define IOCACHE_DIAG_REG_DEBUG 0x4 44 #define IOCACHE_SYNC_FAIL_DEBUG 0x8 45 46 #define MAX_RETRY 10 47 48 /* Flag which enables the streaming buffer */ 49 int stream_buf_on = 1; 50 /* 51 * This is the number of pages that a mapping request needs before we force 52 * the streaming buffer sync code to use diagnostic registers. This value 53 * was determined through a series of test runs measuring dma mapping 54 * setup performance. 55 */ 56 int stream_buf_sync_using_diag = 36; 57 58 int 59 stream_buf_init(struct sbus_soft_state *softsp, caddr_t address) 60 { 61 uchar_t version; 62 #ifdef DEBUG 63 debug_info = 1; 64 debug_print_level = 0; 65 #endif 66 version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT); 67 version &= 0xf; 68 69 if (stream_buf_on == 0 || version == 0) { 70 softsp->stream_buf_off = STREAM_BUF_OFF; 71 if (version == 0) 72 cmn_err(CE_CONT, "Disabling streaming buffer due to " 73 "SYSIO Rev %d.\n", version); 74 return (DDI_SUCCESS); 75 } 76 77 /* 78 * Simply add each registers offset to the base address 79 * to calculate the already mapped virtual address of 80 * the device register... 81 * 82 * define a macro for the pointer arithmetic; all registers 83 * are 64 bits wide and are defined as uint64_t's. 84 */ 85 86 #define REG_ADDR(b, o) (uint64_t *)((caddr_t)(b) + (o)) 87 88 softsp->str_buf_ctrl_reg = REG_ADDR(address, OFF_STR_BUF_CTRL_REG); 89 softsp->str_buf_flush_reg = REG_ADDR(address, OFF_STR_BUF_FLUSH_REG); 90 softsp->str_buf_sync_reg = REG_ADDR(address, OFF_STR_BUF_SYNC_REG); 91 softsp->str_buf_pg_tag_diag = REG_ADDR(address, STR_BUF_PAGE_TAG_DIAG); 92 93 #undef REG_ADDR 94 95 DPRINTF(IOCACHE_REGISTERS_DEBUG, ("Streaming buffer control reg: 0x%p, " 96 "Streaming buffer flush reg: 0x%p, Streaming buffer sync reg: 0x%p", 97 (void *)softsp->str_buf_ctrl_reg, (void *)softsp->str_buf_flush_reg, 98 (void *)softsp->str_buf_sync_reg)); 99 100 /* Initialize stream buffer sync reg mutex */ 101 mutex_init(&softsp->sync_reg_lock, NULL, MUTEX_DEFAULT, NULL); 102 103 /* Turn on per instance streaming buffer flag */ 104 softsp->stream_buf_off = 0; 105 106 /* Set the hardware registers */ 107 (void) stream_buf_resume_init(softsp); 108 109 return (DDI_SUCCESS); 110 } 111 112 int 113 stream_buf_uninit(struct sbus_soft_state *softsp) 114 { 115 /* Turn off per instance streaming buffer flag */ 116 softsp->stream_buf_off = 1; 117 118 /* Turn off the streaming buffer */ 119 *softsp->str_buf_ctrl_reg = STREAM_BUF_DISABLE; 120 121 return (DDI_SUCCESS); 122 } 123 /* 124 * Initialize stream buf hardware when the system is being resumed. 125 * (Subset of stream_buf_init()) 126 */ 127 int 128 stream_buf_resume_init(struct sbus_soft_state *softsp) 129 { 130 uchar_t version; 131 132 version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT); 133 version &= 0xf; 134 135 if (stream_buf_on == 0 || version == 0) { 136 softsp->stream_buf_off = STREAM_BUF_OFF; 137 return (DDI_SUCCESS); 138 } 139 140 /* Turn on the streaming buffer */ 141 *softsp->str_buf_ctrl_reg = STREAM_BUF_ENABLE; 142 143 return (DDI_SUCCESS); 144 } 145 146 /* 147 * The SYSIO spec says that it will get back to us within 0.5 seconds, 148 * but loaded systems have seen response times over 1.5 seconds. We 149 * err on the side of caution and set the timeout to be 10 seconds. 150 */ 151 #define SCACHE_NSEC_WAIT (10ull * NANOSEC) 152 153 /* 154 * We want to avoid using gethrtime every time we check sync_flag, 155 * so we take SCACHE_SPIN laps before beginning to use gethrtime. 156 */ 157 #define SCACHE_SPIN 10000000 158 159 void 160 sync_stream_buf(struct sbus_soft_state *softsp, ioaddr_t addr, uint_t npages, 161 int *sync_flag, uint64_t phys_sync_flag) 162 { 163 #ifndef lint 164 volatile uint64_t tmp; 165 #endif 166 167 int cntr = 0; 168 169 if (softsp->stream_buf_off != 0) 170 return; 171 172 DPRINTF(IOCACHE_SYNC_DEBUG, ("sync_stream_buf: ioaddr 0x%x, page cnt " 173 "0x%x, sync flag 0x%p, sync flag pf 0x%lx\n", addr, npages, 174 (void *)sync_flag, phys_sync_flag)); 175 176 ASSERT(npages > (uint_t)0); 177 178 /* Acquire the sync lock */ 179 mutex_enter(&softsp->sync_reg_lock); 180 181 *sync_flag = 0; 182 183 if (npages > stream_buf_sync_using_diag) { 184 int i; 185 volatile uint64_t *reg_addr; 186 uint64_t reg; 187 uint_t ioaddr; 188 uint_t hiaddr = addr + (npages * IOMMU_PAGESIZE); 189 int do_sync = 0; 190 191 for (i = 0, reg_addr = softsp->str_buf_pg_tag_diag; 192 i < STREAM_CACHE_LINES; i++, reg_addr++) { 193 194 /* Read the page tag diag reg */ 195 reg = *reg_addr; 196 #ifdef DEBUG 197 { 198 uint_t hi, lo; 199 hi = (uint_t)(reg >> 32); 200 lo = (uint_t)(reg & 0xffffffff); 201 DPRINTF(IOCACHE_DIAG_REG_DEBUG, 202 ("IO cache line diag " 203 "reg addr 0x%p, hi0x%x lo0x%x\n", 204 (void *)reg_addr, hi, lo)); 205 } 206 #endif /* DEBUG */ 207 /* Check for a valid line */ 208 if (reg & STR_PG_VALID) { 209 ioaddr = (uint_t)reg << STR_PG_SHIFT; 210 211 DPRINTF(IOCACHE_DIAG_REG_DEBUG, ("ioaddr 0x%x, " 212 "range base 0x%x, range extent 0x%x\n", 213 ioaddr, addr, 214 addr + (npages * IOMMU_PAGESIZE))); 215 if (ioaddr >= addr && ioaddr <= hiaddr) { 216 *softsp->str_buf_flush_reg = (uint64_t) 217 ioaddr; 218 do_sync = 1; 219 } 220 } 221 } 222 223 if (!do_sync) { 224 mutex_exit(&softsp->sync_reg_lock); 225 return; 226 } 227 } else { 228 do { 229 *softsp->str_buf_flush_reg = (uint64_t)addr; 230 addr += IOMMU_PAGESIZE; 231 npages--; 232 } while (npages > (uint_t)0); 233 } 234 235 /* Ask the hardware to flag when the flush is complete */ 236 *softsp->str_buf_sync_reg = phys_sync_flag; 237 238 #ifndef lint 239 /* 240 * Due to the sun4u memory models, this noncached load will sync the 241 * order of all prior loads and stores regardless of cacheability. 242 * No membar_stst() is needed after zeroing the flush sync flag. 243 */ 244 tmp = *softsp->sbus_ctrl_reg; 245 #endif 246 247 /* 248 * Begin spinning on the hardware sync register. We'll spin for 249 * a while (SCACHE_SPIN) before using gethrtime, but once that time 250 * is up we'll drop into an inner loop where we use gethrtime on 251 * every iteration. Once SCACHE_NSEC_WAIT nanoseconds have 252 * elapsed, we'll assume a Bad Thing has happened and toss. 253 */ 254 while (!*((volatile int *)sync_flag)) { 255 if (cntr++ == SCACHE_SPIN) { 256 /* 257 * If we're here, then we've spun long enough 258 * to justify use of gethrtime each iteration. 259 */ 260 hrtime_t nsec_start, nsectowait, nsec_current; 261 nsectowait = SCACHE_NSEC_WAIT; 262 nsec_current = nsec_start = gethrtime(); 263 264 while (!*((volatile int *)sync_flag)) { 265 /* 266 * Double check the sync flag again before 267 * we panic in case we get preempted. 268 * See bugid 4126896 269 */ 270 nsec_current = gethrtime(); 271 if ((nsec_current - nsec_start) > nsectowait && 272 !*((volatile int *)sync_flag)) { 273 /* 274 * Trouble. The SYSIO chip has 275 * seemingly gone AWOL. Vomit. 276 */ 277 panic("streaming buffer timed out"); 278 } 279 } 280 } 281 } 282 283 /* Finally, drop the sync lock */ 284 mutex_exit(&softsp->sync_reg_lock); 285 } 286