1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/conf.h> 31 #include <sys/ddi.h> 32 #include <sys/sunddi.h> 33 #include <sys/ddi_impldefs.h> 34 #include <sys/cmn_err.h> 35 #include <vm/hat_sfmmu.h> 36 37 #include <sys/iommu.h> 38 #include <sys/iocache.h> 39 #include <sys/sysiosbus.h> 40 41 #include <sys/nexusdebug.h> 42 #include <sys/debug.h> 43 44 #define IOCACHE_REGISTERS_DEBUG 0x1 45 #define IOCACHE_SYNC_DEBUG 0x2 46 #define IOCACHE_DIAG_REG_DEBUG 0x4 47 #define IOCACHE_SYNC_FAIL_DEBUG 0x8 48 49 #define MAX_RETRY 10 50 51 /* Flag which enables the streaming buffer */ 52 int stream_buf_on = 1; 53 /* 54 * This is the number of pages that a mapping request needs before we force 55 * the streaming buffer sync code to use diagnostic registers. This value 56 * was determined through a series of test runs measuring dma mapping 57 * setup performance. 58 */ 59 int stream_buf_sync_using_diag = 36; 60 61 int 62 stream_buf_init(struct sbus_soft_state *softsp, caddr_t address) 63 { 64 uchar_t version; 65 #ifdef DEBUG 66 debug_info = 1; 67 debug_print_level = 0; 68 #endif 69 version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT); 70 version &= 0xf; 71 72 if (stream_buf_on == 0 || version == 0) { 73 softsp->stream_buf_off = STREAM_BUF_OFF; 74 if (version == 0) 75 cmn_err(CE_CONT, "Disabling streaming buffer due to " 76 "SYSIO Rev %d.\n", version); 77 return (DDI_SUCCESS); 78 } 79 80 /* 81 * Simply add each registers offset to the base address 82 * to calculate the already mapped virtual address of 83 * the device register... 84 * 85 * define a macro for the pointer arithmetic; all registers 86 * are 64 bits wide and are defined as uint64_t's. 87 */ 88 89 #define REG_ADDR(b, o) (uint64_t *)((caddr_t)(b) + (o)) 90 91 softsp->str_buf_ctrl_reg = REG_ADDR(address, OFF_STR_BUF_CTRL_REG); 92 softsp->str_buf_flush_reg = REG_ADDR(address, OFF_STR_BUF_FLUSH_REG); 93 softsp->str_buf_sync_reg = REG_ADDR(address, OFF_STR_BUF_SYNC_REG); 94 softsp->str_buf_pg_tag_diag = REG_ADDR(address, STR_BUF_PAGE_TAG_DIAG); 95 96 #undef REG_ADDR 97 98 DPRINTF(IOCACHE_REGISTERS_DEBUG, ("Streaming buffer control reg: 0x%p, " 99 "Streaming buffer flush reg: 0x%p, Streaming buffer sync reg: 0x%p", 100 softsp->str_buf_ctrl_reg, softsp->str_buf_flush_reg, 101 softsp->str_buf_sync_reg)); 102 103 /* Initialize stream buffer sync reg mutex */ 104 mutex_init(&softsp->sync_reg_lock, NULL, MUTEX_DEFAULT, NULL); 105 106 /* Turn on per instance streaming buffer flag */ 107 softsp->stream_buf_off = 0; 108 109 /* Set the hardware registers */ 110 (void) stream_buf_resume_init(softsp); 111 112 return (DDI_SUCCESS); 113 } 114 115 int 116 stream_buf_uninit(struct sbus_soft_state *softsp) 117 { 118 /* Turn off per instance streaming buffer flag */ 119 softsp->stream_buf_off = 1; 120 121 /* Turn off the streaming buffer */ 122 *softsp->str_buf_ctrl_reg = STREAM_BUF_DISABLE; 123 124 return (DDI_SUCCESS); 125 } 126 /* 127 * Initialize stream buf hardware when the system is being resumed. 128 * (Subset of stream_buf_init()) 129 */ 130 int 131 stream_buf_resume_init(struct sbus_soft_state *softsp) 132 { 133 uchar_t version; 134 135 version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT); 136 version &= 0xf; 137 138 if (stream_buf_on == 0 || version == 0) { 139 softsp->stream_buf_off = STREAM_BUF_OFF; 140 return (DDI_SUCCESS); 141 } 142 143 /* Turn on the streaming buffer */ 144 *softsp->str_buf_ctrl_reg = STREAM_BUF_ENABLE; 145 146 return (DDI_SUCCESS); 147 } 148 149 /* 150 * The SYSIO spec says that it will get back to us within 0.5 seconds, 151 * but loaded systems have seen response times over 1.5 seconds. We 152 * err on the side of caution and set the timeout to be 10 seconds. 153 */ 154 #define SCACHE_NSEC_WAIT (10ull * NANOSEC) 155 156 /* 157 * We want to avoid using gethrtime every time we check sync_flag, 158 * so we take SCACHE_SPIN laps before beginning to use gethrtime. 159 */ 160 #define SCACHE_SPIN 10000000 161 162 void 163 sync_stream_buf(struct sbus_soft_state *softsp, ioaddr_t addr, uint_t npages, 164 int *sync_flag, uint64_t phys_sync_flag) 165 { 166 #ifndef lint 167 volatile uint64_t tmp; 168 #endif 169 170 int cntr = 0; 171 172 if (softsp->stream_buf_off != 0) 173 return; 174 175 DPRINTF(IOCACHE_SYNC_DEBUG, ("sync_stream_buf: ioaddr 0x%x, page cnt " 176 "0x%x, sync flag 0x%p, sync flag pf 0x%lx\n", addr, npages, 177 sync_flag, phys_sync_flag)); 178 179 ASSERT(npages > (uint_t)0); 180 181 /* Acquire the sync lock */ 182 mutex_enter(&softsp->sync_reg_lock); 183 184 *sync_flag = 0; 185 186 if (npages > stream_buf_sync_using_diag) { 187 int i; 188 volatile uint64_t *reg_addr; 189 uint64_t reg; 190 uint_t ioaddr; 191 uint_t hiaddr = addr + (npages * IOMMU_PAGESIZE); 192 int do_sync = 0; 193 194 for (i = 0, reg_addr = softsp->str_buf_pg_tag_diag; 195 i < STREAM_CACHE_LINES; i++, reg_addr++) { 196 197 /* Read the page tag diag reg */ 198 reg = *reg_addr; 199 #ifdef DEBUG 200 { 201 uint_t hi, lo; 202 hi = (uint_t)(reg >> 32); 203 lo = (uint_t)(reg & 0xffffffff); 204 DPRINTF(IOCACHE_DIAG_REG_DEBUG, 205 ("IO cache line diag " 206 "reg addr 0x%p, hi0x%x lo0x%x\n", 207 reg_addr, hi, lo)); 208 } 209 #endif /* DEBUG */ 210 /* Check for a valid line */ 211 if (reg & STR_PG_VALID) { 212 ioaddr = (uint_t)reg << STR_PG_SHIFT; 213 214 DPRINTF(IOCACHE_DIAG_REG_DEBUG, ("ioaddr 0x%x, " 215 "range base 0x%x, range extent 0x%x\n", 216 ioaddr, addr, 217 addr + (npages * IOMMU_PAGESIZE))); 218 if (ioaddr >= addr && ioaddr <= hiaddr) { 219 *softsp->str_buf_flush_reg = (uint64_t) 220 ioaddr; 221 do_sync = 1; 222 } 223 } 224 } 225 226 if (!do_sync) { 227 mutex_exit(&softsp->sync_reg_lock); 228 return; 229 } 230 } else { 231 do { 232 *softsp->str_buf_flush_reg = (uint64_t)addr; 233 addr += IOMMU_PAGESIZE; 234 npages--; 235 } while (npages > (uint_t)0); 236 } 237 238 /* Ask the hardware to flag when the flush is complete */ 239 *softsp->str_buf_sync_reg = phys_sync_flag; 240 241 #ifndef lint 242 /* 243 * Due to the sun4u memory models, this noncached load will sync the 244 * order of all prior loads and stores regardless of cacheability. 245 * No membar_stst() is needed after zeroing the flush sync flag. 246 */ 247 tmp = *softsp->sbus_ctrl_reg; 248 #endif 249 250 /* 251 * Begin spinning on the hardware sync register. We'll spin for 252 * a while (SCACHE_SPIN) before using gethrtime, but once that time 253 * is up we'll drop into an inner loop where we use gethrtime on 254 * every iteration. Once SCACHE_NSEC_WAIT nanoseconds have 255 * elapsed, we'll assume a Bad Thing has happened and toss. 256 */ 257 while (!*((volatile int *)sync_flag)) { 258 if (cntr++ == SCACHE_SPIN) { 259 /* 260 * If we're here, then we've spun long enough 261 * to justify use of gethrtime each iteration. 262 */ 263 hrtime_t nsec_start, nsectowait, nsec_current; 264 nsectowait = SCACHE_NSEC_WAIT; 265 nsec_current = nsec_start = gethrtime(); 266 267 while (!*((volatile int *)sync_flag)) { 268 /* 269 * Double check the sync flag again before 270 * we panic in case we get preempted. 271 * See bugid 4126896 272 */ 273 nsec_current = gethrtime(); 274 if ((nsec_current - nsec_start) > nsectowait && 275 !*((volatile int *)sync_flag)) { 276 /* 277 * Trouble. The SYSIO chip has 278 * seemingly gone AWOL. Vomit. 279 */ 280 panic("streaming buffer timed out"); 281 } 282 } 283 } 284 } 285 286 /* Finally, drop the sync lock */ 287 mutex_exit(&softsp->sync_reg_lock); 288 } 289