1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/types.h>
27 #include <sys/conf.h>
28 #include <sys/ddi.h>
29 #include <sys/sunddi.h>
30 #include <sys/ddi_impldefs.h>
31 #include <sys/cmn_err.h>
32 #include <vm/hat_sfmmu.h>
33
34 #include <sys/iommu.h>
35 #include <sys/iocache.h>
36 #include <sys/sysiosbus.h>
37
38 #include <sys/nexusdebug.h>
39 #include <sys/debug.h>
40
41 #define IOCACHE_REGISTERS_DEBUG 0x1
42 #define IOCACHE_SYNC_DEBUG 0x2
43 #define IOCACHE_DIAG_REG_DEBUG 0x4
44 #define IOCACHE_SYNC_FAIL_DEBUG 0x8
45
46 #define MAX_RETRY 10
47
48 /* Flag which enables the streaming buffer */
49 int stream_buf_on = 1;
50 /*
51 * This is the number of pages that a mapping request needs before we force
52 * the streaming buffer sync code to use diagnostic registers. This value
53 * was determined through a series of test runs measuring dma mapping
54 * setup performance.
55 */
56 int stream_buf_sync_using_diag = 36;
57
58 int
stream_buf_init(struct sbus_soft_state * softsp,caddr_t address)59 stream_buf_init(struct sbus_soft_state *softsp, caddr_t address)
60 {
61 uchar_t version;
62 #ifdef DEBUG
63 debug_info = 1;
64 debug_print_level = 0;
65 #endif
66 version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT);
67 version &= 0xf;
68
69 if (stream_buf_on == 0 || version == 0) {
70 softsp->stream_buf_off = STREAM_BUF_OFF;
71 if (version == 0)
72 cmn_err(CE_CONT, "Disabling streaming buffer due to "
73 "SYSIO Rev %d.\n", version);
74 return (DDI_SUCCESS);
75 }
76
77 /*
78 * Simply add each registers offset to the base address
79 * to calculate the already mapped virtual address of
80 * the device register...
81 *
82 * define a macro for the pointer arithmetic; all registers
83 * are 64 bits wide and are defined as uint64_t's.
84 */
85
86 #define REG_ADDR(b, o) (uint64_t *)((caddr_t)(b) + (o))
87
88 softsp->str_buf_ctrl_reg = REG_ADDR(address, OFF_STR_BUF_CTRL_REG);
89 softsp->str_buf_flush_reg = REG_ADDR(address, OFF_STR_BUF_FLUSH_REG);
90 softsp->str_buf_sync_reg = REG_ADDR(address, OFF_STR_BUF_SYNC_REG);
91 softsp->str_buf_pg_tag_diag = REG_ADDR(address, STR_BUF_PAGE_TAG_DIAG);
92
93 #undef REG_ADDR
94
95 DPRINTF(IOCACHE_REGISTERS_DEBUG, ("Streaming buffer control reg: 0x%p, "
96 "Streaming buffer flush reg: 0x%p, Streaming buffer sync reg: 0x%p",
97 (void *)softsp->str_buf_ctrl_reg, (void *)softsp->str_buf_flush_reg,
98 (void *)softsp->str_buf_sync_reg));
99
100 /* Initialize stream buffer sync reg mutex */
101 mutex_init(&softsp->sync_reg_lock, NULL, MUTEX_DEFAULT, NULL);
102
103 /* Turn on per instance streaming buffer flag */
104 softsp->stream_buf_off = 0;
105
106 /* Set the hardware registers */
107 (void) stream_buf_resume_init(softsp);
108
109 return (DDI_SUCCESS);
110 }
111
112 int
stream_buf_uninit(struct sbus_soft_state * softsp)113 stream_buf_uninit(struct sbus_soft_state *softsp)
114 {
115 /* Turn off per instance streaming buffer flag */
116 softsp->stream_buf_off = 1;
117
118 /* Turn off the streaming buffer */
119 *softsp->str_buf_ctrl_reg = STREAM_BUF_DISABLE;
120
121 return (DDI_SUCCESS);
122 }
123 /*
124 * Initialize stream buf hardware when the system is being resumed.
125 * (Subset of stream_buf_init())
126 */
127 int
stream_buf_resume_init(struct sbus_soft_state * softsp)128 stream_buf_resume_init(struct sbus_soft_state *softsp)
129 {
130 uchar_t version;
131
132 version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT);
133 version &= 0xf;
134
135 if (stream_buf_on == 0 || version == 0) {
136 softsp->stream_buf_off = STREAM_BUF_OFF;
137 return (DDI_SUCCESS);
138 }
139
140 /* Turn on the streaming buffer */
141 *softsp->str_buf_ctrl_reg = STREAM_BUF_ENABLE;
142
143 return (DDI_SUCCESS);
144 }
145
146 /*
147 * The SYSIO spec says that it will get back to us within 0.5 seconds,
148 * but loaded systems have seen response times over 1.5 seconds. We
149 * err on the side of caution and set the timeout to be 10 seconds.
150 */
151 #define SCACHE_NSEC_WAIT (10ull * NANOSEC)
152
153 /*
154 * We want to avoid using gethrtime every time we check sync_flag,
155 * so we take SCACHE_SPIN laps before beginning to use gethrtime.
156 */
157 #define SCACHE_SPIN 10000000
158
159 void
sync_stream_buf(struct sbus_soft_state * softsp,ioaddr_t addr,uint_t npages,int * sync_flag,uint64_t phys_sync_flag)160 sync_stream_buf(struct sbus_soft_state *softsp, ioaddr_t addr, uint_t npages,
161 int *sync_flag, uint64_t phys_sync_flag)
162 {
163 #ifndef lint
164 volatile uint64_t tmp;
165 #endif
166
167 int cntr = 0;
168
169 if (softsp->stream_buf_off != 0)
170 return;
171
172 DPRINTF(IOCACHE_SYNC_DEBUG, ("sync_stream_buf: ioaddr 0x%x, page cnt "
173 "0x%x, sync flag 0x%p, sync flag pf 0x%lx\n", addr, npages,
174 (void *)sync_flag, phys_sync_flag));
175
176 ASSERT(npages > (uint_t)0);
177
178 /* Acquire the sync lock */
179 mutex_enter(&softsp->sync_reg_lock);
180
181 *sync_flag = 0;
182
183 if (npages > stream_buf_sync_using_diag) {
184 int i;
185 volatile uint64_t *reg_addr;
186 uint64_t reg;
187 uint_t ioaddr;
188 uint_t hiaddr = addr + (npages * IOMMU_PAGESIZE);
189 int do_sync = 0;
190
191 for (i = 0, reg_addr = softsp->str_buf_pg_tag_diag;
192 i < STREAM_CACHE_LINES; i++, reg_addr++) {
193
194 /* Read the page tag diag reg */
195 reg = *reg_addr;
196 #ifdef DEBUG
197 {
198 uint_t hi, lo;
199 hi = (uint_t)(reg >> 32);
200 lo = (uint_t)(reg & 0xffffffff);
201 DPRINTF(IOCACHE_DIAG_REG_DEBUG,
202 ("IO cache line diag "
203 "reg addr 0x%p, hi0x%x lo0x%x\n",
204 (void *)reg_addr, hi, lo));
205 }
206 #endif /* DEBUG */
207 /* Check for a valid line */
208 if (reg & STR_PG_VALID) {
209 ioaddr = (uint_t)reg << STR_PG_SHIFT;
210
211 DPRINTF(IOCACHE_DIAG_REG_DEBUG, ("ioaddr 0x%x, "
212 "range base 0x%x, range extent 0x%x\n",
213 ioaddr, addr,
214 addr + (npages * IOMMU_PAGESIZE)));
215 if (ioaddr >= addr && ioaddr <= hiaddr) {
216 *softsp->str_buf_flush_reg = (uint64_t)
217 ioaddr;
218 do_sync = 1;
219 }
220 }
221 }
222
223 if (!do_sync) {
224 mutex_exit(&softsp->sync_reg_lock);
225 return;
226 }
227 } else {
228 do {
229 *softsp->str_buf_flush_reg = (uint64_t)addr;
230 addr += IOMMU_PAGESIZE;
231 npages--;
232 } while (npages > (uint_t)0);
233 }
234
235 /* Ask the hardware to flag when the flush is complete */
236 *softsp->str_buf_sync_reg = phys_sync_flag;
237
238 #ifndef lint
239 /*
240 * Due to the sun4u memory models, this noncached load will sync the
241 * order of all prior loads and stores regardless of cacheability.
242 * No membar_stst() is needed after zeroing the flush sync flag.
243 */
244 tmp = *softsp->sbus_ctrl_reg;
245 #endif
246
247 /*
248 * Begin spinning on the hardware sync register. We'll spin for
249 * a while (SCACHE_SPIN) before using gethrtime, but once that time
250 * is up we'll drop into an inner loop where we use gethrtime on
251 * every iteration. Once SCACHE_NSEC_WAIT nanoseconds have
252 * elapsed, we'll assume a Bad Thing has happened and toss.
253 */
254 while (!*((volatile int *)sync_flag)) {
255 if (cntr++ == SCACHE_SPIN) {
256 /*
257 * If we're here, then we've spun long enough
258 * to justify use of gethrtime each iteration.
259 */
260 hrtime_t nsec_start, nsectowait, nsec_current;
261 nsectowait = SCACHE_NSEC_WAIT;
262 nsec_current = nsec_start = gethrtime();
263
264 while (!*((volatile int *)sync_flag)) {
265 /*
266 * Double check the sync flag again before
267 * we panic in case we get preempted.
268 * See bugid 4126896
269 */
270 nsec_current = gethrtime();
271 if ((nsec_current - nsec_start) > nsectowait &&
272 !*((volatile int *)sync_flag)) {
273 /*
274 * Trouble. The SYSIO chip has
275 * seemingly gone AWOL. Vomit.
276 */
277 panic("streaming buffer timed out");
278 }
279 }
280 }
281 }
282
283 /* Finally, drop the sync lock */
284 mutex_exit(&softsp->sync_reg_lock);
285 }
286