1 /******************************************************************************
2
3 � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc.
4 All rights reserved.
5
6 This is proprietary source code of Freescale Semiconductor Inc.,
7 and its use is subject to the NetComm Device Drivers EULA.
8 The copyright notice above does not evidence any actual or intended
9 publication of such source code.
10
11 ALTERNATIVELY, redistribution and use in source and binary forms, with
12 or without modification, are permitted provided that the following
13 conditions are met:
14 * Redistributions of source code must retain the above copyright
15 notice, this list of conditions and the following disclaimer.
16 * Redistributions in binary form must reproduce the above copyright
17 notice, this list of conditions and the following disclaimer in the
18 documentation and/or other materials provided with the distribution.
19 * Neither the name of Freescale Semiconductor nor the
20 names of its contributors may be used to endorse or promote products
21 derived from this software without specific prior written permission.
22
23 THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
24 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26 DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
27 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
30 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34
35 **************************************************************************/
36 /******************************************************************************
37 @File bman_low.c
38
39 @Description BM low-level implementation
40 *//***************************************************************************/
41 #include <sys/cdefs.h>
42 #include <sys/types.h>
43 #include <machine/atomic.h>
44
45 #include "std_ext.h"
46 #include "core_ext.h"
47 #include "xx_ext.h"
48 #include "error_ext.h"
49
50 #include "bman_private.h"
51
52
53 /***************************/
54 /* Portal register assists */
55 /***************************/
56
57 /* Cache-inhibited register offsets */
58 #define REG_RCR_PI_CINH 0x0000
59 #define REG_RCR_CI_CINH 0x0004
60 #define REG_RCR_ITR 0x0008
61 #define REG_CFG 0x0100
62 #define REG_SCN(n) (0x0200 + ((n) << 2))
63 #define REG_ISR 0x0e00
64 #define REG_IER 0x0e04
65 #define REG_ISDR 0x0e08
66 #define REG_IIR 0x0e0c
67
68 /* Cache-enabled register offsets */
69 #define CL_CR 0x0000
70 #define CL_RR0 0x0100
71 #define CL_RR1 0x0140
72 #define CL_RCR 0x1000
73 #define CL_RCR_PI_CENA 0x3000
74 #define CL_RCR_CI_CENA 0x3100
75
76 /* The h/w design requires mappings to be size-aligned so that "add"s can be
77 * reduced to "or"s. The primitives below do the same for s/w. */
78
ptr_ADD(void * a,uintptr_t b)79 static __inline__ void *ptr_ADD(void *a, uintptr_t b)
80 {
81 return (void *)((uintptr_t)a + b);
82 }
83
84 /* Bitwise-OR two pointers */
ptr_OR(void * a,uintptr_t b)85 static __inline__ void *ptr_OR(void *a, uintptr_t b)
86 {
87 return (void *)((uintptr_t)a | b);
88 }
89
90 /* Cache-inhibited register access */
__bm_in(struct bm_addr * bm,uintptr_t offset)91 static __inline__ uint32_t __bm_in(struct bm_addr *bm, uintptr_t offset)
92 {
93 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
94 return GET_UINT32(*tmp);
95 }
__bm_out(struct bm_addr * bm,uintptr_t offset,uint32_t val)96 static __inline__ void __bm_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
97 {
98 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
99 WRITE_UINT32(*tmp, val);
100 }
101 #define bm_in(reg) __bm_in(&portal->addr, REG_##reg)
102 #define bm_out(reg, val) __bm_out(&portal->addr, REG_##reg, val)
103
104 /* Convert 'n' cachelines to a pointer value for bitwise OR */
105 #define bm_cl(n) (void *)((n) << 6)
106
107 /* Cache-enabled (index) register access */
__bm_cl_touch_ro(struct bm_addr * bm,uintptr_t offset)108 static __inline__ void __bm_cl_touch_ro(struct bm_addr *bm, uintptr_t offset)
109 {
110 dcbt_ro(ptr_ADD(bm->addr_ce, offset));
111 }
__bm_cl_touch_rw(struct bm_addr * bm,uintptr_t offset)112 static __inline__ void __bm_cl_touch_rw(struct bm_addr *bm, uintptr_t offset)
113 {
114 dcbt_rw(ptr_ADD(bm->addr_ce, offset));
115 }
__bm_cl_in(struct bm_addr * bm,uintptr_t offset)116 static __inline__ uint32_t __bm_cl_in(struct bm_addr *bm, uintptr_t offset)
117 {
118 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
119 return GET_UINT32(*tmp);
120 }
__bm_cl_out(struct bm_addr * bm,uintptr_t offset,uint32_t val)121 static __inline__ void __bm_cl_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
122 {
123 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
124 WRITE_UINT32(*tmp, val);
125 dcbf(tmp);
126 }
__bm_cl_invalidate(struct bm_addr * bm,uintptr_t offset)127 static __inline__ void __bm_cl_invalidate(struct bm_addr *bm, uintptr_t offset)
128 {
129 dcbi(ptr_ADD(bm->addr_ce, offset));
130 }
131 #define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, CL_##reg##_CENA)
132 #define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, CL_##reg##_CENA)
133 #define bm_cl_in(reg) __bm_cl_in(&portal->addr, CL_##reg##_CENA)
134 #define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, CL_##reg##_CENA, val)
135 #define bm_cl_invalidate(reg) __bm_cl_invalidate(&portal->addr, CL_##reg##_CENA)
136
137 /* Cyclic helper for rings. TODO: once we are able to do fine-grain perf
138 * analysis, look at using the "extra" bit in the ring index registers to avoid
139 * cyclic issues. */
cyc_diff(uint8_t ringsize,uint8_t first,uint8_t last)140 static __inline__ uint8_t cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
141 {
142 /* 'first' is included, 'last' is excluded */
143 if (first <= last)
144 return (uint8_t)(last - first);
145 return (uint8_t)(ringsize + last - first);
146 }
147
148 /* --------------- */
149 /* --- RCR API --- */
150
151 /* It's safer to code in terms of the 'rcr' object than the 'portal' object,
152 * because the latter runs the risk of copy-n-paste errors from other code where
153 * we could manipulate some other structure within 'portal'. */
154 /* #define RCR_API_START() register struct bm_rcr *rcr = &portal->rcr */
155
156 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
157 #define RCR_CARRYCLEAR(p) \
158 (void *)((uintptr_t)(p) & (~(uintptr_t)(BM_RCR_SIZE << 6)))
159
160 /* Bit-wise logic to convert a ring pointer to a ring index */
RCR_PTR2IDX(struct bm_rcr_entry * e)161 static __inline__ uint8_t RCR_PTR2IDX(struct bm_rcr_entry *e)
162 {
163 return (uint8_t)(((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1));
164 }
165
166 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
RCR_INC(struct bm_rcr * rcr)167 static __inline__ void RCR_INC(struct bm_rcr *rcr)
168 {
169 /* NB: this is odd-looking, but experiments show that it generates
170 * fast code with essentially no branching overheads. We increment to
171 * the next RCR pointer and handle overflow and 'vbit'. */
172 struct bm_rcr_entry *partial = rcr->cursor + 1;
173 rcr->cursor = RCR_CARRYCLEAR(partial);
174 if (partial != rcr->cursor)
175 rcr->vbit ^= BM_RCR_VERB_VBIT;
176 }
177
bm_rcr_init(struct bm_portal * portal,e_BmPortalProduceMode pmode,e_BmPortalRcrConsumeMode cmode)178 t_Error bm_rcr_init(struct bm_portal *portal,
179 e_BmPortalProduceMode pmode,
180 e_BmPortalRcrConsumeMode cmode)
181 {
182 register struct bm_rcr *rcr = &portal->rcr;
183 uint32_t cfg;
184 uint8_t pi;
185
186 rcr->ring = ptr_ADD(portal->addr.addr_ce, CL_RCR);
187 rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
188 pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
189 rcr->cursor = rcr->ring + pi;
190 rcr->vbit = (uint8_t)((bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0);
191 rcr->available = (uint8_t)(BM_RCR_SIZE - 1 - cyc_diff(BM_RCR_SIZE, rcr->ci, pi));
192 rcr->ithresh = (uint8_t)bm_in(RCR_ITR);
193 #ifdef BM_CHECKING
194 rcr->busy = 0;
195 rcr->pmode = pmode;
196 rcr->cmode = cmode;
197 #else
198 UNUSED(cmode);
199 #endif /* BM_CHECKING */
200 cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
201 bm_out(CFG, cfg);
202 return 0;
203 }
204
bm_rcr_finish(struct bm_portal * portal)205 void bm_rcr_finish(struct bm_portal *portal)
206 {
207 register struct bm_rcr *rcr = &portal->rcr;
208 uint8_t pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
209 uint8_t ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
210 ASSERT_COND(!rcr->busy);
211 if (pi != RCR_PTR2IDX(rcr->cursor))
212 REPORT_ERROR(WARNING, E_INVALID_STATE, ("losing uncommitted RCR entries"));
213 if (ci != rcr->ci)
214 REPORT_ERROR(WARNING, E_INVALID_STATE, ("missing existing RCR completions"));
215 if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
216 REPORT_ERROR(WARNING, E_INVALID_STATE, ("RCR destroyed unquiesced"));
217 }
218
bm_rcr_start(struct bm_portal * portal)219 struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
220 {
221 register struct bm_rcr *rcr = &portal->rcr;
222 ASSERT_COND(!rcr->busy);
223 if (!rcr->available)
224 return NULL;
225 #ifdef BM_CHECKING
226 rcr->busy = 1;
227 #endif /* BM_CHECKING */
228 dcbz_64(rcr->cursor);
229 return rcr->cursor;
230 }
231
bm_rcr_abort(struct bm_portal * portal)232 void bm_rcr_abort(struct bm_portal *portal)
233 {
234 register struct bm_rcr *rcr = &portal->rcr;
235 ASSERT_COND(rcr->busy);
236 #ifdef BM_CHECKING
237 rcr->busy = 0;
238 #else
239 UNUSED(rcr);
240 #endif /* BM_CHECKING */
241 }
242
bm_rcr_pend_and_next(struct bm_portal * portal,uint8_t myverb)243 struct bm_rcr_entry *bm_rcr_pend_and_next(struct bm_portal *portal, uint8_t myverb)
244 {
245 register struct bm_rcr *rcr = &portal->rcr;
246 ASSERT_COND(rcr->busy);
247 ASSERT_COND(rcr->pmode != e_BmPortalPVB);
248 if (rcr->available == 1)
249 return NULL;
250 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
251 dcbf_64(rcr->cursor);
252 RCR_INC(rcr);
253 rcr->available--;
254 dcbz_64(rcr->cursor);
255 return rcr->cursor;
256 }
257
bm_rcr_pci_commit(struct bm_portal * portal,uint8_t myverb)258 void bm_rcr_pci_commit(struct bm_portal *portal, uint8_t myverb)
259 {
260 register struct bm_rcr *rcr = &portal->rcr;
261 ASSERT_COND(rcr->busy);
262 ASSERT_COND(rcr->pmode == e_BmPortalPCI);
263 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
264 RCR_INC(rcr);
265 rcr->available--;
266 mb();
267 bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
268 #ifdef BM_CHECKING
269 rcr->busy = 0;
270 #endif /* BM_CHECKING */
271 }
272
bm_rcr_pce_prefetch(struct bm_portal * portal)273 void bm_rcr_pce_prefetch(struct bm_portal *portal)
274 {
275 ASSERT_COND(((struct bm_rcr *)&portal->rcr)->pmode == e_BmPortalPCE);
276 bm_cl_invalidate(RCR_PI);
277 bm_cl_touch_rw(RCR_PI);
278 }
279
bm_rcr_pce_commit(struct bm_portal * portal,uint8_t myverb)280 void bm_rcr_pce_commit(struct bm_portal *portal, uint8_t myverb)
281 {
282 register struct bm_rcr *rcr = &portal->rcr;
283 ASSERT_COND(rcr->busy);
284 ASSERT_COND(rcr->pmode == e_BmPortalPCE);
285 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
286 RCR_INC(rcr);
287 rcr->available--;
288 wmb();
289 bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
290 #ifdef BM_CHECKING
291 rcr->busy = 0;
292 #endif /* BM_CHECKING */
293 }
294
bm_rcr_pvb_commit(struct bm_portal * portal,uint8_t myverb)295 void bm_rcr_pvb_commit(struct bm_portal *portal, uint8_t myverb)
296 {
297 register struct bm_rcr *rcr = &portal->rcr;
298 struct bm_rcr_entry *rcursor;
299 ASSERT_COND(rcr->busy);
300 ASSERT_COND(rcr->pmode == e_BmPortalPVB);
301 rmb();
302 rcursor = rcr->cursor;
303 rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
304 dcbf_64(rcursor);
305 RCR_INC(rcr);
306 rcr->available--;
307 #ifdef BM_CHECKING
308 rcr->busy = 0;
309 #endif /* BM_CHECKING */
310 }
311
312
bm_rcr_cci_update(struct bm_portal * portal)313 uint8_t bm_rcr_cci_update(struct bm_portal *portal)
314 {
315 register struct bm_rcr *rcr = &portal->rcr;
316 uint8_t diff, old_ci = rcr->ci;
317 ASSERT_COND(rcr->cmode == e_BmPortalRcrCCI);
318 rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
319 diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
320 rcr->available += diff;
321 return diff;
322 }
323
324
bm_rcr_cce_prefetch(struct bm_portal * portal)325 void bm_rcr_cce_prefetch(struct bm_portal *portal)
326 {
327 ASSERT_COND(((struct bm_rcr *)&portal->rcr)->cmode == e_BmPortalRcrCCE);
328 bm_cl_touch_ro(RCR_CI);
329 }
330
331
bm_rcr_cce_update(struct bm_portal * portal)332 uint8_t bm_rcr_cce_update(struct bm_portal *portal)
333 {
334 register struct bm_rcr *rcr = &portal->rcr;
335 uint8_t diff, old_ci = rcr->ci;
336 ASSERT_COND(rcr->cmode == e_BmPortalRcrCCE);
337 rcr->ci = (uint8_t)(bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1));
338 bm_cl_invalidate(RCR_CI);
339 diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
340 rcr->available += diff;
341 return diff;
342 }
343
344
bm_rcr_get_ithresh(struct bm_portal * portal)345 uint8_t bm_rcr_get_ithresh(struct bm_portal *portal)
346 {
347 register struct bm_rcr *rcr = &portal->rcr;
348 return rcr->ithresh;
349 }
350
351
bm_rcr_set_ithresh(struct bm_portal * portal,uint8_t ithresh)352 void bm_rcr_set_ithresh(struct bm_portal *portal, uint8_t ithresh)
353 {
354 register struct bm_rcr *rcr = &portal->rcr;
355 rcr->ithresh = ithresh;
356 bm_out(RCR_ITR, ithresh);
357 }
358
359
bm_rcr_get_avail(struct bm_portal * portal)360 uint8_t bm_rcr_get_avail(struct bm_portal *portal)
361 {
362 register struct bm_rcr *rcr = &portal->rcr;
363 return rcr->available;
364 }
365
366
bm_rcr_get_fill(struct bm_portal * portal)367 uint8_t bm_rcr_get_fill(struct bm_portal *portal)
368 {
369 register struct bm_rcr *rcr = &portal->rcr;
370 return (uint8_t)(BM_RCR_SIZE - 1 - rcr->available);
371 }
372
373
374 /* ------------------------------ */
375 /* --- Management command API --- */
376
377 /* It's safer to code in terms of the 'mc' object than the 'portal' object,
378 * because the latter runs the risk of copy-n-paste errors from other code where
379 * we could manipulate some other structure within 'portal'. */
380 /* #define MC_API_START() register struct bm_mc *mc = &portal->mc */
381
382
bm_mc_init(struct bm_portal * portal)383 t_Error bm_mc_init(struct bm_portal *portal)
384 {
385 register struct bm_mc *mc = &portal->mc;
386 mc->cr = ptr_ADD(portal->addr.addr_ce, CL_CR);
387 mc->rr = ptr_ADD(portal->addr.addr_ce, CL_RR0);
388 mc->rridx = (uint8_t)((mc->cr->__dont_write_directly__verb & BM_MCC_VERB_VBIT) ?
389 0 : 1);
390 mc->vbit = (uint8_t)(mc->rridx ? BM_MCC_VERB_VBIT : 0);
391 #ifdef BM_CHECKING
392 mc->state = mc_idle;
393 #endif /* BM_CHECKING */
394 return 0;
395 }
396
397
bm_mc_finish(struct bm_portal * portal)398 void bm_mc_finish(struct bm_portal *portal)
399 {
400 register struct bm_mc *mc = &portal->mc;
401 ASSERT_COND(mc->state == mc_idle);
402 #ifdef BM_CHECKING
403 if (mc->state != mc_idle)
404 REPORT_ERROR(WARNING, E_INVALID_STATE, ("Losing incomplete MC command"));
405 #else
406 UNUSED(mc);
407 #endif /* BM_CHECKING */
408 }
409
410
bm_mc_start(struct bm_portal * portal)411 struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
412 {
413 register struct bm_mc *mc = &portal->mc;
414 ASSERT_COND(mc->state == mc_idle);
415 #ifdef BM_CHECKING
416 mc->state = mc_user;
417 #endif /* BM_CHECKING */
418 dcbz_64(mc->cr);
419 return mc->cr;
420 }
421
422
bm_mc_abort(struct bm_portal * portal)423 void bm_mc_abort(struct bm_portal *portal)
424 {
425 register struct bm_mc *mc = &portal->mc;
426 ASSERT_COND(mc->state == mc_user);
427 #ifdef BM_CHECKING
428 mc->state = mc_idle;
429 #else
430 UNUSED(mc);
431 #endif /* BM_CHECKING */
432 }
433
434
bm_mc_commit(struct bm_portal * portal,uint8_t myverb)435 void bm_mc_commit(struct bm_portal *portal, uint8_t myverb)
436 {
437 register struct bm_mc *mc = &portal->mc;
438 ASSERT_COND(mc->state == mc_user);
439 rmb();
440 mc->cr->__dont_write_directly__verb = (uint8_t)(myverb | mc->vbit);
441 dcbf_64(mc->cr);
442 dcbit_ro(mc->rr + mc->rridx);
443 #ifdef BM_CHECKING
444 mc->state = mc_hw;
445 #endif /* BM_CHECKING */
446 }
447
448
bm_mc_result(struct bm_portal * portal)449 struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
450 {
451 register struct bm_mc *mc = &portal->mc;
452 struct bm_mc_result *rr = mc->rr + mc->rridx;
453 ASSERT_COND(mc->state == mc_hw);
454 /* The inactive response register's verb byte always returns zero until
455 * its command is submitted and completed. This includes the valid-bit,
456 * in case you were wondering... */
457 if (!rr->verb) {
458 dcbit_ro(rr);
459 return NULL;
460 }
461 mc->rridx ^= 1;
462 mc->vbit ^= BM_MCC_VERB_VBIT;
463 #ifdef BM_CHECKING
464 mc->state = mc_idle;
465 #endif /* BM_CHECKING */
466 return rr;
467 }
468
469 /* ------------------------------------- */
470 /* --- Portal interrupt register API --- */
471
472 #define SCN_REG(bpid) REG_SCN((bpid) / 32)
473 #define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
bm_isr_bscn_mask(struct bm_portal * portal,uint8_t bpid,int enable)474 void bm_isr_bscn_mask(struct bm_portal *portal, uint8_t bpid, int enable)
475 {
476 uint32_t val;
477 ASSERT_COND(bpid < BM_MAX_NUM_OF_POOLS);
478 /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
479 val = __bm_in(&portal->addr, SCN_REG(bpid));
480 if (enable)
481 val |= SCN_BIT(bpid);
482 else
483 val &= ~SCN_BIT(bpid);
484 __bm_out(&portal->addr, SCN_REG(bpid), val);
485 }
486
487
__bm_isr_read(struct bm_portal * portal,enum bm_isr_reg n)488 uint32_t __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
489 {
490 return __bm_in(&portal->addr, REG_ISR + (n << 2));
491 }
492
493
__bm_isr_write(struct bm_portal * portal,enum bm_isr_reg n,uint32_t val)494 void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, uint32_t val)
495 {
496 __bm_out(&portal->addr, REG_ISR + (n << 2), val);
497 }
498
499