xref: /freebsd/sys/contrib/ncsw/Peripherals/BM/bman_low.c (revision 43a5ec4eb41567cc92586503212743d89686d78f)
1 /******************************************************************************
2 
3  � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc.
4  All rights reserved.
5 
6  This is proprietary source code of Freescale Semiconductor Inc.,
7  and its use is subject to the NetComm Device Drivers EULA.
8  The copyright notice above does not evidence any actual or intended
9  publication of such source code.
10 
11  ALTERNATIVELY, redistribution and use in source and binary forms, with
12  or without modification, are permitted provided that the following
13  conditions are met:
14      * Redistributions of source code must retain the above copyright
15        notice, this list of conditions and the following disclaimer.
16      * Redistributions in binary form must reproduce the above copyright
17        notice, this list of conditions and the following disclaimer in the
18        documentation and/or other materials provided with the distribution.
19      * Neither the name of Freescale Semiconductor nor the
20        names of its contributors may be used to endorse or promote products
21        derived from this software without specific prior written permission.
22 
23  THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
24  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26  DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
27  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
30  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  *
34 
35  **************************************************************************/
36 /******************************************************************************
37  @File          bman_low.c
38 
39  @Description   BM low-level implementation
40 *//***************************************************************************/
41 #include <sys/cdefs.h>
42 #include <sys/types.h>
43 #include <machine/atomic.h>
44 
45 #include "std_ext.h"
46 #include "core_ext.h"
47 #include "xx_ext.h"
48 #include "error_ext.h"
49 
50 #include "bman_private.h"
51 
52 
53 /***************************/
54 /* Portal register assists */
55 /***************************/
56 
57 /* Cache-inhibited register offsets */
58 #define REG_RCR_PI_CINH     0x0000
59 #define REG_RCR_CI_CINH     0x0004
60 #define REG_RCR_ITR         0x0008
61 #define REG_CFG             0x0100
62 #define REG_SCN(n)          (0x0200 + ((n) << 2))
63 #define REG_ISR             0x0e00
64 #define REG_IER             0x0e04
65 #define REG_ISDR            0x0e08
66 #define REG_IIR             0x0e0c
67 
68 /* Cache-enabled register offsets */
69 #define CL_CR               0x0000
70 #define CL_RR0              0x0100
71 #define CL_RR1              0x0140
72 #define CL_RCR              0x1000
73 #define CL_RCR_PI_CENA      0x3000
74 #define CL_RCR_CI_CENA      0x3100
75 
76 /* The h/w design requires mappings to be size-aligned so that "add"s can be
77  * reduced to "or"s. The primitives below do the same for s/w. */
78 
79 static __inline__ void *ptr_ADD(void *a, uintptr_t b)
80 {
81     return (void *)((uintptr_t)a + b);
82 }
83 
84 /* Bitwise-OR two pointers */
85 static __inline__ void *ptr_OR(void *a, uintptr_t b)
86 {
87     return (void *)((uintptr_t)a | b);
88 }
89 
90 /* Cache-inhibited register access */
91 static __inline__ uint32_t __bm_in(struct bm_addr *bm, uintptr_t offset)
92 {
93     uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
94     return GET_UINT32(*tmp);
95 }
96 static __inline__ void __bm_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
97 {
98     uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
99     WRITE_UINT32(*tmp, val);
100 }
101 #define bm_in(reg)        __bm_in(&portal->addr, REG_##reg)
102 #define bm_out(reg, val)    __bm_out(&portal->addr, REG_##reg, val)
103 
104 /* Convert 'n' cachelines to a pointer value for bitwise OR */
105 #define bm_cl(n)        (void *)((n) << 6)
106 
107 /* Cache-enabled (index) register access */
108 static __inline__ void __bm_cl_touch_ro(struct bm_addr *bm, uintptr_t offset)
109 {
110     dcbt_ro(ptr_ADD(bm->addr_ce, offset));
111 }
112 static __inline__ void __bm_cl_touch_rw(struct bm_addr *bm, uintptr_t offset)
113 {
114     dcbt_rw(ptr_ADD(bm->addr_ce, offset));
115 }
116 static __inline__ uint32_t __bm_cl_in(struct bm_addr *bm, uintptr_t offset)
117 {
118     uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
119     return GET_UINT32(*tmp);
120 }
121 static __inline__ void __bm_cl_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
122 {
123     uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
124     WRITE_UINT32(*tmp, val);
125     dcbf(tmp);
126 }
127 static __inline__ void __bm_cl_invalidate(struct bm_addr *bm, uintptr_t offset)
128 {
129     dcbi(ptr_ADD(bm->addr_ce, offset));
130 }
131 #define bm_cl_touch_ro(reg)    __bm_cl_touch_ro(&portal->addr, CL_##reg##_CENA)
132 #define bm_cl_touch_rw(reg)    __bm_cl_touch_rw(&portal->addr, CL_##reg##_CENA)
133 #define bm_cl_in(reg)        __bm_cl_in(&portal->addr, CL_##reg##_CENA)
134 #define bm_cl_out(reg, val)    __bm_cl_out(&portal->addr, CL_##reg##_CENA, val)
135 #define bm_cl_invalidate(reg) __bm_cl_invalidate(&portal->addr, CL_##reg##_CENA)
136 
137 /* Cyclic helper for rings. TODO: once we are able to do fine-grain perf
138  * analysis, look at using the "extra" bit in the ring index registers to avoid
139  * cyclic issues. */
140 static __inline__ uint8_t cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
141 {
142     /* 'first' is included, 'last' is excluded */
143     if (first <= last)
144         return (uint8_t)(last - first);
145     return (uint8_t)(ringsize + last - first);
146 }
147 
148 /* --------------- */
149 /* --- RCR API --- */
150 
151 /* It's safer to code in terms of the 'rcr' object than the 'portal' object,
152  * because the latter runs the risk of copy-n-paste errors from other code where
153  * we could manipulate some other structure within 'portal'. */
154 /* #define RCR_API_START()      register struct bm_rcr *rcr = &portal->rcr */
155 
156 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
157 #define RCR_CARRYCLEAR(p) \
158     (void *)((uintptr_t)(p) & (~(uintptr_t)(BM_RCR_SIZE << 6)))
159 
160 /* Bit-wise logic to convert a ring pointer to a ring index */
161 static __inline__ uint8_t RCR_PTR2IDX(struct bm_rcr_entry *e)
162 {
163     return (uint8_t)(((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1));
164 }
165 
166 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
167 static __inline__ void RCR_INC(struct bm_rcr *rcr)
168 {
169     /* NB: this is odd-looking, but experiments show that it generates
170      * fast code with essentially no branching overheads. We increment to
171      * the next RCR pointer and handle overflow and 'vbit'. */
172     struct bm_rcr_entry *partial = rcr->cursor + 1;
173     rcr->cursor = RCR_CARRYCLEAR(partial);
174     if (partial != rcr->cursor)
175         rcr->vbit ^= BM_RCR_VERB_VBIT;
176 }
177 
178 t_Error bm_rcr_init(struct bm_portal *portal,
179                     e_BmPortalProduceMode pmode,
180                     e_BmPortalRcrConsumeMode cmode)
181 {
182     register struct bm_rcr *rcr = &portal->rcr;
183     uint32_t cfg;
184     uint8_t pi;
185 
186     rcr->ring = ptr_ADD(portal->addr.addr_ce, CL_RCR);
187     rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
188     pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
189     rcr->cursor = rcr->ring + pi;
190     rcr->vbit = (uint8_t)((bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ?  BM_RCR_VERB_VBIT : 0);
191     rcr->available = (uint8_t)(BM_RCR_SIZE - 1 - cyc_diff(BM_RCR_SIZE, rcr->ci, pi));
192     rcr->ithresh = (uint8_t)bm_in(RCR_ITR);
193 #ifdef BM_CHECKING
194     rcr->busy = 0;
195     rcr->pmode = pmode;
196     rcr->cmode = cmode;
197 #else
198     UNUSED(cmode);
199 #endif /* BM_CHECKING */
200     cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
201     bm_out(CFG, cfg);
202     return 0;
203 }
204 
205 void bm_rcr_finish(struct bm_portal *portal)
206 {
207     register struct bm_rcr *rcr = &portal->rcr;
208     uint8_t pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
209     uint8_t ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
210     ASSERT_COND(!rcr->busy);
211     if (pi != RCR_PTR2IDX(rcr->cursor))
212         REPORT_ERROR(WARNING, E_INVALID_STATE, ("losing uncommitted RCR entries"));
213     if (ci != rcr->ci)
214         REPORT_ERROR(WARNING, E_INVALID_STATE, ("missing existing RCR completions"));
215     if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
216         REPORT_ERROR(WARNING, E_INVALID_STATE, ("RCR destroyed unquiesced"));
217 }
218 
219 struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
220 {
221     register struct bm_rcr *rcr = &portal->rcr;
222     ASSERT_COND(!rcr->busy);
223     if (!rcr->available)
224         return NULL;
225 #ifdef BM_CHECKING
226     rcr->busy = 1;
227 #endif /* BM_CHECKING */
228     dcbz_64(rcr->cursor);
229     return rcr->cursor;
230 }
231 
232 void bm_rcr_abort(struct bm_portal *portal)
233 {
234     register struct bm_rcr *rcr = &portal->rcr;
235     ASSERT_COND(rcr->busy);
236 #ifdef BM_CHECKING
237     rcr->busy = 0;
238 #else
239     UNUSED(rcr);
240 #endif /* BM_CHECKING */
241 }
242 
243 struct bm_rcr_entry *bm_rcr_pend_and_next(struct bm_portal *portal, uint8_t myverb)
244 {
245     register struct bm_rcr *rcr = &portal->rcr;
246     ASSERT_COND(rcr->busy);
247     ASSERT_COND(rcr->pmode != e_BmPortalPVB);
248     if (rcr->available == 1)
249         return NULL;
250     rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
251     dcbf_64(rcr->cursor);
252     RCR_INC(rcr);
253     rcr->available--;
254     dcbz_64(rcr->cursor);
255     return rcr->cursor;
256 }
257 
258 void bm_rcr_pci_commit(struct bm_portal *portal, uint8_t myverb)
259 {
260     register struct bm_rcr *rcr = &portal->rcr;
261     ASSERT_COND(rcr->busy);
262     ASSERT_COND(rcr->pmode == e_BmPortalPCI);
263     rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
264     RCR_INC(rcr);
265     rcr->available--;
266     mb();
267     bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
268 #ifdef BM_CHECKING
269     rcr->busy = 0;
270 #endif /* BM_CHECKING */
271 }
272 
273 void bm_rcr_pce_prefetch(struct bm_portal *portal)
274 {
275     ASSERT_COND(((struct bm_rcr *)&portal->rcr)->pmode == e_BmPortalPCE);
276     bm_cl_invalidate(RCR_PI);
277     bm_cl_touch_rw(RCR_PI);
278 }
279 
280 void bm_rcr_pce_commit(struct bm_portal *portal, uint8_t myverb)
281 {
282     register struct bm_rcr *rcr = &portal->rcr;
283     ASSERT_COND(rcr->busy);
284     ASSERT_COND(rcr->pmode == e_BmPortalPCE);
285     rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
286     RCR_INC(rcr);
287     rcr->available--;
288     wmb();
289     bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
290 #ifdef BM_CHECKING
291     rcr->busy = 0;
292 #endif /* BM_CHECKING */
293 }
294 
295 void bm_rcr_pvb_commit(struct bm_portal *portal, uint8_t myverb)
296 {
297     register struct bm_rcr *rcr = &portal->rcr;
298     struct bm_rcr_entry *rcursor;
299     ASSERT_COND(rcr->busy);
300     ASSERT_COND(rcr->pmode == e_BmPortalPVB);
301     rmb();
302     rcursor = rcr->cursor;
303     rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
304     dcbf_64(rcursor);
305     RCR_INC(rcr);
306     rcr->available--;
307 #ifdef BM_CHECKING
308     rcr->busy = 0;
309 #endif /* BM_CHECKING */
310 }
311 
312 
313 uint8_t bm_rcr_cci_update(struct bm_portal *portal)
314 {
315     register struct bm_rcr *rcr = &portal->rcr;
316     uint8_t diff, old_ci = rcr->ci;
317     ASSERT_COND(rcr->cmode == e_BmPortalRcrCCI);
318     rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
319     diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
320     rcr->available += diff;
321     return diff;
322 }
323 
324 
325 void bm_rcr_cce_prefetch(struct bm_portal *portal)
326 {
327     ASSERT_COND(((struct bm_rcr *)&portal->rcr)->cmode == e_BmPortalRcrCCE);
328     bm_cl_touch_ro(RCR_CI);
329 }
330 
331 
332 uint8_t bm_rcr_cce_update(struct bm_portal *portal)
333 {
334     register struct bm_rcr *rcr = &portal->rcr;
335     uint8_t diff, old_ci = rcr->ci;
336     ASSERT_COND(rcr->cmode == e_BmPortalRcrCCE);
337     rcr->ci = (uint8_t)(bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1));
338     bm_cl_invalidate(RCR_CI);
339     diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
340     rcr->available += diff;
341     return diff;
342 }
343 
344 
345 uint8_t bm_rcr_get_ithresh(struct bm_portal *portal)
346 {
347     register struct bm_rcr *rcr = &portal->rcr;
348     return rcr->ithresh;
349 }
350 
351 
352 void bm_rcr_set_ithresh(struct bm_portal *portal, uint8_t ithresh)
353 {
354     register struct bm_rcr *rcr = &portal->rcr;
355     rcr->ithresh = ithresh;
356     bm_out(RCR_ITR, ithresh);
357 }
358 
359 
360 uint8_t bm_rcr_get_avail(struct bm_portal *portal)
361 {
362     register struct bm_rcr *rcr = &portal->rcr;
363     return rcr->available;
364 }
365 
366 
367 uint8_t bm_rcr_get_fill(struct bm_portal *portal)
368 {
369     register struct bm_rcr *rcr = &portal->rcr;
370     return (uint8_t)(BM_RCR_SIZE - 1 - rcr->available);
371 }
372 
373 
374 /* ------------------------------ */
375 /* --- Management command API --- */
376 
377 /* It's safer to code in terms of the 'mc' object than the 'portal' object,
378  * because the latter runs the risk of copy-n-paste errors from other code where
379  * we could manipulate some other structure within 'portal'. */
380 /* #define MC_API_START()      register struct bm_mc *mc = &portal->mc */
381 
382 
383 t_Error bm_mc_init(struct bm_portal *portal)
384 {
385     register struct bm_mc *mc = &portal->mc;
386     mc->cr = ptr_ADD(portal->addr.addr_ce, CL_CR);
387     mc->rr = ptr_ADD(portal->addr.addr_ce, CL_RR0);
388     mc->rridx = (uint8_t)((mc->cr->__dont_write_directly__verb & BM_MCC_VERB_VBIT) ?
389             0 : 1);
390     mc->vbit = (uint8_t)(mc->rridx ? BM_MCC_VERB_VBIT : 0);
391 #ifdef BM_CHECKING
392     mc->state = mc_idle;
393 #endif /* BM_CHECKING */
394     return 0;
395 }
396 
397 
398 void bm_mc_finish(struct bm_portal *portal)
399 {
400     register struct bm_mc *mc = &portal->mc;
401     ASSERT_COND(mc->state == mc_idle);
402 #ifdef BM_CHECKING
403     if (mc->state != mc_idle)
404         REPORT_ERROR(WARNING, E_INVALID_STATE, ("Losing incomplete MC command"));
405 #else
406     UNUSED(mc);
407 #endif /* BM_CHECKING */
408 }
409 
410 
411 struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
412 {
413     register struct bm_mc *mc = &portal->mc;
414     ASSERT_COND(mc->state == mc_idle);
415 #ifdef BM_CHECKING
416     mc->state = mc_user;
417 #endif /* BM_CHECKING */
418     dcbz_64(mc->cr);
419     return mc->cr;
420 }
421 
422 
423 void bm_mc_abort(struct bm_portal *portal)
424 {
425     register struct bm_mc *mc = &portal->mc;
426     ASSERT_COND(mc->state == mc_user);
427 #ifdef BM_CHECKING
428     mc->state = mc_idle;
429 #else
430     UNUSED(mc);
431 #endif /* BM_CHECKING */
432 }
433 
434 
435 void bm_mc_commit(struct bm_portal *portal, uint8_t myverb)
436 {
437     register struct bm_mc *mc = &portal->mc;
438     ASSERT_COND(mc->state == mc_user);
439     rmb();
440     mc->cr->__dont_write_directly__verb = (uint8_t)(myverb | mc->vbit);
441     dcbf_64(mc->cr);
442     dcbit_ro(mc->rr + mc->rridx);
443 #ifdef BM_CHECKING
444     mc->state = mc_hw;
445 #endif /* BM_CHECKING */
446 }
447 
448 
449 struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
450 {
451     register struct bm_mc *mc = &portal->mc;
452     struct bm_mc_result *rr = mc->rr + mc->rridx;
453     ASSERT_COND(mc->state == mc_hw);
454     /* The inactive response register's verb byte always returns zero until
455      * its command is submitted and completed. This includes the valid-bit,
456      * in case you were wondering... */
457     if (!rr->verb) {
458         dcbit_ro(rr);
459         return NULL;
460     }
461     mc->rridx ^= 1;
462     mc->vbit ^= BM_MCC_VERB_VBIT;
463 #ifdef BM_CHECKING
464     mc->state = mc_idle;
465 #endif /* BM_CHECKING */
466     return rr;
467 }
468 
469 /* ------------------------------------- */
470 /* --- Portal interrupt register API --- */
471 
472 #define SCN_REG(bpid) REG_SCN((bpid) / 32)
473 #define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
474 void bm_isr_bscn_mask(struct bm_portal *portal, uint8_t bpid, int enable)
475 {
476     uint32_t val;
477     ASSERT_COND(bpid < BM_MAX_NUM_OF_POOLS);
478     /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
479     val = __bm_in(&portal->addr, SCN_REG(bpid));
480     if (enable)
481         val |= SCN_BIT(bpid);
482     else
483         val &= ~SCN_BIT(bpid);
484     __bm_out(&portal->addr, SCN_REG(bpid), val);
485 }
486 
487 
488 uint32_t __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
489 {
490     return __bm_in(&portal->addr, REG_ISR + (n << 2));
491 }
492 
493 
494 void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, uint32_t val)
495 {
496     __bm_out(&portal->addr, REG_ISR + (n << 2), val);
497 }
498 
499