1 /**
2 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions, and the following disclaimer,
9 * without modification.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The names of the above-listed copyright holders may not be used
14 * to endorse or promote products derived from this software without
15 * specific prior written permission.
16 *
17 * ALTERNATIVELY, this software may be distributed under the terms of the
18 * GNU General Public License ("GPL") version 2, as published by the Free
19 * Software Foundation.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
23 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <interface/compat/vchi_bsd.h>
35
36 #include <sys/malloc.h>
37 #include <sys/rwlock.h>
38
39 #include <vm/vm.h>
40 #include <vm/pmap.h>
41 #include <vm/vm_extern.h>
42 #include <vm/vm_kern.h>
43 #include <vm/vm_map.h>
44 #include <vm/vm_object.h>
45 #include <vm/vm_page.h>
46 #include <vm/vm_pager.h>
47 #include <vm/vm_param.h>
48
49 #include <machine/bus.h>
50 #include <machine/cpu.h>
51 #include <arm/broadcom/bcm2835/bcm2835_mbox.h>
52 #include <arm/broadcom/bcm2835/bcm2835_vcbus.h>
53
54 MALLOC_DEFINE(M_VCPAGELIST, "vcpagelist", "VideoCore pagelist memory");
55
56 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
57
58 #define VCHIQ_DOORBELL_IRQ IRQ_ARM_DOORBELL_0
59 #define VCHIQ_ARM_ADDRESS(x) ((void *)PHYS_TO_VCBUS(pmap_kextract((vm_offset_t)(x))))
60
61 #include "vchiq_arm.h"
62 #include "vchiq_2835.h"
63 #include "vchiq_connected.h"
64 #include "vchiq_killable.h"
65
66 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
67
68 /*
69 * XXXMDC
70 * Do this less ad-hoc-y -- e.g.
71 * https://github.com/raspberrypi/linux/commit/c683db8860a80562a2bb5b451d77b3e471d24f36
72 */
73 #if defined(__aarch64__)
74 int g_cache_line_size = 64;
75 #else
76 int g_cache_line_size = 32;
77 #endif
78 static int g_fragment_size;
79
80 unsigned int g_long_bulk_space = 0;
81 #define VM_PAGE_TO_VC_BULK_PAGE(x) (\
82 g_long_bulk_space ? VM_PAGE_TO_PHYS(x)\
83 : PHYS_TO_VCBUS(VM_PAGE_TO_PHYS(x))\
84 )
85
86 typedef struct vchiq_2835_state_struct {
87 int inited;
88 VCHIQ_ARM_STATE_T arm_state;
89 } VCHIQ_2835_ARM_STATE_T;
90
91 static char *g_slot_mem;
92 static int g_slot_mem_size;
93 vm_paddr_t g_slot_phys;
94 /* BSD DMA */
95 bus_dma_tag_t bcm_slots_dma_tag;
96 bus_dmamap_t bcm_slots_dma_map;
97
98 static char *g_fragments_base;
99 static char *g_free_fragments;
100 struct semaphore g_free_fragments_sema;
101
102 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
103
104 typedef struct bulkinfo_struct {
105 PAGELIST_T *pagelist;
106 bus_dma_tag_t pagelist_dma_tag;
107 bus_dmamap_t pagelist_dma_map;
108 void *buf;
109 size_t size;
110 } BULKINFO_T;
111
112 static int
113 create_pagelist(char __user *buf, size_t count, unsigned short type,
114 struct proc *p, BULKINFO_T *bi);
115
116 static void
117 free_pagelist(BULKINFO_T *bi, int actual);
118
119 static void
vchiq_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int err)120 vchiq_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
121 {
122 bus_addr_t *addr;
123
124 if (err)
125 return;
126
127 addr = (bus_addr_t*)arg;
128 *addr = PHYS_TO_VCBUS(segs[0].ds_addr);
129 }
130
131 #if defined(__aarch64__) /* See comment in free_pagelist */
132 static int
invalidate_cachelines_in_range_of_ppage(vm_page_t p,size_t offset,size_t count)133 invalidate_cachelines_in_range_of_ppage(
134 vm_page_t p,
135 size_t offset,
136 size_t count
137 )
138 {
139 if(offset + count > PAGE_SIZE){ return EINVAL; }
140 uint8_t *dst = pmap_quick_enter_page(p);
141 if (!dst){
142 return ENOMEM;
143 }
144 cpu_dcache_inv_range(dst + offset, count);
145 pmap_quick_remove_page(dst);
146 return 0;
147 }
148
149 /* XXXMDC bulk instead of loading and invalidating single pages? */
150 static void
invalidate_cachelines_in_range_of_ppage_seq(vm_page_t * p,size_t start,size_t count)151 invalidate_cachelines_in_range_of_ppage_seq(vm_page_t *p, size_t start,
152 size_t count)
153 {
154 if (start >= PAGE_SIZE)
155 goto invalid_input;
156
157 #define _NEXT_AT(x,_m) (((x)+((_m)-1)) & ~((_m)-1)) /* for power of two m */
158 size_t offset = _NEXT_AT(start,g_cache_line_size);
159 #undef _NEXT_AT
160 count = (offset < start + count) ? count - (offset - start) : 0;
161 offset = offset & (PAGE_SIZE - 1);
162 for (size_t done = 0; count > done;
163 p++, done += PAGE_SIZE - offset, offset = 0) {
164 size_t in_page = PAGE_SIZE - offset;
165 size_t todo = (count-done > in_page) ? in_page : count-done;
166 int e = invalidate_cachelines_in_range_of_ppage(*p, offset, todo);
167 if (e != 0)
168 goto problem_in_loop;
169 }
170 return;
171
172 problem_in_loop:
173 invalid_input:
174 WARN_ON(1);
175 return;
176 }
177 #endif
178
179 static int
copyout_page(vm_page_t p,size_t offset,void * kaddr,size_t size)180 copyout_page(vm_page_t p, size_t offset, void *kaddr, size_t size)
181 {
182 uint8_t *dst;
183
184 dst = pmap_quick_enter_page(p);
185 if (!dst)
186 return ENOMEM;
187
188 memcpy(dst + offset, kaddr, size);
189
190 pmap_quick_remove_page(dst);
191
192 return 0;
193 }
194
195 int __init
vchiq_platform_init(VCHIQ_STATE_T * state)196 vchiq_platform_init(VCHIQ_STATE_T *state)
197 {
198 VCHIQ_SLOT_ZERO_T *vchiq_slot_zero;
199 int frag_mem_size;
200 int err;
201 int i;
202
203 /* Allocate space for the channels in coherent memory */
204 g_slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
205 g_fragment_size = 2*g_cache_line_size;
206 frag_mem_size = PAGE_ALIGN(g_fragment_size * MAX_FRAGMENTS);
207
208 err = bus_dma_tag_create(
209 NULL,
210 PAGE_SIZE, 0, /* alignment, boundary */
211 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
212 BUS_SPACE_MAXADDR, /* highaddr */
213 NULL, NULL, /* filter, filterarg */
214 g_slot_mem_size + frag_mem_size, 1, /* maxsize, nsegments */
215 g_slot_mem_size + frag_mem_size, 0, /* maxsegsize, flags */
216 NULL, NULL, /* lockfunc, lockarg */
217 &bcm_slots_dma_tag);
218
219 err = bus_dmamem_alloc(bcm_slots_dma_tag, (void **)&g_slot_mem,
220 BUS_DMA_COHERENT | BUS_DMA_WAITOK, &bcm_slots_dma_map);
221 if (err) {
222 vchiq_log_error(vchiq_core_log_level, "Unable to allocate channel memory");
223 err = -ENOMEM;
224 goto failed_alloc;
225 }
226
227 err = bus_dmamap_load(bcm_slots_dma_tag, bcm_slots_dma_map, g_slot_mem,
228 g_slot_mem_size + frag_mem_size, vchiq_dmamap_cb,
229 &g_slot_phys, 0);
230
231 if (err) {
232 vchiq_log_error(vchiq_core_log_level, "cannot load DMA map");
233 err = -ENOMEM;
234 goto failed_load;
235 }
236
237 WARN_ON(((size_t)g_slot_mem & (PAGE_SIZE - 1)) != 0);
238
239 vchiq_slot_zero = vchiq_init_slots(g_slot_mem, g_slot_mem_size);
240 if (!vchiq_slot_zero) {
241 err = -EINVAL;
242 goto failed_init_slots;
243 }
244
245 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
246 (int)g_slot_phys + g_slot_mem_size;
247 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
248 MAX_FRAGMENTS;
249
250 g_fragments_base = (char *)(g_slot_mem + g_slot_mem_size);
251 g_slot_mem_size += frag_mem_size;
252
253 g_free_fragments = g_fragments_base;
254 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
255 *(char **)&g_fragments_base[i*g_fragment_size] =
256 &g_fragments_base[(i + 1)*g_fragment_size];
257 }
258 *(char **)&g_fragments_base[i*g_fragment_size] = NULL;
259 _sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
260
261 if (vchiq_init_state(state, vchiq_slot_zero, 0/*slave*/) !=
262 VCHIQ_SUCCESS) {
263 err = -EINVAL;
264 goto failed_vchiq_init;
265 }
266
267 bcm_mbox_write(BCM2835_MBOX_CHAN_VCHIQ, (unsigned int)g_slot_phys);
268
269 vchiq_log_info(vchiq_arm_log_level,
270 "vchiq_init - done (slots %zx, phys %zx)",
271 (size_t)vchiq_slot_zero, g_slot_phys);
272
273 vchiq_call_connected_callbacks();
274
275 return 0;
276
277 failed_vchiq_init:
278 failed_init_slots:
279 bus_dmamap_unload(bcm_slots_dma_tag, bcm_slots_dma_map);
280 failed_load:
281 bus_dmamem_free(bcm_slots_dma_tag, g_slot_mem, bcm_slots_dma_map);
282 failed_alloc:
283 bus_dma_tag_destroy(bcm_slots_dma_tag);
284
285 return err;
286 }
287
288 void __exit
vchiq_platform_exit(VCHIQ_STATE_T * state)289 vchiq_platform_exit(VCHIQ_STATE_T *state)
290 {
291
292 bus_dmamap_unload(bcm_slots_dma_tag, bcm_slots_dma_map);
293 bus_dmamem_free(bcm_slots_dma_tag, g_slot_mem, bcm_slots_dma_map);
294 bus_dma_tag_destroy(bcm_slots_dma_tag);
295 }
296
297 VCHIQ_STATUS_T
vchiq_platform_init_state(VCHIQ_STATE_T * state)298 vchiq_platform_init_state(VCHIQ_STATE_T *state)
299 {
300 VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
301 state->platform_state = kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T), GFP_KERNEL);
302 ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 1;
303 status = vchiq_arm_init_state(state, &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state);
304 if(status != VCHIQ_SUCCESS)
305 {
306 ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 0;
307 }
308 return status;
309 }
310
311 VCHIQ_ARM_STATE_T*
vchiq_platform_get_arm_state(VCHIQ_STATE_T * state)312 vchiq_platform_get_arm_state(VCHIQ_STATE_T *state)
313 {
314 if(!((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited)
315 {
316 BUG();
317 }
318 return &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state;
319 }
320
321 VCHIQ_STATUS_T
vchiq_prepare_bulk_data(VCHIQ_BULK_T * bulk,VCHI_MEM_HANDLE_T memhandle,void * offset,int size,int dir)322 vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle,
323 void *offset, int size, int dir)
324 {
325 BULKINFO_T *bi;
326 int ret;
327
328 WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID);
329 bi = malloc(sizeof(*bi), M_VCPAGELIST, M_WAITOK | M_ZERO);
330
331 ret = create_pagelist((char __user *)offset, size,
332 (dir == VCHIQ_BULK_RECEIVE)
333 ? PAGELIST_READ
334 : PAGELIST_WRITE,
335 current,
336 bi);
337 if (ret != 0)
338 return VCHIQ_ERROR;
339
340 bulk->handle = memhandle;
341 bulk->data = VCHIQ_ARM_ADDRESS(bi->pagelist);
342
343 /* Store the pagelist address in remote_data, which isn't used by the
344 slave. */
345 bulk->remote_data = bi;
346
347 return VCHIQ_SUCCESS;
348 }
349
350 void
vchiq_complete_bulk(VCHIQ_BULK_T * bulk)351 vchiq_complete_bulk(VCHIQ_BULK_T *bulk)
352 {
353 if (bulk && bulk->remote_data && bulk->actual)
354 free_pagelist((BULKINFO_T *)bulk->remote_data, bulk->actual);
355 }
356
357 void
vchiq_transfer_bulk(VCHIQ_BULK_T * bulk)358 vchiq_transfer_bulk(VCHIQ_BULK_T *bulk)
359 {
360 /*
361 * This should only be called on the master (VideoCore) side, but
362 * provide an implementation to avoid the need for ifdefery.
363 */
364 BUG();
365 }
366
367 void
vchiq_dump_platform_state(void * dump_context)368 vchiq_dump_platform_state(void *dump_context)
369 {
370 char buf[80];
371 int len;
372 len = snprintf(buf, sizeof(buf),
373 " Platform: 2835 (VC master)");
374 vchiq_dump(dump_context, buf, len + 1);
375 }
376
377 VCHIQ_STATUS_T
vchiq_platform_suspend(VCHIQ_STATE_T * state)378 vchiq_platform_suspend(VCHIQ_STATE_T *state)
379 {
380 return VCHIQ_ERROR;
381 }
382
383 VCHIQ_STATUS_T
vchiq_platform_resume(VCHIQ_STATE_T * state)384 vchiq_platform_resume(VCHIQ_STATE_T *state)
385 {
386 return VCHIQ_SUCCESS;
387 }
388
389 void
vchiq_platform_paused(VCHIQ_STATE_T * state)390 vchiq_platform_paused(VCHIQ_STATE_T *state)
391 {
392 }
393
394 void
vchiq_platform_resumed(VCHIQ_STATE_T * state)395 vchiq_platform_resumed(VCHIQ_STATE_T *state)
396 {
397 }
398
399 int
vchiq_platform_videocore_wanted(VCHIQ_STATE_T * state)400 vchiq_platform_videocore_wanted(VCHIQ_STATE_T* state)
401 {
402 return 1; // autosuspend not supported - videocore always wanted
403 }
404
405 int
vchiq_platform_use_suspend_timer(void)406 vchiq_platform_use_suspend_timer(void)
407 {
408 return 0;
409 }
410 void
vchiq_dump_platform_use_state(VCHIQ_STATE_T * state)411 vchiq_dump_platform_use_state(VCHIQ_STATE_T *state)
412 {
413 vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
414 }
415 void
vchiq_platform_handle_timeout(VCHIQ_STATE_T * state)416 vchiq_platform_handle_timeout(VCHIQ_STATE_T *state)
417 {
418 (void)state;
419 }
420 /*
421 * Local functions
422 */
423
424 static void
pagelist_page_free(vm_page_t pp)425 pagelist_page_free(vm_page_t pp)
426 {
427 vm_page_unwire(pp, PQ_INACTIVE);
428 }
429
430 /* There is a potential problem with partial cache lines (pages?)
431 ** at the ends of the block when reading. If the CPU accessed anything in
432 ** the same line (page?) then it may have pulled old data into the cache,
433 ** obscuring the new data underneath. We can solve this by transferring the
434 ** partial cache lines separately, and allowing the ARM to copy into the
435 ** cached area.
436
437 ** N.B. This implementation plays slightly fast and loose with the Linux
438 ** driver programming rules, e.g. its use of __virt_to_bus instead of
439 ** dma_map_single, but it isn't a multi-platform driver and it benefits
440 ** from increased speed as a result.
441 */
442
443
444 static int
create_pagelist(char __user * buf,size_t count,unsigned short type,struct proc * p,BULKINFO_T * bi)445 create_pagelist(char __user *buf, size_t count, unsigned short type,
446 struct proc *p, BULKINFO_T *bi)
447 {
448 PAGELIST_T *pagelist;
449 vm_page_t* pages;
450 uint32_t *addrs;
451 unsigned int num_pages, i;
452 vm_offset_t offset;
453 int pagelist_size;
454 char *addr, *base_addr, *next_addr;
455 int run, addridx, actual_pages;
456 int err;
457 vm_paddr_t pagelist_phys;
458 vm_paddr_t pa;
459
460 offset = (vm_offset_t)buf & (PAGE_SIZE - 1);
461 num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE;
462
463 bi->pagelist = NULL;
464 bi->buf = buf;
465 bi->size = count;
466
467 /* Allocate enough storage to hold the page pointers and the page
468 ** list
469 */
470 pagelist_size = sizeof(PAGELIST_T) +
471 (num_pages * sizeof(unsigned long)) +
472 (num_pages * sizeof(pages[0]));
473
474 err = bus_dma_tag_create(
475 NULL,
476 PAGE_SIZE, 0, /* alignment, boundary */
477 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
478 BUS_SPACE_MAXADDR, /* highaddr */
479 NULL, NULL, /* filter, filterarg */
480 pagelist_size, 1, /* maxsize, nsegments */
481 pagelist_size, 0, /* maxsegsize, flags */
482 NULL, NULL, /* lockfunc, lockarg */
483 &bi->pagelist_dma_tag);
484
485 err = bus_dmamem_alloc(bi->pagelist_dma_tag, (void **)&pagelist,
486 BUS_DMA_COHERENT | BUS_DMA_WAITOK, &bi->pagelist_dma_map);
487 if (err || !pagelist) {
488 vchiq_log_error(vchiq_core_log_level, "Unable to allocate pagelist memory");
489 err = -ENOMEM;
490 goto failed_alloc;
491 }
492
493 err = bus_dmamap_load(bi->pagelist_dma_tag, bi->pagelist_dma_map, pagelist,
494 pagelist_size, vchiq_dmamap_cb,
495 &pagelist_phys, 0);
496
497 if (err) {
498 vchiq_log_error(vchiq_core_log_level, "cannot load DMA map for pagelist memory");
499 err = -ENOMEM;
500 bi->pagelist = pagelist;
501 goto failed_load;
502 }
503
504 vchiq_log_trace(vchiq_arm_log_level,
505 "create_pagelist - %zx (%zu bytes @%p)", (size_t)pagelist, count, buf);
506
507 addrs = pagelist->addrs;
508 pages = (vm_page_t*)(addrs + num_pages);
509
510 actual_pages = vm_fault_quick_hold_pages(&p->p_vmspace->vm_map,
511 (vm_offset_t)buf, count,
512 (type == PAGELIST_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, pages, num_pages);
513
514 if (actual_pages != num_pages) {
515 if (actual_pages > 0)
516 vm_page_unhold_pages(pages, actual_pages);
517 err = -ENOMEM;
518 bi->pagelist = pagelist;
519 goto failed_hold;
520 }
521
522 pagelist->length = count;
523 pagelist->type = type;
524 pagelist->offset = offset;
525
526 /* Group the pages into runs of contiguous pages */
527
528 size_t run_ceil = g_long_bulk_space ? 0x100 : PAGE_SIZE;
529 unsigned int pg_addr_rshift = g_long_bulk_space ? 4 : 0;
530 base_addr = (void *) VM_PAGE_TO_VC_BULK_PAGE(pages[0]);
531 next_addr = base_addr + PAGE_SIZE;
532 addridx = 0;
533 run = 0;
534 #define _PG_BLOCK(base,run) \
535 ((((size_t) (base)) >> pg_addr_rshift) & ~(run_ceil-1)) + (run)
536 for (i = 1; i < num_pages; i++) {
537 addr = (void *)VM_PAGE_TO_VC_BULK_PAGE(pages[i]);
538 if ((addr == next_addr) && (run < run_ceil - 1)) {
539 next_addr += PAGE_SIZE;
540 run++;
541 } else {
542 addrs[addridx++] = (uint32_t) _PG_BLOCK(base_addr,run);
543 base_addr = addr;
544 next_addr = addr + PAGE_SIZE;
545 run = 0;
546 }
547 }
548 addrs[addridx++] = _PG_BLOCK(base_addr, run);
549 #undef _PG_BLOCK
550
551 /* Partial cache lines (fragments) require special measures */
552 if ((type == PAGELIST_READ) &&
553 ((pagelist->offset & (g_cache_line_size - 1)) ||
554 ((pagelist->offset + pagelist->length) &
555 (g_cache_line_size - 1)))) {
556 char *fragments;
557
558 if (down_interruptible(&g_free_fragments_sema) != 0) {
559 free(pagelist, M_VCPAGELIST);
560 return -EINTR;
561 }
562
563 WARN_ON(g_free_fragments == NULL);
564
565 down(&g_free_fragments_mutex);
566 fragments = g_free_fragments;
567 WARN_ON(fragments == NULL);
568 g_free_fragments = *(char **) g_free_fragments;
569 up(&g_free_fragments_mutex);
570 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS
571 + (fragments - g_fragments_base)/g_fragment_size;
572 #if defined(__aarch64__)
573 bus_dmamap_sync(bcm_slots_dma_tag, bcm_slots_dma_map,
574 BUS_DMASYNC_PREREAD);
575 #endif
576 }
577
578 #if defined(__aarch64__)
579 if(type == PAGELIST_READ) {
580 cpu_dcache_wbinv_range(buf, count);
581 } else {
582 cpu_dcache_wb_range(buf, count);
583 }
584 dsb(sy);
585 #else
586 pa = pmap_extract(PCPU_GET(curpmap), (vm_offset_t)buf);
587 dcache_wbinv_poc((vm_offset_t)buf, pa, count);
588 #endif
589
590 bus_dmamap_sync(bi->pagelist_dma_tag, bi->pagelist_dma_map,
591 BUS_DMASYNC_PREWRITE);
592
593 bi->pagelist = pagelist;
594
595 return 0;
596
597 failed_hold:
598 bus_dmamap_unload(bi->pagelist_dma_tag,bi->pagelist_dma_map);
599 failed_load:
600 bus_dmamem_free(bi->pagelist_dma_tag, bi->pagelist, bi->pagelist_dma_map);
601 failed_alloc:
602 bus_dma_tag_destroy(bi->pagelist_dma_tag);
603
604 return err;
605 }
606
607 static void
free_pagelist(BULKINFO_T * bi,int actual)608 free_pagelist(BULKINFO_T *bi, int actual)
609 {
610 vm_page_t*pages;
611 unsigned int num_pages, i;
612 PAGELIST_T *pagelist;
613
614 pagelist = bi->pagelist;
615
616 vchiq_log_trace(vchiq_arm_log_level,
617 "free_pagelist - %zx, %d (%lu bytes @%p)",
618 (size_t)pagelist, (int)actual, (unsigned long)pagelist->length,
619 bi->buf);
620
621 num_pages =
622 (pagelist->length + pagelist->offset + PAGE_SIZE - 1) /
623 PAGE_SIZE;
624
625 pages = (vm_page_t*)(pagelist->addrs + num_pages);
626
627 #if defined(__aarch64__)
628 /*
629 * On arm64, even if the user keeps their end of the bargain
630 * -- do NOT touch the buffers sent to VC -- but reads around the
631 * pagelist after the invalidation above, the arm might preemptively
632 * load (and validate) cache lines for areas inside the page list,
633 * so we must invalidate them again.
634 *
635 * The functional test does it and without this it doesn't pass.
636 *
637 * XXXMDC might it be enough to invalidate a couple of pages at
638 * the ends of the page list?
639 */
640 if(pagelist->type >= PAGELIST_READ && actual > 0)
641 invalidate_cachelines_in_range_of_ppage_seq(pages,
642 pagelist->offset, actual);
643 #endif
644
645 /* Deal with any partial cache lines (fragments) */
646 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
647 char *fragments = g_fragments_base +
648 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS)*g_fragment_size;
649 int head_bytes, tail_bytes;
650 head_bytes = (g_cache_line_size - pagelist->offset) &
651 (g_cache_line_size - 1);
652 tail_bytes = (pagelist->offset + actual) &
653 (g_cache_line_size - 1);
654
655 if ((actual >= 0) && (head_bytes != 0)) {
656 if (head_bytes > actual)
657 head_bytes = actual;
658
659 copyout_page(pages[0],
660 pagelist->offset,
661 fragments,
662 head_bytes);
663 }
664
665 if ((actual >= 0) && (head_bytes < actual) &&
666 (tail_bytes != 0)) {
667
668 copyout_page(pages[num_pages-1],
669 (((vm_offset_t)bi->buf + actual) % PAGE_SIZE) - tail_bytes,
670 fragments + g_cache_line_size,
671 tail_bytes);
672 }
673
674 down(&g_free_fragments_mutex);
675 *(char **) fragments = g_free_fragments;
676 g_free_fragments = fragments;
677 up(&g_free_fragments_mutex);
678 up(&g_free_fragments_sema);
679 }
680
681 if (pagelist->type != PAGELIST_WRITE) {
682 for (i = 0; i < num_pages; i++) {
683 vm_page_dirty(pages[i]);
684 pagelist_page_free(pages[i]);
685 }
686 }
687
688 #if defined(__aarch64__)
689 /* XXXMDC necessary? */
690 dsb(sy);
691 #endif
692
693 bus_dmamap_unload(bi->pagelist_dma_tag, bi->pagelist_dma_map);
694 bus_dmamem_free(bi->pagelist_dma_tag, bi->pagelist, bi->pagelist_dma_map);
695 bus_dma_tag_destroy(bi->pagelist_dma_tag);
696
697 free(bi, M_VCPAGELIST);
698 }
699