217 static int __init ps3_mm_get_repository_highmem(struct mem_region *r)  in ps3_mm_get_repository_highmem()  argument
223 	result = ps3_repository_read_highmem_info(0, &r->base, &r->size);  in ps3_mm_get_repository_highmem()
228 	if (!r->base || !r->size) {  in ps3_mm_get_repository_highmem()
233 	r->offset = r->base - map.rm.size;  in ps3_mm_get_repository_highmem()
236 	    __func__, __LINE__, r->base, r->size);  in ps3_mm_get_repository_highmem()
243 	r->size = r->base = r->offset = 0;  in ps3_mm_get_repository_highmem()
247 static int ps3_mm_set_repository_highmem(const struct mem_region *r)  in ps3_mm_set_repository_highmem()  argument
251 	return r ? ps3_repository_write_highmem_info(0, r->base, r->size) :  in ps3_mm_set_repository_highmem()
257  * @r: pointer to a struct mem_region to accept initialized values
264 static int ps3_mm_region_create(struct mem_region *r, unsigned long size)  in ps3_mm_region_create()  argument
269 	r->size = ALIGN_DOWN(size, 1 << PAGE_SHIFT_16M);  in ps3_mm_region_create()
272 	DBG("%s:%d actual     %llxh\n", __func__, __LINE__, r->size);  in ps3_mm_region_create()
274 		size - r->size, (size - r->size) / 1024 / 1024);  in ps3_mm_region_create()
276 	if (r->size == 0) {  in ps3_mm_region_create()
282 	result = lv1_allocate_memory(r->size, PAGE_SHIFT_16M, 0,  in ps3_mm_region_create()
283 		ALLOCATE_MEMORY_TRY_ALT_UNIT, &r->base, &muid);  in ps3_mm_region_create()
285 	if (result || r->base < map.rm.size) {  in ps3_mm_region_create()
291 	r->destroy = 1;  in ps3_mm_region_create()
292 	r->offset = r->base - map.rm.size;  in ps3_mm_region_create()
296 	r->size = r->base = r->offset = 0;  in ps3_mm_region_create()
302  * @r: pointer to struct mem_region
305 static void ps3_mm_region_destroy(struct mem_region *r)  in ps3_mm_region_destroy()  argument
309 	if (!r->destroy) {  in ps3_mm_region_destroy()
313 	if (r->base) {  in ps3_mm_region_destroy()
314 		result = lv1_release_memory(r->base);  in ps3_mm_region_destroy()
320 		r->size = r->base = r->offset = 0;  in ps3_mm_region_destroy()
333  * @r: pointer to dma region structure
337 static unsigned long dma_sb_lpar_to_bus(struct ps3_dma_region *r,  in dma_sb_lpar_to_bus()  argument
342 	BUG_ON(lpar_addr < r->offset);  in dma_sb_lpar_to_bus()
343 	BUG_ON(lpar_addr >= r->offset + r->len);  in dma_sb_lpar_to_bus()
344 	return r->bus_addr + lpar_addr - r->offset;  in dma_sb_lpar_to_bus()
348 static void  __maybe_unused _dma_dump_region(const struct ps3_dma_region *r,  in _dma_dump_region()  argument
351 	DBG("%s:%d: dev        %llu:%llu\n", func, line, r->dev->bus_id,  in _dma_dump_region()
352 		r->dev->dev_id);  in _dma_dump_region()
353 	DBG("%s:%d: page_size  %u\n", func, line, r->page_size);  in _dma_dump_region()
354 	DBG("%s:%d: bus_addr   %lxh\n", func, line, r->bus_addr);  in _dma_dump_region()
355 	DBG("%s:%d: len        %lxh\n", func, line, r->len);  in _dma_dump_region()
356 	DBG("%s:%d: offset     %lxh\n", func, line, r->offset);  in _dma_dump_region()
386 	DBG("%s:%d: r.dev        %llu:%llu\n", func, line,  in _dma_dump_chunk()
388 	DBG("%s:%d: r.bus_addr   %lxh\n", func, line, c->region->bus_addr);  in _dma_dump_chunk()
389 	DBG("%s:%d: r.page_size  %u\n", func, line, c->region->page_size);  in _dma_dump_chunk()
390 	DBG("%s:%d: r.len        %lxh\n", func, line, c->region->len);  in _dma_dump_chunk()
391 	DBG("%s:%d: r.offset     %lxh\n", func, line, c->region->offset);  in _dma_dump_chunk()
397 static struct dma_chunk * dma_find_chunk(struct ps3_dma_region *r,  in dma_find_chunk()  argument
401 	unsigned long aligned_bus = ALIGN_DOWN(bus_addr, 1 << r->page_size);  in dma_find_chunk()
403 					      1 << r->page_size);  in dma_find_chunk()
405 	list_for_each_entry(c, &r->chunk_list.head, link) {  in dma_find_chunk()
426 static struct dma_chunk *dma_find_chunk_lpar(struct ps3_dma_region *r,  in dma_find_chunk_lpar()  argument
430 	unsigned long aligned_lpar = ALIGN_DOWN(lpar_addr, 1 << r->page_size);  in dma_find_chunk_lpar()
432 					      1 << r->page_size);  in dma_find_chunk_lpar()
434 	list_for_each_entry(c, &r->chunk_list.head, link) {  in dma_find_chunk_lpar()
476 	struct ps3_dma_region *r = c->region;  in dma_ioc0_free_chunk()  local
479 	for (iopage = 0; iopage < (c->len >> r->page_size); iopage++) {  in dma_ioc0_free_chunk()
480 		offset = (1 << r->page_size) * iopage;  in dma_ioc0_free_chunk()
485 				       r->ioid,  in dma_ioc0_free_chunk()
490 		    r->ioid);  in dma_ioc0_free_chunk()
504  * @r: Pointer to a struct ps3_dma_region.
513 static int dma_sb_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,  in dma_sb_map_pages()  argument
525 	c->region = r;  in dma_sb_map_pages()
527 	c->bus_addr = dma_sb_lpar_to_bus(r, c->lpar_addr);  in dma_sb_map_pages()
540 	list_add(&c->link, &r->chunk_list.head);  in dma_sb_map_pages()
553 static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,  in dma_ioc0_map_pages()  argument
570 	c->region = r;  in dma_ioc0_map_pages()
574 	if (list_empty(&r->chunk_list.head)) {  in dma_ioc0_map_pages()
576 		c->bus_addr = r->bus_addr;  in dma_ioc0_map_pages()
579 		last  = list_entry(r->chunk_list.head.next,  in dma_ioc0_map_pages()
589 	pages = len >> r->page_size;  in dma_ioc0_map_pages()
591 	    r->page_size, r->len, pages, iopte_flag);  in dma_ioc0_map_pages()
593 		offset = (1 << r->page_size) * iopage;  in dma_ioc0_map_pages()
597 				       r->ioid,  in dma_ioc0_map_pages()
606 		    r->ioid);  in dma_ioc0_map_pages()
610 	list_add(&c->link, &r->chunk_list.head);  in dma_ioc0_map_pages()
621 			      r->ioid,  in dma_ioc0_map_pages()
632  * @r: Pointer to a struct ps3_dma_region.
638 static int dma_sb_region_create(struct ps3_dma_region *r)  in dma_sb_region_create()  argument
645 	BUG_ON(!r);  in dma_sb_region_create()
647 	if (!r->dev->bus_id) {  in dma_sb_region_create()
649 			r->dev->bus_id, r->dev->dev_id);  in dma_sb_region_create()
654 	    __LINE__, r->len, r->page_size, r->offset);  in dma_sb_region_create()
656 	BUG_ON(!r->len);  in dma_sb_region_create()
657 	BUG_ON(!r->page_size);  in dma_sb_region_create()
658 	BUG_ON(!r->region_ops);  in dma_sb_region_create()
660 	INIT_LIST_HEAD(&r->chunk_list.head);  in dma_sb_region_create()
661 	spin_lock_init(&r->chunk_list.lock);  in dma_sb_region_create()
663 	result = lv1_allocate_device_dma_region(r->dev->bus_id, r->dev->dev_id,  in dma_sb_region_create()
664 		roundup_pow_of_two(r->len), r->page_size, r->region_type,  in dma_sb_region_create()
666 	r->bus_addr = bus_addr;  in dma_sb_region_create()
671 		r->len = r->bus_addr = 0;  in dma_sb_region_create()
677 static int dma_ioc0_region_create(struct ps3_dma_region *r)  in dma_ioc0_region_create()  argument
682 	INIT_LIST_HEAD(&r->chunk_list.head);  in dma_ioc0_region_create()
683 	spin_lock_init(&r->chunk_list.lock);  in dma_ioc0_region_create()
686 					 r->len,  in dma_ioc0_region_create()
687 					 r->page_size,  in dma_ioc0_region_create()
689 	r->bus_addr = bus_addr;  in dma_ioc0_region_create()
693 		r->len = r->bus_addr = 0;  in dma_ioc0_region_create()
696 	    r->len, r->page_size, r->bus_addr);  in dma_ioc0_region_create()
702  * @r: Pointer to a struct ps3_dma_region.
708 static int dma_sb_region_free(struct ps3_dma_region *r)  in dma_sb_region_free()  argument
714 	BUG_ON(!r);  in dma_sb_region_free()
716 	if (!r->dev->bus_id) {  in dma_sb_region_free()
718 			r->dev->bus_id, r->dev->dev_id);  in dma_sb_region_free()
722 	list_for_each_entry_safe(c, tmp, &r->chunk_list.head, link) {  in dma_sb_region_free()
727 	result = lv1_free_device_dma_region(r->dev->bus_id, r->dev->dev_id,  in dma_sb_region_free()
728 		r->bus_addr);  in dma_sb_region_free()
734 	r->bus_addr = 0;  in dma_sb_region_free()
739 static int dma_ioc0_region_free(struct ps3_dma_region *r)  in dma_ioc0_region_free()  argument
745 	list_for_each_entry_safe(c, n, &r->chunk_list.head, link) {  in dma_ioc0_region_free()
750 	result = lv1_release_io_segment(0, r->bus_addr);  in dma_ioc0_region_free()
756 	r->bus_addr = 0;  in dma_ioc0_region_free()
764  * @r: Pointer to a struct ps3_dma_region.
773 static int dma_sb_map_area(struct ps3_dma_region *r, unsigned long virt_addr,  in dma_sb_map_area()  argument
782 	unsigned long aligned_phys = ALIGN_DOWN(phys_addr, 1 << r->page_size);  in dma_sb_map_area()
784 					      1 << r->page_size);  in dma_sb_map_area()
785 	*bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));  in dma_sb_map_area()
801 	spin_lock_irqsave(&r->chunk_list.lock, flags);  in dma_sb_map_area()
802 	c = dma_find_chunk(r, *bus_addr, len);  in dma_sb_map_area()
808 		spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_sb_map_area()
812 	result = dma_sb_map_pages(r, aligned_phys, aligned_len, &c, iopte_flag);  in dma_sb_map_area()
818 		spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_sb_map_area()
824 	spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_sb_map_area()
828 static int dma_ioc0_map_area(struct ps3_dma_region *r, unsigned long virt_addr,  in dma_ioc0_map_area()  argument
837 	unsigned long aligned_phys = ALIGN_DOWN(phys_addr, 1 << r->page_size);  in dma_ioc0_map_area()
839 					      1 << r->page_size);  in dma_ioc0_map_area()
846 	spin_lock_irqsave(&r->chunk_list.lock, flags);  in dma_ioc0_map_area()
847 	c = dma_find_chunk_lpar(r, ps3_mm_phys_to_lpar(phys_addr), len);  in dma_ioc0_map_area()
854 		spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_ioc0_map_area()
858 	result = dma_ioc0_map_pages(r, aligned_phys, aligned_len, &c,  in dma_ioc0_map_area()
865 		spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_ioc0_map_area()
873 	spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_ioc0_map_area()
879  * @r: Pointer to a struct ps3_dma_region.
886 static int dma_sb_unmap_area(struct ps3_dma_region *r, dma_addr_t bus_addr,  in dma_sb_unmap_area()  argument
892 	spin_lock_irqsave(&r->chunk_list.lock, flags);  in dma_sb_unmap_area()
893 	c = dma_find_chunk(r, bus_addr, len);  in dma_sb_unmap_area()
897 			1 << r->page_size);  in dma_sb_unmap_area()
899 			- aligned_bus, 1 << r->page_size);  in dma_sb_unmap_area()
918 	spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_sb_unmap_area()
922 static int dma_ioc0_unmap_area(struct ps3_dma_region *r,  in dma_ioc0_unmap_area()  argument
929 	spin_lock_irqsave(&r->chunk_list.lock, flags);  in dma_ioc0_unmap_area()
930 	c = dma_find_chunk(r, bus_addr, len);  in dma_ioc0_unmap_area()
934 							1 << r->page_size);  in dma_ioc0_unmap_area()
937 						      1 << r->page_size);  in dma_ioc0_unmap_area()
956 	spin_unlock_irqrestore(&r->chunk_list.lock, flags);  in dma_ioc0_unmap_area()
963  * @r: Pointer to a struct ps3_dma_region.
969 static int dma_sb_region_create_linear(struct ps3_dma_region *r)  in dma_sb_region_create_linear()  argument
975 	if (r->len > 16*1024*1024) {	/* FIXME: need proper fix */  in dma_sb_region_create_linear()
977 		if (r->page_size != PS3_DMA_16M) {  in dma_sb_region_create_linear()
980 			r->page_size = PS3_DMA_16M;  in dma_sb_region_create_linear()
981 			r->len = ALIGN(r->len, 1 << r->page_size);  in dma_sb_region_create_linear()
985 	result = dma_sb_region_create(r);  in dma_sb_region_create_linear()
988 	if (r->offset < map.rm.size) {  in dma_sb_region_create_linear()
990 		virt_addr = map.rm.base + r->offset;  in dma_sb_region_create_linear()
991 		len = map.rm.size - r->offset;  in dma_sb_region_create_linear()
992 		if (len > r->len)  in dma_sb_region_create_linear()
993 			len = r->len;  in dma_sb_region_create_linear()
994 		result = dma_sb_map_area(r, virt_addr, len, &tmp,  in dma_sb_region_create_linear()
1000 	if (r->offset + r->len > map.rm.size) {  in dma_sb_region_create_linear()
1003 		len = r->len;  in dma_sb_region_create_linear()
1004 		if (r->offset >= map.rm.size)  in dma_sb_region_create_linear()
1005 			virt_addr += r->offset - map.rm.size;  in dma_sb_region_create_linear()
1007 			len -= map.rm.size - r->offset;  in dma_sb_region_create_linear()
1008 		result = dma_sb_map_area(r, virt_addr, len, &tmp,  in dma_sb_region_create_linear()
1019  * @r: Pointer to a struct ps3_dma_region.
1024 static int dma_sb_region_free_linear(struct ps3_dma_region *r)  in dma_sb_region_free_linear()  argument
1030 	if (r->offset < map.rm.size) {  in dma_sb_region_free_linear()
1032 		lpar_addr = map.rm.base + r->offset;  in dma_sb_region_free_linear()
1033 		len = map.rm.size - r->offset;  in dma_sb_region_free_linear()
1034 		if (len > r->len)  in dma_sb_region_free_linear()
1035 			len = r->len;  in dma_sb_region_free_linear()
1036 		bus_addr = dma_sb_lpar_to_bus(r, lpar_addr);  in dma_sb_region_free_linear()
1037 		result = dma_sb_unmap_area(r, bus_addr, len);  in dma_sb_region_free_linear()
1041 	if (r->offset + r->len > map.rm.size) {  in dma_sb_region_free_linear()
1044 		len = r->len;  in dma_sb_region_free_linear()
1045 		if (r->offset >= map.rm.size)  in dma_sb_region_free_linear()
1046 			lpar_addr += r->offset - map.rm.size;  in dma_sb_region_free_linear()
1048 			len -= map.rm.size - r->offset;  in dma_sb_region_free_linear()
1049 		bus_addr = dma_sb_lpar_to_bus(r, lpar_addr);  in dma_sb_region_free_linear()
1050 		result = dma_sb_unmap_area(r, bus_addr, len);  in dma_sb_region_free_linear()
1054 	result = dma_sb_region_free(r);  in dma_sb_region_free_linear()
1062  * @r: Pointer to a struct ps3_dma_region.
1072 static int dma_sb_map_area_linear(struct ps3_dma_region *r,  in dma_sb_map_area_linear()  argument
1078 	*bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));  in dma_sb_map_area_linear()
1084  * @r: Pointer to a struct ps3_dma_region.
1091 static int dma_sb_unmap_area_linear(struct ps3_dma_region *r,  in dma_sb_unmap_area_linear()  argument
1119 	struct ps3_dma_region *r, enum ps3_dma_page_size page_size,  in ps3_dma_region_init()  argument
1127 	r->dev = dev;  in ps3_dma_region_init()
1128 	r->page_size = page_size;  in ps3_dma_region_init()
1129 	r->region_type = region_type;  in ps3_dma_region_init()
1130 	r->offset = lpar_addr;  in ps3_dma_region_init()
1131 	if (r->offset >= map.rm.size)  in ps3_dma_region_init()
1132 		r->offset -= map.r1.offset;  in ps3_dma_region_init()
1133 	r->len = len ? len : ALIGN(map.total, 1 << r->page_size);  in ps3_dma_region_init()
1135 	dev->core.dma_mask = &r->dma_mask;  in ps3_dma_region_init()
1147 		r->region_ops =  (USE_DYNAMIC_DMA)  in ps3_dma_region_init()
1152 		r->region_ops = &ps3_dma_ioc0_region_ops;  in ps3_dma_region_init()
1162 int ps3_dma_region_create(struct ps3_dma_region *r)  in ps3_dma_region_create()  argument
1164 	BUG_ON(!r);  in ps3_dma_region_create()
1165 	BUG_ON(!r->region_ops);  in ps3_dma_region_create()
1166 	BUG_ON(!r->region_ops->create);  in ps3_dma_region_create()
1167 	return r->region_ops->create(r);  in ps3_dma_region_create()
1171 int ps3_dma_region_free(struct ps3_dma_region *r)  in ps3_dma_region_free()  argument
1173 	BUG_ON(!r);  in ps3_dma_region_free()
1174 	BUG_ON(!r->region_ops);  in ps3_dma_region_free()
1175 	BUG_ON(!r->region_ops->free);  in ps3_dma_region_free()
1176 	return r->region_ops->free(r);  in ps3_dma_region_free()
1180 int ps3_dma_map(struct ps3_dma_region *r, unsigned long virt_addr,  in ps3_dma_map()  argument
1184 	return r->region_ops->map(r, virt_addr, len, bus_addr, iopte_flag);  in ps3_dma_map()
1187 int ps3_dma_unmap(struct ps3_dma_region *r, dma_addr_t bus_addr,  in ps3_dma_unmap()  argument
1190 	return r->region_ops->unmap(r, bus_addr, len);  in ps3_dma_unmap()