xref: /linux/arch/powerpc/platforms/cell/iommu.c (revision f24e9f586b377749dff37554696cf3a105540c94)
1 /*
2  * IOMMU implementation for Cell Broadband Processor Architecture
3  * We just establish a linear mapping at boot by setting all the
4  * IOPT cache entries in the CPU.
5  * The mapping functions should be identical to pci_direct_iommu,
6  * except for the handling of the high order bit that is required
7  * by the Spider bridge. These should be split into a separate
8  * file at the point where we get a different bridge chip.
9  *
10  * Copyright (C) 2005 IBM Deutschland Entwicklung GmbH,
11  *			 Arnd Bergmann <arndb@de.ibm.com>
12  *
13  * Based on linear mapping
14  * Copyright (C) 2003 Benjamin Herrenschmidt (benh@kernel.crashing.org)
15  *
16  * This program is free software; you can redistribute it and/or
17  * modify it under the terms of the GNU General Public License
18  * as published by the Free Software Foundation; either version
19  * 2 of the License, or (at your option) any later version.
20  */
21 
22 #undef DEBUG
23 
24 #include <linux/kernel.h>
25 #include <linux/pci.h>
26 #include <linux/delay.h>
27 #include <linux/string.h>
28 #include <linux/init.h>
29 #include <linux/bootmem.h>
30 #include <linux/mm.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/kernel.h>
33 #include <linux/compiler.h>
34 
35 #include <asm/sections.h>
36 #include <asm/iommu.h>
37 #include <asm/io.h>
38 #include <asm/prom.h>
39 #include <asm/pci-bridge.h>
40 #include <asm/machdep.h>
41 #include <asm/pmac_feature.h>
42 #include <asm/abs_addr.h>
43 #include <asm/system.h>
44 #include <asm/ppc-pci.h>
45 #include <asm/udbg.h>
46 
47 #include "iommu.h"
48 
49 static inline unsigned long
50 get_iopt_entry(unsigned long real_address, unsigned long ioid,
51 			 unsigned long prot)
52 {
53 	return (prot & IOPT_PROT_MASK)
54 	     | (IOPT_COHERENT)
55 	     | (IOPT_ORDER_VC)
56 	     | (real_address & IOPT_RPN_MASK)
57 	     | (ioid & IOPT_IOID_MASK);
58 }
59 
60 typedef struct {
61 	unsigned long val;
62 } ioste;
63 
64 static inline ioste
65 mk_ioste(unsigned long val)
66 {
67 	ioste ioste = { .val = val, };
68 	return ioste;
69 }
70 
71 static inline ioste
72 get_iost_entry(unsigned long iopt_base, unsigned long io_address, unsigned page_size)
73 {
74 	unsigned long ps;
75 	unsigned long iostep;
76 	unsigned long nnpt;
77 	unsigned long shift;
78 
79 	switch (page_size) {
80 	case 0x1000000:
81 		ps = IOST_PS_16M;
82 		nnpt = 0;  /* one page per segment */
83 		shift = 5; /* segment has 16 iopt entries */
84 		break;
85 
86 	case 0x100000:
87 		ps = IOST_PS_1M;
88 		nnpt = 0;  /* one page per segment */
89 		shift = 1; /* segment has 256 iopt entries */
90 		break;
91 
92 	case 0x10000:
93 		ps = IOST_PS_64K;
94 		nnpt = 0x07; /* 8 pages per io page table */
95 		shift = 0;   /* all entries are used */
96 		break;
97 
98 	case 0x1000:
99 		ps = IOST_PS_4K;
100 		nnpt = 0x7f; /* 128 pages per io page table */
101 		shift = 0;   /* all entries are used */
102 		break;
103 
104 	default: /* not a known compile time constant */
105 		{
106 			/* BUILD_BUG_ON() is not usable here */
107 			extern void __get_iost_entry_bad_page_size(void);
108 			__get_iost_entry_bad_page_size();
109 		}
110 		break;
111 	}
112 
113 	iostep = iopt_base +
114 			 /* need 8 bytes per iopte */
115 			(((io_address / page_size * 8)
116 			 /* align io page tables on 4k page boundaries */
117 				 << shift)
118 			 /* nnpt+1 pages go into each iopt */
119 				 & ~(nnpt << 12));
120 
121 	nnpt++; /* this seems to work, but the documentation is not clear
122 		   about wether we put nnpt or nnpt-1 into the ioste bits.
123 		   In theory, this can't work for 4k pages. */
124 	return mk_ioste(IOST_VALID_MASK
125 			| (iostep & IOST_PT_BASE_MASK)
126 			| ((nnpt << 5) & IOST_NNPT_MASK)
127 			| (ps & IOST_PS_MASK));
128 }
129 
130 /* compute the address of an io pte */
131 static inline unsigned long
132 get_ioptep(ioste iost_entry, unsigned long io_address)
133 {
134 	unsigned long iopt_base;
135 	unsigned long page_size;
136 	unsigned long page_number;
137 	unsigned long iopt_offset;
138 
139 	iopt_base = iost_entry.val & IOST_PT_BASE_MASK;
140 	page_size = iost_entry.val & IOST_PS_MASK;
141 
142 	/* decode page size to compute page number */
143 	page_number = (io_address & 0x0fffffff) >> (10 + 2 * page_size);
144 	/* page number is an offset into the io page table */
145 	iopt_offset = (page_number << 3) & 0x7fff8ul;
146 	return iopt_base + iopt_offset;
147 }
148 
149 /* compute the tag field of the iopt cache entry */
150 static inline unsigned long
151 get_ioc_tag(ioste iost_entry, unsigned long io_address)
152 {
153 	unsigned long iopte = get_ioptep(iost_entry, io_address);
154 
155 	return IOPT_VALID_MASK
156 	     | ((iopte & 0x00000000000000ff8ul) >> 3)
157 	     | ((iopte & 0x0000003fffffc0000ul) >> 9);
158 }
159 
160 /* compute the hashed 6 bit index for the 4-way associative pte cache */
161 static inline unsigned long
162 get_ioc_hash(ioste iost_entry, unsigned long io_address)
163 {
164 	unsigned long iopte = get_ioptep(iost_entry, io_address);
165 
166 	return ((iopte & 0x000000000000001f8ul) >> 3)
167 	     ^ ((iopte & 0x00000000000020000ul) >> 17)
168 	     ^ ((iopte & 0x00000000000010000ul) >> 15)
169 	     ^ ((iopte & 0x00000000000008000ul) >> 13)
170 	     ^ ((iopte & 0x00000000000004000ul) >> 11)
171 	     ^ ((iopte & 0x00000000000002000ul) >> 9)
172 	     ^ ((iopte & 0x00000000000001000ul) >> 7);
173 }
174 
175 /* same as above, but pretend that we have a simpler 1-way associative
176    pte cache with an 8 bit index */
177 static inline unsigned long
178 get_ioc_hash_1way(ioste iost_entry, unsigned long io_address)
179 {
180 	unsigned long iopte = get_ioptep(iost_entry, io_address);
181 
182 	return ((iopte & 0x000000000000001f8ul) >> 3)
183 	     ^ ((iopte & 0x00000000000020000ul) >> 17)
184 	     ^ ((iopte & 0x00000000000010000ul) >> 15)
185 	     ^ ((iopte & 0x00000000000008000ul) >> 13)
186 	     ^ ((iopte & 0x00000000000004000ul) >> 11)
187 	     ^ ((iopte & 0x00000000000002000ul) >> 9)
188 	     ^ ((iopte & 0x00000000000001000ul) >> 7)
189 	     ^ ((iopte & 0x0000000000000c000ul) >> 8);
190 }
191 
192 static inline ioste
193 get_iost_cache(void __iomem *base, unsigned long index)
194 {
195 	unsigned long __iomem *p = (base + IOC_ST_CACHE_DIR);
196 	return mk_ioste(in_be64(&p[index]));
197 }
198 
199 static inline void
200 set_iost_cache(void __iomem *base, unsigned long index, ioste ste)
201 {
202 	unsigned long __iomem *p = (base + IOC_ST_CACHE_DIR);
203 	pr_debug("ioste %02lx was %016lx, store %016lx", index,
204 			get_iost_cache(base, index).val, ste.val);
205 	out_be64(&p[index], ste.val);
206 	pr_debug(" now %016lx\n", get_iost_cache(base, index).val);
207 }
208 
209 static inline unsigned long
210 get_iopt_cache(void __iomem *base, unsigned long index, unsigned long *tag)
211 {
212 	unsigned long __iomem *tags = (void *)(base + IOC_PT_CACHE_DIR);
213 	unsigned long __iomem *p = (void *)(base + IOC_PT_CACHE_REG);
214 
215 	*tag = tags[index];
216 	rmb();
217 	return *p;
218 }
219 
220 static inline void
221 set_iopt_cache(void __iomem *base, unsigned long index,
222 		 unsigned long tag, unsigned long val)
223 {
224 	unsigned long __iomem *tags = base + IOC_PT_CACHE_DIR;
225 	unsigned long __iomem *p = base + IOC_PT_CACHE_REG;
226 
227 	out_be64(p, val);
228 	out_be64(&tags[index], tag);
229 }
230 
231 static inline void
232 set_iost_origin(void __iomem *base)
233 {
234 	unsigned long __iomem *p = base + IOC_ST_ORIGIN;
235 	unsigned long origin = IOSTO_ENABLE | IOSTO_SW;
236 
237 	pr_debug("iost_origin %016lx, now %016lx\n", in_be64(p), origin);
238 	out_be64(p, origin);
239 }
240 
241 static inline void
242 set_iocmd_config(void __iomem *base)
243 {
244 	unsigned long __iomem *p = base + 0xc00;
245 	unsigned long conf;
246 
247 	conf = in_be64(p);
248 	pr_debug("iost_conf %016lx, now %016lx\n", conf, conf | IOCMD_CONF_TE);
249 	out_be64(p, conf | IOCMD_CONF_TE);
250 }
251 
252 static void enable_mapping(void __iomem *base, void __iomem *mmio_base)
253 {
254 	set_iocmd_config(base);
255 	set_iost_origin(mmio_base);
256 }
257 
258 static void iommu_dev_setup_null(struct pci_dev *d) { }
259 static void iommu_bus_setup_null(struct pci_bus *b) { }
260 
261 struct cell_iommu {
262 	unsigned long base;
263 	unsigned long mmio_base;
264 	void __iomem *mapped_base;
265 	void __iomem *mapped_mmio_base;
266 };
267 
268 static struct cell_iommu cell_iommus[NR_CPUS];
269 
270 /* initialize the iommu to support a simple linear mapping
271  * for each DMA window used by any device. For now, we
272  * happen to know that there is only one DMA window in use,
273  * starting at iopt_phys_offset. */
274 static void cell_do_map_iommu(struct cell_iommu *iommu,
275 			      unsigned int ioid,
276 			      unsigned long map_start,
277 			      unsigned long map_size)
278 {
279 	unsigned long io_address, real_address;
280 	void __iomem *ioc_base, *ioc_mmio_base;
281 	ioste ioste;
282 	unsigned long index;
283 
284 	/* we pretend the io page table was at a very high address */
285 	const unsigned long fake_iopt = 0x10000000000ul;
286 	const unsigned long io_page_size = 0x1000000; /* use 16M pages */
287 	const unsigned long io_segment_size = 0x10000000; /* 256M */
288 
289 	ioc_base = iommu->mapped_base;
290 	ioc_mmio_base = iommu->mapped_mmio_base;
291 
292 	for (real_address = 0, io_address = map_start;
293 	     io_address <= map_start + map_size;
294 	     real_address += io_page_size, io_address += io_page_size) {
295 		ioste = get_iost_entry(fake_iopt, io_address, io_page_size);
296 		if ((real_address % io_segment_size) == 0) /* segment start */
297 			set_iost_cache(ioc_mmio_base,
298 				       io_address >> 28, ioste);
299 		index = get_ioc_hash_1way(ioste, io_address);
300 		pr_debug("addr %08lx, index %02lx, ioste %016lx\n",
301 					 io_address, index, ioste.val);
302 		set_iopt_cache(ioc_mmio_base,
303 			get_ioc_hash_1way(ioste, io_address),
304 			get_ioc_tag(ioste, io_address),
305 			get_iopt_entry(real_address, ioid, IOPT_PROT_RW));
306 	}
307 }
308 
309 static void iommu_devnode_setup(struct device_node *d)
310 {
311 	const unsigned int *ioid;
312 	unsigned long map_start, map_size, token;
313 	const unsigned long *dma_window;
314 	struct cell_iommu *iommu;
315 
316 	ioid = get_property(d, "ioid", NULL);
317 	if (!ioid)
318 		pr_debug("No ioid entry found !\n");
319 
320 	dma_window = get_property(d, "ibm,dma-window", NULL);
321 	if (!dma_window)
322 		pr_debug("No ibm,dma-window entry found !\n");
323 
324 	map_start = dma_window[1];
325 	map_size = dma_window[2];
326 	token = dma_window[0] >> 32;
327 
328 	iommu = &cell_iommus[token];
329 
330 	cell_do_map_iommu(iommu, *ioid, map_start, map_size);
331 }
332 
333 static void iommu_bus_setup(struct pci_bus *b)
334 {
335 	struct device_node *d = (struct device_node *)b->sysdata;
336 	iommu_devnode_setup(d);
337 }
338 
339 
340 static int cell_map_iommu_hardcoded(int num_nodes)
341 {
342 	struct cell_iommu *iommu = NULL;
343 
344 	pr_debug("%s(%d): Using hardcoded defaults\n", __FUNCTION__, __LINE__);
345 
346 	/* node 0 */
347 	iommu = &cell_iommus[0];
348 	iommu->mapped_base = ioremap(0x20000511000, 0x1000);
349 	iommu->mapped_mmio_base = ioremap(0x20000510000, 0x1000);
350 
351 	enable_mapping(iommu->mapped_base, iommu->mapped_mmio_base);
352 
353 	cell_do_map_iommu(iommu, 0x048a,
354 			  0x20000000ul,0x20000000ul);
355 
356 	if (num_nodes < 2)
357 		return 0;
358 
359 	/* node 1 */
360 	iommu = &cell_iommus[1];
361 	iommu->mapped_base = ioremap(0x30000511000, 0x1000);
362 	iommu->mapped_mmio_base = ioremap(0x30000510000, 0x1000);
363 
364 	enable_mapping(iommu->mapped_base, iommu->mapped_mmio_base);
365 
366 	cell_do_map_iommu(iommu, 0x048a,
367 			  0x20000000,0x20000000ul);
368 
369 	return 0;
370 }
371 
372 
373 static int cell_map_iommu(void)
374 {
375 	unsigned int num_nodes = 0;
376 	const unsigned int *node_id;
377 	const unsigned long *base, *mmio_base;
378 	struct device_node *dn;
379 	struct cell_iommu *iommu = NULL;
380 
381 	/* determine number of nodes (=iommus) */
382 	pr_debug("%s(%d): determining number of nodes...", __FUNCTION__, __LINE__);
383 	for(dn = of_find_node_by_type(NULL, "cpu");
384 	    dn;
385 	    dn = of_find_node_by_type(dn, "cpu")) {
386 		node_id = get_property(dn, "node-id", NULL);
387 
388 		if (num_nodes < *node_id)
389 			num_nodes = *node_id;
390 		}
391 
392 	num_nodes++;
393 	pr_debug("%i found.\n", num_nodes);
394 
395 	/* map the iommu registers for each node */
396 	pr_debug("%s(%d): Looping through nodes\n", __FUNCTION__, __LINE__);
397 	for(dn = of_find_node_by_type(NULL, "cpu");
398 	    dn;
399 	    dn = of_find_node_by_type(dn, "cpu")) {
400 
401 		node_id = get_property(dn, "node-id", NULL);
402 		base = get_property(dn, "ioc-cache", NULL);
403 		mmio_base = get_property(dn, "ioc-translation", NULL);
404 
405 		if (!base || !mmio_base || !node_id)
406 			return cell_map_iommu_hardcoded(num_nodes);
407 
408 		iommu = &cell_iommus[*node_id];
409 		iommu->base = *base;
410 		iommu->mmio_base = *mmio_base;
411 
412 		iommu->mapped_base = ioremap(*base, 0x1000);
413 		iommu->mapped_mmio_base = ioremap(*mmio_base, 0x1000);
414 
415 		enable_mapping(iommu->mapped_base,
416 			       iommu->mapped_mmio_base);
417 
418 		/* everything else will be done in iommu_bus_setup */
419 	}
420 
421 	return 1;
422 }
423 
424 static void *cell_alloc_coherent(struct device *hwdev, size_t size,
425 			   dma_addr_t *dma_handle, gfp_t flag)
426 {
427 	void *ret;
428 
429 	ret = (void *)__get_free_pages(flag, get_order(size));
430 	if (ret != NULL) {
431 		memset(ret, 0, size);
432 		*dma_handle = virt_to_abs(ret) | CELL_DMA_VALID;
433 	}
434 	return ret;
435 }
436 
437 static void cell_free_coherent(struct device *hwdev, size_t size,
438 				 void *vaddr, dma_addr_t dma_handle)
439 {
440 	free_pages((unsigned long)vaddr, get_order(size));
441 }
442 
443 static dma_addr_t cell_map_single(struct device *hwdev, void *ptr,
444 		size_t size, enum dma_data_direction direction)
445 {
446 	return virt_to_abs(ptr) | CELL_DMA_VALID;
447 }
448 
449 static void cell_unmap_single(struct device *hwdev, dma_addr_t dma_addr,
450 		size_t size, enum dma_data_direction direction)
451 {
452 }
453 
454 static int cell_map_sg(struct device *hwdev, struct scatterlist *sg,
455 		int nents, enum dma_data_direction direction)
456 {
457 	int i;
458 
459 	for (i = 0; i < nents; i++, sg++) {
460 		sg->dma_address = (page_to_phys(sg->page) + sg->offset)
461 					| CELL_DMA_VALID;
462 		sg->dma_length = sg->length;
463 	}
464 
465 	return nents;
466 }
467 
468 static void cell_unmap_sg(struct device *hwdev, struct scatterlist *sg,
469 		int nents, enum dma_data_direction direction)
470 {
471 }
472 
473 static int cell_dma_supported(struct device *dev, u64 mask)
474 {
475 	return mask < 0x100000000ull;
476 }
477 
478 static struct dma_mapping_ops cell_iommu_ops = {
479 	.alloc_coherent = cell_alloc_coherent,
480 	.free_coherent = cell_free_coherent,
481 	.map_single = cell_map_single,
482 	.unmap_single = cell_unmap_single,
483 	.map_sg = cell_map_sg,
484 	.unmap_sg = cell_unmap_sg,
485 	.dma_supported = cell_dma_supported,
486 };
487 
488 void cell_init_iommu(void)
489 {
490 	int setup_bus = 0;
491 
492 	if (of_find_node_by_path("/mambo")) {
493 		pr_info("Not using iommu on systemsim\n");
494 	} else {
495 
496 		if (!(of_chosen &&
497 		      get_property(of_chosen, "linux,iommu-off", NULL)))
498 			setup_bus = cell_map_iommu();
499 
500 		if (setup_bus) {
501 			pr_debug("%s: IOMMU mapping activated\n", __FUNCTION__);
502 			ppc_md.iommu_dev_setup = iommu_dev_setup_null;
503 			ppc_md.iommu_bus_setup = iommu_bus_setup;
504 		} else {
505 			pr_debug("%s: IOMMU mapping activated, "
506 				 "no device action necessary\n", __FUNCTION__);
507 			/* Direct I/O, IOMMU off */
508 			ppc_md.iommu_dev_setup = iommu_dev_setup_null;
509 			ppc_md.iommu_bus_setup = iommu_bus_setup_null;
510 		}
511 	}
512 
513 	pci_dma_ops = cell_iommu_ops;
514 }
515