xref: /linux/arch/arm/mm/cache-uniphier.c (revision 0ea5c948cb64bab5bc7a5516774eb8536f05aa0d)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2015-2016 Socionext Inc.
4  *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5  */
6 
7 #define pr_fmt(fmt)		"uniphier: " fmt
8 
9 #include <linux/bitops.h>
10 #include <linux/init.h>
11 #include <linux/io.h>
12 #include <linux/log2.h>
13 #include <linux/of_address.h>
14 #include <linux/slab.h>
15 #include <asm/hardware/cache-uniphier.h>
16 #include <asm/outercache.h>
17 
18 /* control registers */
19 #define UNIPHIER_SSCC		0x0	/* Control Register */
20 #define    UNIPHIER_SSCC_BST			BIT(20)	/* UCWG burst read */
21 #define    UNIPHIER_SSCC_ACT			BIT(19)	/* Inst-Data separate */
22 #define    UNIPHIER_SSCC_WTG			BIT(18)	/* WT gathering on */
23 #define    UNIPHIER_SSCC_PRD			BIT(17)	/* enable pre-fetch */
24 #define    UNIPHIER_SSCC_ON			BIT(0)	/* enable cache */
25 #define UNIPHIER_SSCLPDAWCR	0x30	/* Unified/Data Active Way Control */
26 #define UNIPHIER_SSCLPIAWCR	0x34	/* Instruction Active Way Control */
27 
28 /* revision registers */
29 #define UNIPHIER_SSCID		0x0	/* ID Register */
30 
31 /* operation registers */
32 #define UNIPHIER_SSCOPE		0x244	/* Cache Operation Primitive Entry */
33 #define    UNIPHIER_SSCOPE_CM_INV		0x0	/* invalidate */
34 #define    UNIPHIER_SSCOPE_CM_CLEAN		0x1	/* clean */
35 #define    UNIPHIER_SSCOPE_CM_FLUSH		0x2	/* flush */
36 #define    UNIPHIER_SSCOPE_CM_SYNC		0x8	/* sync (drain bufs) */
37 #define    UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH	0x9	/* flush p-fetch buf */
38 #define UNIPHIER_SSCOQM		0x248	/* Cache Operation Queue Mode */
39 #define    UNIPHIER_SSCOQM_S_MASK		(0x3 << 17)
40 #define    UNIPHIER_SSCOQM_S_RANGE		(0x0 << 17)
41 #define    UNIPHIER_SSCOQM_S_ALL		(0x1 << 17)
42 #define    UNIPHIER_SSCOQM_CE			BIT(15)	/* notify completion */
43 #define    UNIPHIER_SSCOQM_CM_INV		0x0	/* invalidate */
44 #define    UNIPHIER_SSCOQM_CM_CLEAN		0x1	/* clean */
45 #define    UNIPHIER_SSCOQM_CM_FLUSH		0x2	/* flush */
46 #define UNIPHIER_SSCOQAD	0x24c	/* Cache Operation Queue Address */
47 #define UNIPHIER_SSCOQSZ	0x250	/* Cache Operation Queue Size */
48 #define UNIPHIER_SSCOPPQSEF	0x25c	/* Cache Operation Queue Set Complete*/
49 #define    UNIPHIER_SSCOPPQSEF_FE		BIT(1)
50 #define    UNIPHIER_SSCOPPQSEF_OE		BIT(0)
51 #define UNIPHIER_SSCOLPQS	0x260	/* Cache Operation Queue Status */
52 #define    UNIPHIER_SSCOLPQS_EF			BIT(2)
53 #define    UNIPHIER_SSCOLPQS_EST		BIT(1)
54 #define    UNIPHIER_SSCOLPQS_QST		BIT(0)
55 
56 /* Is the operation region specified by address range? */
57 #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
58 		((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
59 
60 /**
61  * struct uniphier_cache_data - UniPhier outer cache specific data
62  *
63  * @ctrl_base: virtual base address of control registers
64  * @rev_base: virtual base address of revision registers
65  * @op_base: virtual base address of operation registers
66  * @way_ctrl_base: virtual address of the way control registers for this
67  *	SoC revision
68  * @way_mask: each bit specifies if the way is present
69  * @nsets: number of associativity sets
70  * @line_size: line size in bytes
71  * @range_op_max_size: max size that can be handled by a single range operation
72  * @list: list node to include this level in the whole cache hierarchy
73  */
74 struct uniphier_cache_data {
75 	void __iomem *ctrl_base;
76 	void __iomem *rev_base;
77 	void __iomem *op_base;
78 	void __iomem *way_ctrl_base;
79 	u32 way_mask;
80 	u32 nsets;
81 	u32 line_size;
82 	u32 range_op_max_size;
83 	struct list_head list;
84 };
85 
86 /*
87  * List of the whole outer cache hierarchy.  This list is only modified during
88  * the early boot stage, so no mutex is taken for the access to the list.
89  */
90 static LIST_HEAD(uniphier_cache_list);
91 
92 /**
93  * __uniphier_cache_sync - perform a sync point for a particular cache level
94  *
95  * @data: cache controller specific data
96  */
__uniphier_cache_sync(struct uniphier_cache_data * data)97 static void __uniphier_cache_sync(struct uniphier_cache_data *data)
98 {
99 	/* This sequence need not be atomic.  Do not disable IRQ. */
100 	writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
101 		       data->op_base + UNIPHIER_SSCOPE);
102 	/* need a read back to confirm */
103 	readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
104 }
105 
106 /**
107  * __uniphier_cache_maint_common - run a queue operation for a particular level
108  *
109  * @data: cache controller specific data
110  * @start: start address of range operation (don't care for "all" operation)
111  * @size: data size of range operation (don't care for "all" operation)
112  * @operation: flags to specify the desired cache operation
113  */
__uniphier_cache_maint_common(struct uniphier_cache_data * data,unsigned long start,unsigned long size,u32 operation)114 static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
115 					  unsigned long start,
116 					  unsigned long size,
117 					  u32 operation)
118 {
119 	unsigned long flags;
120 
121 	/*
122 	 * No spin lock is necessary here because:
123 	 *
124 	 * [1] This outer cache controller is able to accept maintenance
125 	 * operations from multiple CPUs at a time in an SMP system; if a
126 	 * maintenance operation is under way and another operation is issued,
127 	 * the new one is stored in the queue.  The controller performs one
128 	 * operation after another.  If the queue is full, the status register,
129 	 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
130 	 * failed.  The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
131 	 * different instances for each CPU, i.e. each CPU can track the status
132 	 * of the maintenance operations triggered by itself.
133 	 *
134 	 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
135 	 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
136 	 * guarantees the registration sequence is atomic; the write access to
137 	 * them are arbitrated by the hardware.  The first accessor to the
138 	 * register, UNIPHIER_SSCOQM, holds the access right and it is released
139 	 * by reading the status register, UNIPHIER_SSCOPPQSEF.  While one CPU
140 	 * is holding the access right, other CPUs fail to register operations.
141 	 * One CPU should not hold the access right for a long time, so local
142 	 * IRQs should be disabled while the following sequence.
143 	 */
144 	local_irq_save(flags);
145 
146 	/* clear the complete notification flag */
147 	writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
148 
149 	do {
150 		/* set cache operation */
151 		writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
152 			       data->op_base + UNIPHIER_SSCOQM);
153 
154 		/* set address range if needed */
155 		if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
156 			writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
157 			writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
158 		}
159 	} while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
160 			  (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
161 
162 	/* wait until the operation is completed */
163 	while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
164 		      UNIPHIER_SSCOLPQS_EF))
165 		cpu_relax();
166 
167 	local_irq_restore(flags);
168 }
169 
__uniphier_cache_maint_all(struct uniphier_cache_data * data,u32 operation)170 static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
171 				       u32 operation)
172 {
173 	__uniphier_cache_maint_common(data, 0, 0,
174 				      UNIPHIER_SSCOQM_S_ALL | operation);
175 
176 	__uniphier_cache_sync(data);
177 }
178 
__uniphier_cache_maint_range(struct uniphier_cache_data * data,unsigned long start,unsigned long end,u32 operation)179 static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
180 					 unsigned long start, unsigned long end,
181 					 u32 operation)
182 {
183 	unsigned long size;
184 
185 	/*
186 	 * If the start address is not aligned,
187 	 * perform a cache operation for the first cache-line
188 	 */
189 	start = start & ~(data->line_size - 1);
190 
191 	size = end - start;
192 
193 	if (unlikely(size >= (unsigned long)(-data->line_size))) {
194 		/* this means cache operation for all range */
195 		__uniphier_cache_maint_all(data, operation);
196 		return;
197 	}
198 
199 	/*
200 	 * If the end address is not aligned,
201 	 * perform a cache operation for the last cache-line
202 	 */
203 	size = ALIGN(size, data->line_size);
204 
205 	while (size) {
206 		unsigned long chunk_size = min_t(unsigned long, size,
207 						 data->range_op_max_size);
208 
209 		__uniphier_cache_maint_common(data, start, chunk_size,
210 					UNIPHIER_SSCOQM_S_RANGE | operation);
211 
212 		start += chunk_size;
213 		size -= chunk_size;
214 	}
215 
216 	__uniphier_cache_sync(data);
217 }
218 
__uniphier_cache_enable(struct uniphier_cache_data * data,bool on)219 static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
220 {
221 	u32 val = 0;
222 
223 	if (on)
224 		val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
225 
226 	writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
227 }
228 
__uniphier_cache_set_active_ways(struct uniphier_cache_data * data)229 static void __init __uniphier_cache_set_active_ways(
230 					struct uniphier_cache_data *data)
231 {
232 	unsigned int cpu;
233 
234 	for_each_possible_cpu(cpu)
235 		writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
236 }
237 
uniphier_cache_maint_range(unsigned long start,unsigned long end,u32 operation)238 static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
239 				       u32 operation)
240 {
241 	struct uniphier_cache_data *data;
242 
243 	list_for_each_entry(data, &uniphier_cache_list, list)
244 		__uniphier_cache_maint_range(data, start, end, operation);
245 }
246 
uniphier_cache_maint_all(u32 operation)247 static void uniphier_cache_maint_all(u32 operation)
248 {
249 	struct uniphier_cache_data *data;
250 
251 	list_for_each_entry(data, &uniphier_cache_list, list)
252 		__uniphier_cache_maint_all(data, operation);
253 }
254 
uniphier_cache_inv_range(unsigned long start,unsigned long end)255 static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
256 {
257 	uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
258 }
259 
uniphier_cache_clean_range(unsigned long start,unsigned long end)260 static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
261 {
262 	uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
263 }
264 
uniphier_cache_flush_range(unsigned long start,unsigned long end)265 static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
266 {
267 	uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
268 }
269 
uniphier_cache_inv_all(void)270 static void __init uniphier_cache_inv_all(void)
271 {
272 	uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
273 }
274 
uniphier_cache_flush_all(void)275 static void uniphier_cache_flush_all(void)
276 {
277 	uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
278 }
279 
uniphier_cache_disable(void)280 static void uniphier_cache_disable(void)
281 {
282 	struct uniphier_cache_data *data;
283 
284 	list_for_each_entry_reverse(data, &uniphier_cache_list, list)
285 		__uniphier_cache_enable(data, false);
286 
287 	uniphier_cache_flush_all();
288 }
289 
uniphier_cache_enable(void)290 static void __init uniphier_cache_enable(void)
291 {
292 	struct uniphier_cache_data *data;
293 
294 	uniphier_cache_inv_all();
295 
296 	list_for_each_entry(data, &uniphier_cache_list, list) {
297 		__uniphier_cache_enable(data, true);
298 		__uniphier_cache_set_active_ways(data);
299 	}
300 }
301 
uniphier_cache_sync(void)302 static void uniphier_cache_sync(void)
303 {
304 	struct uniphier_cache_data *data;
305 
306 	list_for_each_entry(data, &uniphier_cache_list, list)
307 		__uniphier_cache_sync(data);
308 }
309 
310 static const struct of_device_id uniphier_cache_match[] __initconst = {
311 	{ .compatible = "socionext,uniphier-system-cache" },
312 	{ /* sentinel */ }
313 };
314 
__uniphier_cache_init(struct device_node * np,unsigned int * cache_level)315 static int __init __uniphier_cache_init(struct device_node *np,
316 					unsigned int *cache_level)
317 {
318 	struct uniphier_cache_data *data;
319 	u32 level, cache_size;
320 	struct device_node *next_np;
321 	int ret = 0;
322 
323 	if (!of_match_node(uniphier_cache_match, np)) {
324 		pr_err("L%d: not compatible with uniphier cache\n",
325 		       *cache_level);
326 		return -EINVAL;
327 	}
328 
329 	if (of_property_read_u32(np, "cache-level", &level)) {
330 		pr_err("L%d: cache-level is not specified\n", *cache_level);
331 		return -EINVAL;
332 	}
333 
334 	if (level != *cache_level) {
335 		pr_err("L%d: cache-level is unexpected value %d\n",
336 		       *cache_level, level);
337 		return -EINVAL;
338 	}
339 
340 	if (!of_property_read_bool(np, "cache-unified")) {
341 		pr_err("L%d: cache-unified is not specified\n", *cache_level);
342 		return -EINVAL;
343 	}
344 
345 	data = kzalloc(sizeof(*data), GFP_KERNEL);
346 	if (!data)
347 		return -ENOMEM;
348 
349 	if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
350 	    !is_power_of_2(data->line_size)) {
351 		pr_err("L%d: cache-line-size is unspecified or invalid\n",
352 		       *cache_level);
353 		ret = -EINVAL;
354 		goto err;
355 	}
356 
357 	if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
358 	    !is_power_of_2(data->nsets)) {
359 		pr_err("L%d: cache-sets is unspecified or invalid\n",
360 		       *cache_level);
361 		ret = -EINVAL;
362 		goto err;
363 	}
364 
365 	if (of_property_read_u32(np, "cache-size", &cache_size) ||
366 	    cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
367 		pr_err("L%d: cache-size is unspecified or invalid\n",
368 		       *cache_level);
369 		ret = -EINVAL;
370 		goto err;
371 	}
372 
373 	data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
374 				 0);
375 
376 	data->ctrl_base = of_iomap(np, 0);
377 	if (!data->ctrl_base) {
378 		pr_err("L%d: failed to map control register\n", *cache_level);
379 		ret = -ENOMEM;
380 		goto err;
381 	}
382 
383 	data->rev_base = of_iomap(np, 1);
384 	if (!data->rev_base) {
385 		pr_err("L%d: failed to map revision register\n", *cache_level);
386 		ret = -ENOMEM;
387 		goto err;
388 	}
389 
390 	data->op_base = of_iomap(np, 2);
391 	if (!data->op_base) {
392 		pr_err("L%d: failed to map operation register\n", *cache_level);
393 		ret = -ENOMEM;
394 		goto err;
395 	}
396 
397 	data->way_ctrl_base = data->ctrl_base + 0xc00;
398 
399 	if (*cache_level == 2) {
400 		u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
401 		/*
402 		 * The size of range operation is limited to (1 << 22) or less
403 		 * for PH-sLD8 or older SoCs.
404 		 */
405 		if (revision <= 0x16)
406 			data->range_op_max_size = (u32)1 << 22;
407 
408 		/*
409 		 * Unfortunatly, the offset address of active way control base
410 		 * varies from SoC to SoC.
411 		 */
412 		switch (revision) {
413 		case 0x11:	/* sLD3 */
414 			data->way_ctrl_base = data->ctrl_base + 0x870;
415 			break;
416 		case 0x12:	/* LD4 */
417 		case 0x16:	/* sld8 */
418 			data->way_ctrl_base = data->ctrl_base + 0x840;
419 			break;
420 		default:
421 			break;
422 		}
423 	}
424 
425 	data->range_op_max_size -= data->line_size;
426 
427 	INIT_LIST_HEAD(&data->list);
428 	list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
429 
430 	/*
431 	 * OK, this level has been successfully initialized.  Look for the next
432 	 * level cache.  Do not roll back even if the initialization of the
433 	 * next level cache fails because we want to continue with available
434 	 * cache levels.
435 	 */
436 	next_np = of_find_next_cache_node(np);
437 	if (next_np) {
438 		(*cache_level)++;
439 		ret = __uniphier_cache_init(next_np, cache_level);
440 	}
441 	of_node_put(next_np);
442 
443 	return ret;
444 err:
445 	iounmap(data->op_base);
446 	iounmap(data->rev_base);
447 	iounmap(data->ctrl_base);
448 	kfree(data);
449 
450 	return ret;
451 }
452 
uniphier_cache_init(void)453 int __init uniphier_cache_init(void)
454 {
455 	struct device_node *np = NULL;
456 	unsigned int cache_level;
457 	int ret = 0;
458 
459 	/* look for level 2 cache */
460 	while ((np = of_find_matching_node(np, uniphier_cache_match)))
461 		if (!of_property_read_u32(np, "cache-level", &cache_level) &&
462 		    cache_level == 2)
463 			break;
464 
465 	if (!np)
466 		return -ENODEV;
467 
468 	ret = __uniphier_cache_init(np, &cache_level);
469 	of_node_put(np);
470 
471 	if (ret) {
472 		/*
473 		 * Error out iif L2 initialization fails.  Continue with any
474 		 * error on L3 or outer because they are optional.
475 		 */
476 		if (cache_level == 2) {
477 			pr_err("failed to initialize L2 cache\n");
478 			return ret;
479 		}
480 
481 		cache_level--;
482 		ret = 0;
483 	}
484 
485 	outer_cache.inv_range = uniphier_cache_inv_range;
486 	outer_cache.clean_range = uniphier_cache_clean_range;
487 	outer_cache.flush_range = uniphier_cache_flush_range;
488 	outer_cache.flush_all = uniphier_cache_flush_all;
489 	outer_cache.disable = uniphier_cache_disable;
490 	outer_cache.sync = uniphier_cache_sync;
491 
492 	uniphier_cache_enable();
493 
494 	pr_info("enabled outer cache (cache level: %d)\n", cache_level);
495 
496 	return ret;
497 }
498