1 // SPDX-License-Identifier: GPL-2.0-only
3  * arch/arm/mm/cache-l2x0.c - L210/L220/L310 cache controller support
21 #include <asm/hardware/cache-l2x0.h>
22 #include <asm/hardware/cache-aurora-l2.h>
23 #include "cache-tauros3.h"
57 	/* wait for cache operation by line or way to complete */  in l2c_wait_mask()
64  * override this if they are running non-secure.
78  * register be written due to a work-around, as platforms running
79  * in non-secure mode may not be able to access this register.
120 		l2x0_data->configure(base);  in l2c_enable()
122 	l2x0_data->unlock(base, num_lock);  in l2c_enable()
155 		l2c_enable(base, l2x0_data->num_lock);  in l2c_resume()
161  * L2C-210 specific code.
163  * The L2C-2x0 PA, set/way and sync operations are atomic, but we must
164  * ensure that no background operation is running.  The way operations
172  * we use sync_reg_offset here so we can share some of this with L2C-310.
192 	if (start & (CACHE_LINE_SIZE - 1)) {  in l2c210_inv_range()
193 		start &= ~(CACHE_LINE_SIZE - 1);  in l2c210_inv_range()
198 	if (end & (CACHE_LINE_SIZE - 1)) {  in l2c210_inv_range()
199 		end &= ~(CACHE_LINE_SIZE - 1);  in l2c210_inv_range()
211 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c210_clean_range()
220 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c210_flush_range()
241 	.type = "L2C-210",
260  * L2C-220 specific code.
264  * imprecise abort.)  Never uses sync_reg_offset, so we hard-code the
267  * However, we can re-use the l2c210_resume call.
291 		unsigned long blk_end = start + min(end - start, 4096UL);  in l2c220_op_pa_range()
314 	if ((start | end) & (CACHE_LINE_SIZE - 1)) {  in l2c220_inv_range()
315 		if (start & (CACHE_LINE_SIZE - 1)) {  in l2c220_inv_range()
316 			start &= ~(CACHE_LINE_SIZE - 1);  in l2c220_inv_range()
321 		if (end & (CACHE_LINE_SIZE - 1)) {  in l2c220_inv_range()
322 			end &= ~(CACHE_LINE_SIZE - 1);  in l2c220_inv_range()
340 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c220_clean_range()
341 	if ((end - start) >= l2x0_size) {  in l2c220_clean_range()
359 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c220_flush_range()
360 	if ((end - start) >= l2x0_size) {  in l2c220_flush_range()
390 	 * Always enable non-secure access to the lockdown registers -  in l2c220_enable()
406 	.type = "L2C-220",
425  * L2C-310 specific code.
427  * Very similar to L2C-210, the PA, set/way and sync operations are atomic,
428  * and the way operations are all background tasks.  However, issuing an
440  * 588369: PL310 R0P0->R1P0, fixed R2P0.
448  * 727915: PL310 R2P0->R3P0, fixed R3P1.
449  *	Affects: clean+invalidate by way
450  *	clean and invalidate by way runs in the background, and a store can
454  * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2.
455  *	Affects: 8x64-bit (double fill) line fetches
464  * 769419: PL310 R0P0->R3P1, fixed R3P2.
472 	if ((start | end) & (CACHE_LINE_SIZE - 1)) {  in l2c310_inv_range_erratum()
479 		if (start & (CACHE_LINE_SIZE - 1)) {  in l2c310_inv_range_erratum()
480 			start &= ~(CACHE_LINE_SIZE - 1);  in l2c310_inv_range_erratum()
486 		if (end & (CACHE_LINE_SIZE - 1)) {  in l2c310_inv_range_erratum()
487 			end &= ~(CACHE_LINE_SIZE - 1);  in l2c310_inv_range_erratum()
508 		unsigned long blk_end = start + min(end - start, 4096UL);  in l2c310_flush_range_erratum()
617 			pr_info("L2C-310 enabling early BRESP for Cortex-A9\n");  in l2c310_enable()
619 			pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n");  in l2c310_enable()
628 		pr_debug("Cortex-A9 ACR=0x%08x\n", acr);  in l2c310_enable()
631 			pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n");  in l2c310_enable()
634 			pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n");  in l2c310_enable()
638 			pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n");  in l2c310_enable()
641 		pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n");  in l2c310_enable()
646 	 * Always enable non-secure access to the lockdown registers -  in l2c310_enable()
660 		pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n",  in l2c310_enable()
671 		pr_info("L2C-310 dynamic clock gating %s, standby mode %s\n",  in l2c310_enable()
686 	const char *errata[8];  in l2c310_fixup()
692 	    fns->inv_range == l2c210_inv_range) {  in l2c310_fixup()
693 		fns->inv_range = l2c310_inv_range_erratum;  in l2c310_fixup()
694 		fns->flush_range = l2c310_flush_range_erratum;  in l2c310_fixup()
701 		fns->flush_all = l2c310_flush_all_erratum;  in l2c310_fixup()
727 		pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");  in l2c310_fixup()
737 	 * If full-line-of-zeros is enabled, we must first disable it in the  in l2c310_disable()
738 	 * Cortex-A9 auxiliary control register before disabling the L2 cache.  in l2c310_disable()
750 	/* Re-enable full-line-of-zeros for Cortex-A9 */  in l2c310_resume()
762 	.type = "L2C-310",
764 	.num_lock = 8,
794 		return -ENOMEM;  in __l2c_init()
809 		pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n",  in __l2c_init()
820 			ways = 8;  in __l2c_init()
834 		/* Assume unknown chips have 8 ways */  in __l2c_init()
835 		ways = 8;  in __l2c_init()
839 	l2x0_way_mask = (1 << ways) - 1;  in __l2c_init()
845 	 * then way_size_0 would be 8k.  in __l2c_init()
847 	 * L2 cache size = number of ways * way size.  in __l2c_init()
851 	l2x0_size = ways * (data->way_size_0 << way_size_bits);  in __l2c_init()
853 	fns = data->outer_cache;  in __l2c_init()
856 	if (data->fixup)  in __l2c_init()
857 		data->fixup(l2x0_base, cache_id, &fns);  in __l2c_init()
865 	 * in non-secure mode accessing the below registers will fault.  in __l2c_init()
870 		data->enable(l2x0_base, data->num_lock);  in __l2c_init()
879 	if (data->save)  in __l2c_init()
880 		data->save(l2x0_base);  in __l2c_init()
882 	/* Re-read it in case some bits are reserved. */  in __l2c_init()
886 		data->type, ways, l2x0_size >> 10);  in __l2c_init()
888 		data->type, cache_id, aux);  in __l2c_init()
920 	if (data->save)  in l2x0_init()
921 		data->save(l2x0_base);  in l2x0_init()
934  * l2x0_cache_size_of_parse() - read cache size parameters from DT
936  * @aux_val: pointer to machine-supplied auxilary register value, to
938  * @aux_mask: pointer to machine-supplied auxilary register mask, to
955 	of_property_read_u32(np, "cache-size", &cache_size);  in l2x0_cache_size_of_parse()
956 	of_property_read_u32(np, "cache-sets", &sets);  in l2x0_cache_size_of_parse()
957 	of_property_read_u32(np, "cache-block-size", &block_size);  in l2x0_cache_size_of_parse()
958 	of_property_read_u32(np, "cache-line-size", &line_size);  in l2x0_cache_size_of_parse()
961 		return -ENODEV;  in l2x0_cache_size_of_parse()
987 	 * way size = cache size / (cache size / (sets * line size))  in l2x0_cache_size_of_parse()
988 	 * way size = sets * line size  in l2x0_cache_size_of_parse()
989 	 * associativity = ways = cache size / way size  in l2x0_cache_size_of_parse()
996 		return -EINVAL;  in l2x0_cache_size_of_parse()
1002 	pr_info("L2C OF: override way size: %d bytes (%dKB)\n",  in l2x0_cache_size_of_parse()
1007 	 * Calculates the bits 17:19 to set for way size:  in l2x0_cache_size_of_parse()
1008 	 * 512KB -> 6, 256KB -> 5, ... 16KB -> 1  in l2x0_cache_size_of_parse()
1010 	way_size_bits = ilog2(way_size >> 10) - 3;  in l2x0_cache_size_of_parse()
1012 		pr_err("L2C OF: cache way size illegal: %dKB is not mapped\n",  in l2x0_cache_size_of_parse()
1014 		return -EINVAL;  in l2x0_cache_size_of_parse()
1037 	of_property_read_u32(np, "arm,tag-latency", &tag);  in l2x0_of_parse()
1040 		val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT;  in l2x0_of_parse()
1043 	of_property_read_u32_array(np, "arm,data-latency",  in l2x0_of_parse()
1048 		val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |  in l2x0_of_parse()
1049 		       ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);  in l2x0_of_parse()
1052 	of_property_read_u32(np, "arm,dirty-latency", &dirty);  in l2x0_of_parse()
1055 		val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;  in l2x0_of_parse()
1058 	if (of_property_read_bool(np, "arm,parity-enable")) {  in l2x0_of_parse()
1061 	} else if (of_property_read_bool(np, "arm,parity-disable")) {  in l2x0_of_parse()
1065 	if (of_property_read_bool(np, "arm,shared-override")) {  in l2x0_of_parse()
1074 	if (assoc > 8) {  in l2x0_of_parse()
1076 		pr_err("l2x0 of: %d calculated, max 8\n", assoc);  in l2x0_of_parse()
1088 	.type = "L2C-210",
1108 	.type = "L2C-220",
1139 	of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));  in l2c310_of_parse()
1142 			L310_LATENCY_CTRL_RD(tag[0] - 1) |  in l2c310_of_parse()
1143 			L310_LATENCY_CTRL_WR(tag[1] - 1) |  in l2c310_of_parse()
1144 			L310_LATENCY_CTRL_SETUP(tag[2] - 1);  in l2c310_of_parse()
1146 	of_property_read_u32_array(np, "arm,data-latency",  in l2c310_of_parse()
1150 			L310_LATENCY_CTRL_RD(data[0] - 1) |  in l2c310_of_parse()
1151 			L310_LATENCY_CTRL_WR(data[1] - 1) |  in l2c310_of_parse()
1152 			L310_LATENCY_CTRL_SETUP(data[2] - 1);  in l2c310_of_parse()
1154 	of_property_read_u32_array(np, "arm,filter-ranges",  in l2c310_of_parse()
1159 		l2x0_saved_regs.filter_start = (filter[0] & ~(SZ_1M - 1))  in l2c310_of_parse()
1171 		case 8:  in l2c310_of_parse()
1176 			pr_err("L2C-310 OF cache associativity %d invalid, only 8 or 16 permitted\n",  in l2c310_of_parse()
1182 	if (of_property_read_bool(np, "arm,shared-override")) {  in l2c310_of_parse()
1187 	if (of_property_read_bool(np, "arm,parity-enable")) {  in l2c310_of_parse()
1190 	} else if (of_property_read_bool(np, "arm,parity-disable")) {  in l2c310_of_parse()
1195 	if (of_property_read_bool(np, "arm,early-bresp-disable"))  in l2c310_of_parse()
1198 	if (of_property_read_bool(np, "arm,full-line-zero-disable"))  in l2c310_of_parse()
1203 	ret = of_property_read_u32(np, "arm,double-linefill", &val);  in l2c310_of_parse()
1209 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1210 		pr_err("L2C-310 OF arm,double-linefill property value is missing\n");  in l2c310_of_parse()
1213 	ret = of_property_read_u32(np, "arm,double-linefill-incr", &val);  in l2c310_of_parse()
1219 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1220 		pr_err("L2C-310 OF arm,double-linefill-incr property value is missing\n");  in l2c310_of_parse()
1223 	ret = of_property_read_u32(np, "arm,double-linefill-wrap", &val);  in l2c310_of_parse()
1229 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1230 		pr_err("L2C-310 OF arm,double-linefill-wrap property value is missing\n");  in l2c310_of_parse()
1233 	ret = of_property_read_u32(np, "arm,prefetch-drop", &val);  in l2c310_of_parse()
1239 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1240 		pr_err("L2C-310 OF arm,prefetch-drop property value is missing\n");  in l2c310_of_parse()
1243 	ret = of_property_read_u32(np, "arm,prefetch-offset", &val);  in l2c310_of_parse()
1247 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1248 		pr_err("L2C-310 OF arm,prefetch-offset property value is missing\n");  in l2c310_of_parse()
1251 	ret = of_property_read_u32(np, "prefetch-data", &val);  in l2c310_of_parse()
1261 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1262 		pr_err("L2C-310 OF prefetch-data property value is missing\n");  in l2c310_of_parse()
1265 	ret = of_property_read_u32(np, "prefetch-instr", &val);  in l2c310_of_parse()
1275 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1276 		pr_err("L2C-310 OF prefetch-instr property value is missing\n");  in l2c310_of_parse()
1284 	ret = of_property_read_u32(np, "arm,dynamic-clock-gating", &val);  in l2c310_of_parse()
1288 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1289 		pr_err("L2C-310 OF dynamic-clock-gating property value is missing or invalid\n");  in l2c310_of_parse()
1291 	ret = of_property_read_u32(np, "arm,standby-mode", &val);  in l2c310_of_parse()
1295 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1296 		pr_err("L2C-310 OF standby-mode property value is missing or invalid\n");  in l2c310_of_parse()
1303 	.type = "L2C-310",
1305 	.num_lock = 8,
1333 	.type = "L2C-310 Coherent",
1335 	.num_lock = 8,
1386 	start &= ~(CACHE_LINE_SIZE - 1);  in aurora_pa_range()
1397 		writel_relaxed(range_end - CACHE_LINE_SIZE, base + offset);  in aurora_pa_range()
1494 	of_property_read_u32(np, "cache-id-part",  in aurora_of_parse()
1498 	l2_wt_override = of_property_read_bool(np, "wt-override");  in aurora_of_parse()
1505 	if (of_property_read_bool(np, "marvell,ecc-enable")) {  in aurora_of_parse()
1510 	if (of_property_read_bool(np, "arm,parity-enable")) {  in aurora_of_parse()
1513 	} else if (of_property_read_bool(np, "arm,parity-disable")) {  in aurora_of_parse()
1564  *   1     0x00000000 - 0x3FFFFFFF    0x80000000    VC
1565  *   2     0x40000000 - 0xBFFFFFFF    0x40000000    SYS
1566  *   3     0xC0000000 - 0xFFFFFFFF    0x80000000    VC
1571  * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2)
1572  * 0xC0000000 - 0xC0001000
1630 		bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));  in bcm_inv_range()
1657 		bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));  in bcm_clean_range()
1671 	if ((end - start) >= l2x0_size) {  in bcm_flush_range()
1689 		bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));  in bcm_flush_range()
1694 /* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */
1696 	.type = "BCM-L2C-310",
1698 	.num_lock = 8,
1737 	.num_lock = 8,
1750 	L2C_ID("arm,l210-cache", of_l2c210_data),
1751 	L2C_ID("arm,l220-cache", of_l2c220_data),
1752 	L2C_ID("arm,pl310-cache", of_l2c310_data),
1753 	L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
1754 	L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data),
1755 	L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data),
1756 	L2C_ID("marvell,tauros3-cache", of_tauros3_data),
1758 	L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
1773 		return -ENODEV;  in l2x0_of_init()
1776 		return -ENODEV;  in l2x0_of_init()
1780 		return -ENOMEM;  in l2x0_of_init()
1784 	data = of_match_node(l2x0_ids, np)->data;  in l2x0_of_init()
1786 	if (of_device_is_compatible(np, "arm,pl310-cache") &&  in l2x0_of_init()
1787 	    of_property_read_bool(np, "arm,io-coherent"))  in l2x0_of_init()
1792 		pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n",  in l2x0_of_init()
1799 	if (!of_property_read_bool(np, "cache-unified"))  in l2x0_of_init()
1802 	if (of_property_read_u32(np, "cache-level", &cache_level))  in l2x0_of_init()
1803 		pr_err("L2C: device tree omits to specify cache-level\n");  in l2x0_of_init()
1808 	nosync = of_property_read_bool(np, "arm,outer-sync-disable");  in l2x0_of_init()
1811 	if (data->save)  in l2x0_of_init()
1812 		data->save(l2x0_base);  in l2x0_of_init()
1816 		if (data->of_parse)  in l2x0_of_init()
1817 			data->of_parse(np, &aux_val, &aux_mask);  in l2x0_of_init()