1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
29e5c33d7SMark Salter /*
39e5c33d7SMark Salter * Provide common bits of early_ioremap() support for architectures needing
49e5c33d7SMark Salter * temporary mappings during boot before ioremap() is available.
59e5c33d7SMark Salter *
69e5c33d7SMark Salter * This is mostly a direct copy of the x86 early_ioremap implementation.
79e5c33d7SMark Salter *
89e5c33d7SMark Salter * (C) Copyright 1995 1996, 2014 Linus Torvalds
99e5c33d7SMark Salter *
109e5c33d7SMark Salter */
119e5c33d7SMark Salter #include <linux/kernel.h>
129e5c33d7SMark Salter #include <linux/init.h>
139e5c33d7SMark Salter #include <linux/io.h>
149e5c33d7SMark Salter #include <linux/module.h>
159e5c33d7SMark Salter #include <linux/slab.h>
169e5c33d7SMark Salter #include <linux/mm.h>
179e5c33d7SMark Salter #include <linux/vmalloc.h>
189e5c33d7SMark Salter #include <asm/fixmap.h>
194f1af60bSArd Biesheuvel #include <asm/early_ioremap.h>
20be4893d9SVlastimil Babka #include "internal.h"
219e5c33d7SMark Salter
229e5c33d7SMark Salter #ifdef CONFIG_MMU
239e5c33d7SMark Salter static int early_ioremap_debug __initdata;
249e5c33d7SMark Salter
early_ioremap_debug_setup(char * str)259e5c33d7SMark Salter static int __init early_ioremap_debug_setup(char *str)
269e5c33d7SMark Salter {
279e5c33d7SMark Salter early_ioremap_debug = 1;
289e5c33d7SMark Salter
299e5c33d7SMark Salter return 0;
309e5c33d7SMark Salter }
319e5c33d7SMark Salter early_param("early_ioremap_debug", early_ioremap_debug_setup);
329e5c33d7SMark Salter
339e5c33d7SMark Salter static int after_paging_init __initdata;
349e5c33d7SMark Salter
early_memremap_pgprot_adjust(resource_size_t phys_addr,unsigned long size,pgprot_t prot)358f716c9bSTom Lendacky pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr,
368f716c9bSTom Lendacky unsigned long size,
378f716c9bSTom Lendacky pgprot_t prot)
388f716c9bSTom Lendacky {
398f716c9bSTom Lendacky return prot;
408f716c9bSTom Lendacky }
418f716c9bSTom Lendacky
early_ioremap_reset(void)429e5c33d7SMark Salter void __init early_ioremap_reset(void)
439e5c33d7SMark Salter {
449e5c33d7SMark Salter after_paging_init = 1;
459e5c33d7SMark Salter }
469e5c33d7SMark Salter
479e5c33d7SMark Salter /*
489e5c33d7SMark Salter * Generally, ioremap() is available after paging_init() has been called.
499e5c33d7SMark Salter * Architectures wanting to allow early_ioremap after paging_init() can
509e5c33d7SMark Salter * define __late_set_fixmap and __late_clear_fixmap to do the right thing.
519e5c33d7SMark Salter */
529e5c33d7SMark Salter #ifndef __late_set_fixmap
__late_set_fixmap(enum fixed_addresses idx,phys_addr_t phys,pgprot_t prot)539e5c33d7SMark Salter static inline void __init __late_set_fixmap(enum fixed_addresses idx,
549e5c33d7SMark Salter phys_addr_t phys, pgprot_t prot)
559e5c33d7SMark Salter {
569e5c33d7SMark Salter BUG();
579e5c33d7SMark Salter }
589e5c33d7SMark Salter #endif
599e5c33d7SMark Salter
609e5c33d7SMark Salter #ifndef __late_clear_fixmap
__late_clear_fixmap(enum fixed_addresses idx)619e5c33d7SMark Salter static inline void __init __late_clear_fixmap(enum fixed_addresses idx)
629e5c33d7SMark Salter {
639e5c33d7SMark Salter BUG();
649e5c33d7SMark Salter }
659e5c33d7SMark Salter #endif
669e5c33d7SMark Salter
679e5c33d7SMark Salter static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
689e5c33d7SMark Salter static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
699e5c33d7SMark Salter static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
709e5c33d7SMark Salter
early_ioremap_setup(void)719e5c33d7SMark Salter void __init early_ioremap_setup(void)
729e5c33d7SMark Salter {
739e5c33d7SMark Salter int i;
749e5c33d7SMark Salter
75*5e074725SLiam Ni for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
76*5e074725SLiam Ni WARN_ON_ONCE(prev_map[i]);
779e5c33d7SMark Salter slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
789e5c33d7SMark Salter }
79*5e074725SLiam Ni }
809e5c33d7SMark Salter
check_early_ioremap_leak(void)819e5c33d7SMark Salter static int __init check_early_ioremap_leak(void)
829e5c33d7SMark Salter {
839e5c33d7SMark Salter int count = 0;
849e5c33d7SMark Salter int i;
859e5c33d7SMark Salter
869e5c33d7SMark Salter for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
879e5c33d7SMark Salter if (prev_map[i])
889e5c33d7SMark Salter count++;
899e5c33d7SMark Salter
909e5c33d7SMark Salter if (WARN(count, KERN_WARNING
919e5c33d7SMark Salter "Debug warning: early ioremap leak of %d areas detected.\n"
929e5c33d7SMark Salter "please boot with early_ioremap_debug and report the dmesg.\n",
939e5c33d7SMark Salter count))
949e5c33d7SMark Salter return 1;
959e5c33d7SMark Salter return 0;
969e5c33d7SMark Salter }
979e5c33d7SMark Salter late_initcall(check_early_ioremap_leak);
989e5c33d7SMark Salter
999e5c33d7SMark Salter static void __init __iomem *
__early_ioremap(resource_size_t phys_addr,unsigned long size,pgprot_t prot)1009e5c33d7SMark Salter __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
1019e5c33d7SMark Salter {
1029e5c33d7SMark Salter unsigned long offset;
1039e5c33d7SMark Salter resource_size_t last_addr;
1049e5c33d7SMark Salter unsigned int nrpages;
1059e5c33d7SMark Salter enum fixed_addresses idx;
1069e5c33d7SMark Salter int i, slot;
1079e5c33d7SMark Salter
1087f6f60a1SDave Young WARN_ON(system_state >= SYSTEM_RUNNING);
1099e5c33d7SMark Salter
1109e5c33d7SMark Salter slot = -1;
1119e5c33d7SMark Salter for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
1129e5c33d7SMark Salter if (!prev_map[i]) {
1139e5c33d7SMark Salter slot = i;
1149e5c33d7SMark Salter break;
1159e5c33d7SMark Salter }
1169e5c33d7SMark Salter }
1179e5c33d7SMark Salter
1187b69d79fSAndy Shevchenko if (WARN(slot < 0, "%s(%pa, %08lx) not found slot\n",
1197b69d79fSAndy Shevchenko __func__, &phys_addr, size))
1209e5c33d7SMark Salter return NULL;
1219e5c33d7SMark Salter
1229e5c33d7SMark Salter /* Don't allow wraparound or zero size */
1239e5c33d7SMark Salter last_addr = phys_addr + size - 1;
1249e5c33d7SMark Salter if (WARN_ON(!size || last_addr < phys_addr))
1259e5c33d7SMark Salter return NULL;
1269e5c33d7SMark Salter
1279e5c33d7SMark Salter prev_size[slot] = size;
1289e5c33d7SMark Salter /*
1299e5c33d7SMark Salter * Mappings have to be page-aligned
1309e5c33d7SMark Salter */
1315d57b014SAlexander Kuleshov offset = offset_in_page(phys_addr);
1329e5c33d7SMark Salter phys_addr &= PAGE_MASK;
1339e5c33d7SMark Salter size = PAGE_ALIGN(last_addr + 1) - phys_addr;
1349e5c33d7SMark Salter
1359e5c33d7SMark Salter /*
1369e5c33d7SMark Salter * Mappings have to fit in the FIX_BTMAP area.
1379e5c33d7SMark Salter */
1389e5c33d7SMark Salter nrpages = size >> PAGE_SHIFT;
1399e5c33d7SMark Salter if (WARN_ON(nrpages > NR_FIX_BTMAPS))
1409e5c33d7SMark Salter return NULL;
1419e5c33d7SMark Salter
1429e5c33d7SMark Salter /*
1439e5c33d7SMark Salter * Ok, go for it..
1449e5c33d7SMark Salter */
1459e5c33d7SMark Salter idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
1469e5c33d7SMark Salter while (nrpages > 0) {
1479e5c33d7SMark Salter if (after_paging_init)
1489e5c33d7SMark Salter __late_set_fixmap(idx, phys_addr, prot);
1499e5c33d7SMark Salter else
1509e5c33d7SMark Salter __early_set_fixmap(idx, phys_addr, prot);
1519e5c33d7SMark Salter phys_addr += PAGE_SIZE;
1529e5c33d7SMark Salter --idx;
1539e5c33d7SMark Salter --nrpages;
1549e5c33d7SMark Salter }
1557b69d79fSAndy Shevchenko WARN(early_ioremap_debug, "%s(%pa, %08lx) [%d] => %08lx + %08lx\n",
1567b69d79fSAndy Shevchenko __func__, &phys_addr, size, slot, offset, slot_virt[slot]);
1579e5c33d7SMark Salter
1589e5c33d7SMark Salter prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
1599e5c33d7SMark Salter return prev_map[slot];
1609e5c33d7SMark Salter }
1619e5c33d7SMark Salter
early_iounmap(void __iomem * addr,unsigned long size)1629e5c33d7SMark Salter void __init early_iounmap(void __iomem *addr, unsigned long size)
1639e5c33d7SMark Salter {
1649e5c33d7SMark Salter unsigned long virt_addr;
1659e5c33d7SMark Salter unsigned long offset;
1669e5c33d7SMark Salter unsigned int nrpages;
1679e5c33d7SMark Salter enum fixed_addresses idx;
1689e5c33d7SMark Salter int i, slot;
1699e5c33d7SMark Salter
1709e5c33d7SMark Salter slot = -1;
1719e5c33d7SMark Salter for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
1729e5c33d7SMark Salter if (prev_map[i] == addr) {
1739e5c33d7SMark Salter slot = i;
1749e5c33d7SMark Salter break;
1759e5c33d7SMark Salter }
1769e5c33d7SMark Salter }
1779e5c33d7SMark Salter
17887005394SStephen Zhang if (WARN(slot < 0, "%s(%p, %08lx) not found slot\n",
17987005394SStephen Zhang __func__, addr, size))
1809e5c33d7SMark Salter return;
1819e5c33d7SMark Salter
1829e5c33d7SMark Salter if (WARN(prev_size[slot] != size,
18387005394SStephen Zhang "%s(%p, %08lx) [%d] size not consistent %08lx\n",
18487005394SStephen Zhang __func__, addr, size, slot, prev_size[slot]))
1859e5c33d7SMark Salter return;
1869e5c33d7SMark Salter
18787005394SStephen Zhang WARN(early_ioremap_debug, "%s(%p, %08lx) [%d]\n",
18887005394SStephen Zhang __func__, addr, size, slot);
1899e5c33d7SMark Salter
1909e5c33d7SMark Salter virt_addr = (unsigned long)addr;
1919e5c33d7SMark Salter if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)))
1929e5c33d7SMark Salter return;
1939e5c33d7SMark Salter
1945d57b014SAlexander Kuleshov offset = offset_in_page(virt_addr);
1959e5c33d7SMark Salter nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
1969e5c33d7SMark Salter
1979e5c33d7SMark Salter idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
1989e5c33d7SMark Salter while (nrpages > 0) {
1999e5c33d7SMark Salter if (after_paging_init)
2009e5c33d7SMark Salter __late_clear_fixmap(idx);
2019e5c33d7SMark Salter else
2029e5c33d7SMark Salter __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR);
2039e5c33d7SMark Salter --idx;
2049e5c33d7SMark Salter --nrpages;
2059e5c33d7SMark Salter }
2069e5c33d7SMark Salter prev_map[slot] = NULL;
2079e5c33d7SMark Salter }
2089e5c33d7SMark Salter
2099e5c33d7SMark Salter /* Remap an IO device */
2109e5c33d7SMark Salter void __init __iomem *
early_ioremap(resource_size_t phys_addr,unsigned long size)2119e5c33d7SMark Salter early_ioremap(resource_size_t phys_addr, unsigned long size)
2129e5c33d7SMark Salter {
2139e5c33d7SMark Salter return __early_ioremap(phys_addr, size, FIXMAP_PAGE_IO);
2149e5c33d7SMark Salter }
2159e5c33d7SMark Salter
2169e5c33d7SMark Salter /* Remap memory */
2179e5c33d7SMark Salter void __init *
early_memremap(resource_size_t phys_addr,unsigned long size)2189e5c33d7SMark Salter early_memremap(resource_size_t phys_addr, unsigned long size)
2199e5c33d7SMark Salter {
2208f716c9bSTom Lendacky pgprot_t prot = early_memremap_pgprot_adjust(phys_addr, size,
2219e5c33d7SMark Salter FIXMAP_PAGE_NORMAL);
2228f716c9bSTom Lendacky
2238f716c9bSTom Lendacky return (__force void *)__early_ioremap(phys_addr, size, prot);
2249e5c33d7SMark Salter }
2252592dbbbSJuergen Gross #ifdef FIXMAP_PAGE_RO
2262592dbbbSJuergen Gross void __init *
early_memremap_ro(resource_size_t phys_addr,unsigned long size)2272592dbbbSJuergen Gross early_memremap_ro(resource_size_t phys_addr, unsigned long size)
2282592dbbbSJuergen Gross {
2298f716c9bSTom Lendacky pgprot_t prot = early_memremap_pgprot_adjust(phys_addr, size,
2308f716c9bSTom Lendacky FIXMAP_PAGE_RO);
2318f716c9bSTom Lendacky
2328f716c9bSTom Lendacky return (__force void *)__early_ioremap(phys_addr, size, prot);
2332592dbbbSJuergen Gross }
2342592dbbbSJuergen Gross #endif
2356b0f68e3SMark Salter
236f88a68faSTom Lendacky #ifdef CONFIG_ARCH_USE_MEMREMAP_PROT
237f88a68faSTom Lendacky void __init *
early_memremap_prot(resource_size_t phys_addr,unsigned long size,unsigned long prot_val)238f88a68faSTom Lendacky early_memremap_prot(resource_size_t phys_addr, unsigned long size,
239f88a68faSTom Lendacky unsigned long prot_val)
240f88a68faSTom Lendacky {
241f88a68faSTom Lendacky return (__force void *)__early_ioremap(phys_addr, size,
242f88a68faSTom Lendacky __pgprot(prot_val));
243f88a68faSTom Lendacky }
244f88a68faSTom Lendacky #endif
245f88a68faSTom Lendacky
2466b0f68e3SMark Salter #define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT)
2476b0f68e3SMark Salter
copy_from_early_mem(void * dest,phys_addr_t src,unsigned long size)2486b0f68e3SMark Salter void __init copy_from_early_mem(void *dest, phys_addr_t src, unsigned long size)
2496b0f68e3SMark Salter {
2506b0f68e3SMark Salter unsigned long slop, clen;
2516b0f68e3SMark Salter char *p;
2526b0f68e3SMark Salter
2536b0f68e3SMark Salter while (size) {
2545d57b014SAlexander Kuleshov slop = offset_in_page(src);
2556b0f68e3SMark Salter clen = size;
2566b0f68e3SMark Salter if (clen > MAX_MAP_CHUNK - slop)
2576b0f68e3SMark Salter clen = MAX_MAP_CHUNK - slop;
2586b0f68e3SMark Salter p = early_memremap(src & PAGE_MASK, clen + slop);
2596b0f68e3SMark Salter memcpy(dest, p + slop, clen);
2606b0f68e3SMark Salter early_memunmap(p, clen + slop);
2616b0f68e3SMark Salter dest += clen;
2626b0f68e3SMark Salter src += clen;
2636b0f68e3SMark Salter size -= clen;
2646b0f68e3SMark Salter }
2656b0f68e3SMark Salter }
2666b0f68e3SMark Salter
2679e5c33d7SMark Salter #else /* CONFIG_MMU */
2689e5c33d7SMark Salter
2699e5c33d7SMark Salter void __init __iomem *
early_ioremap(resource_size_t phys_addr,unsigned long size)2709e5c33d7SMark Salter early_ioremap(resource_size_t phys_addr, unsigned long size)
2719e5c33d7SMark Salter {
2729e5c33d7SMark Salter return (__force void __iomem *)phys_addr;
2739e5c33d7SMark Salter }
2749e5c33d7SMark Salter
2759e5c33d7SMark Salter /* Remap memory */
2769e5c33d7SMark Salter void __init *
early_memremap(resource_size_t phys_addr,unsigned long size)2779e5c33d7SMark Salter early_memremap(resource_size_t phys_addr, unsigned long size)
2789e5c33d7SMark Salter {
2799e5c33d7SMark Salter return (void *)phys_addr;
2809e5c33d7SMark Salter }
2812592dbbbSJuergen Gross void __init *
early_memremap_ro(resource_size_t phys_addr,unsigned long size)2822592dbbbSJuergen Gross early_memremap_ro(resource_size_t phys_addr, unsigned long size)
2832592dbbbSJuergen Gross {
2842592dbbbSJuergen Gross return (void *)phys_addr;
2852592dbbbSJuergen Gross }
2869e5c33d7SMark Salter
early_iounmap(void __iomem * addr,unsigned long size)2879e5c33d7SMark Salter void __init early_iounmap(void __iomem *addr, unsigned long size)
2889e5c33d7SMark Salter {
2899e5c33d7SMark Salter }
2909e5c33d7SMark Salter
2919e5c33d7SMark Salter #endif /* CONFIG_MMU */
2929e5c33d7SMark Salter
2939e5c33d7SMark Salter
early_memunmap(void * addr,unsigned long size)2949e5c33d7SMark Salter void __init early_memunmap(void *addr, unsigned long size)
2959e5c33d7SMark Salter {
2969e5c33d7SMark Salter early_iounmap((__force void __iomem *)addr, size);
2979e5c33d7SMark Salter }
298