Lines Matching +full:dma +full:- +full:window
1 // SPDX-License-Identifier: GPL-2.0-or-later
10 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
21 #include <linux/dma-mapping.h>
33 #include <asm/pci-bridge.h>
37 #include <asm/ppc-pci.h>
66 INIT_LIST_HEAD_RCU(&tbl->it_group_list); in iommu_pseries_alloc_table()
67 kref_init(&tbl->it_kref); in iommu_pseries_alloc_table()
84 table_group->ops = &spapr_tce_table_group_ops; in iommu_pseries_alloc_group()
85 table_group->pgsizes = SZ_4K; in iommu_pseries_alloc_group()
88 table_group->tables[0] = iommu_pseries_alloc_table(node); in iommu_pseries_alloc_group()
89 if (table_group->tables[0]) in iommu_pseries_alloc_group()
103 if (table_group->group) { in iommu_pseries_free_group()
104 iommu_group_put(table_group->group); in iommu_pseries_free_group()
105 BUG_ON(table_group->group); in iommu_pseries_free_group()
109 /* Default DMA window table is at index 0, while DDW at 1. SR-IOV in iommu_pseries_free_group()
112 if (table_group->tables[0]) in iommu_pseries_free_group()
113 iommu_tce_table_put(table_group->tables[0]); in iommu_pseries_free_group()
115 if (table_group->tables[1]) in iommu_pseries_free_group()
116 iommu_tce_table_put(table_group->tables[1]); in iommu_pseries_free_group()
129 const unsigned long tceshift = tbl->it_page_shift; in tce_build_pSeries()
137 tcep = ((__be64 *)tbl->it_base) + index; in tce_build_pSeries()
139 while (npages--) { in tce_build_pSeries()
155 tcep = ((__be64 *)tbl->it_base) + index; in tce_clear_pSeries()
157 while (npages--) in tce_clear_pSeries()
165 tcep = ((__be64 *)tbl->it_base) + index; in tce_get_pseries()
173 unsigned long cb = ALIGN(sizeof(tbl->it_userspace[0]) * tbl->it_size, PAGE_SIZE); in pseries_tce_iommu_userspace_view_alloc()
176 if (tbl->it_indirect_levels) /* Impossible */ in pseries_tce_iommu_userspace_view_alloc()
177 return -EPERM; in pseries_tce_iommu_userspace_view_alloc()
179 WARN_ON(tbl->it_userspace); in pseries_tce_iommu_userspace_view_alloc()
183 return -ENOMEM; in pseries_tce_iommu_userspace_view_alloc()
185 tbl->it_userspace = (__be64 *) uas; in pseries_tce_iommu_userspace_view_alloc()
193 vfree(tbl->it_userspace); in tce_iommu_userspace_view_free()
194 tbl->it_userspace = NULL; in tce_iommu_userspace_view_free()
199 if (!tbl->it_userspace) in tce_free_pSeries()
222 while (npages--) { in tce_build_pSeriesLP()
229 (npages_start - (npages + 1))); in tce_build_pSeriesLP()
262 const unsigned long tceshift = tbl->it_page_shift; in tce_buildmulti_pSeriesLP()
265 return tce_build_pSeriesLP(tbl->it_index, tcenum, in tce_buildmulti_pSeriesLP()
282 return tce_build_pSeriesLP(tbl->it_index, tcenum, in tce_buildmulti_pSeriesLP()
307 rc = plpar_tce_put_indirect((u64)tbl->it_index, in tce_buildmulti_pSeriesLP()
312 npages -= limit; in tce_buildmulti_pSeriesLP()
321 (npages_start - (npages + limit))); in tce_buildmulti_pSeriesLP()
327 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); in tce_buildmulti_pSeriesLP()
340 while (npages--) { in tce_free_pSeriesLP()
362 return tce_free_pSeriesLP(tbl->it_index, tcenum, in tce_freemulti_pSeriesLP()
363 tbl->it_page_shift, npages); in tce_freemulti_pSeriesLP()
368 rc = plpar_tce_stuff((u64)tbl->it_index, in tce_freemulti_pSeriesLP()
369 (u64)tcenum << tbl->it_page_shift, 0, limit); in tce_freemulti_pSeriesLP()
371 rpages -= limit; in tce_freemulti_pSeriesLP()
378 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); in tce_freemulti_pSeriesLP()
389 rc = plpar_tce_get((u64)tbl->it_index, in tce_get_pSeriesLP()
390 (u64)tcenum << tbl->it_page_shift, &tce_ret); in tce_get_pSeriesLP()
394 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); in tce_get_pSeriesLP()
417 /* Dynamic DMA Window support */
432 /* prevents races between memory on/offline and window creation */
434 /* protects initializing window twice for same device */
446 tce_shift = be32_to_cpu(maprange->tce_shift); in tce_clearrange_multi_pSeriesLP()
452 num_tce += next & (tce_size - 1); in tce_clearrange_multi_pSeriesLP()
453 next &= ~(tce_size - 1); in tce_clearrange_multi_pSeriesLP()
456 num_tce |= tce_size - 1; in tce_clearrange_multi_pSeriesLP()
465 dma_offset = next + be64_to_cpu(maprange->dma_base); in tce_clearrange_multi_pSeriesLP()
467 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn), in tce_clearrange_multi_pSeriesLP()
471 num_tce -= limit; in tce_clearrange_multi_pSeriesLP()
488 unsigned long tceshift = be32_to_cpu(maprange->tce_shift); in tce_setrange_multi_pSeriesLP()
490 be64_to_cpu(maprange->dma_base); in tce_setrange_multi_pSeriesLP()
495 return tce_build_pSeriesLP(be32_to_cpu(maprange->liobn), in tce_setrange_multi_pSeriesLP()
507 return -ENOMEM; in tce_setrange_multi_pSeriesLP()
514 liobn = (u64)be32_to_cpu(maprange->liobn); in tce_setrange_multi_pSeriesLP()
515 tce_shift = be32_to_cpu(maprange->tce_shift); in tce_setrange_multi_pSeriesLP()
521 num_tce += next & (tce_size - 1); in tce_setrange_multi_pSeriesLP()
522 next &= ~(tce_size - 1); in tce_setrange_multi_pSeriesLP()
525 num_tce |= tce_size - 1; in tce_setrange_multi_pSeriesLP()
535 dma_offset = next + be64_to_cpu(maprange->dma_base); in tce_setrange_multi_pSeriesLP()
547 num_tce -= limit; in tce_setrange_multi_pSeriesLP()
567 tbl->it_busno = busno; in iommu_table_setparms_common()
568 tbl->it_index = liobn; in iommu_table_setparms_common()
569 tbl->it_offset = win_addr >> page_shift; in iommu_table_setparms_common()
570 tbl->it_size = window_size >> page_shift; in iommu_table_setparms_common()
571 tbl->it_page_shift = page_shift; in iommu_table_setparms_common()
572 tbl->it_base = (unsigned long)base; in iommu_table_setparms_common()
573 tbl->it_blocksize = 16; in iommu_table_setparms_common()
574 tbl->it_type = TCE_PCI; in iommu_table_setparms_common()
575 tbl->it_ops = table_ops; in iommu_table_setparms_common()
588 /* Test if we are going over 2GB of DMA space */ in iommu_table_setparms()
589 if (phb->dma_window_base_cur + phb->dma_window_size > SZ_2G) { in iommu_table_setparms()
594 node = phb->dn; in iommu_table_setparms()
595 basep = of_get_property(node, "linux,tce-base", NULL); in iommu_table_setparms()
596 sizep = of_get_property(node, "linux,tce-size", NULL); in iommu_table_setparms()
603 iommu_table_setparms_common(tbl, phb->bus->number, 0, phb->dma_window_base_cur, in iommu_table_setparms()
604 phb->dma_window_size, IOMMU_PAGE_SHIFT_4K, in iommu_table_setparms()
608 memset((void *)tbl->it_base, 0, *sizep); in iommu_table_setparms()
610 phb->dma_window_base_cur += phb->dma_window_size; in iommu_table_setparms()
634 if (bus->self) { in pci_dma_bus_setup_pSeries()
636 * device-side of the bridge in iommu_dev_setup_pSeries(). in pci_dma_bus_setup_pSeries()
648 isa_dn = isa_dn->parent; in pci_dma_bus_setup_pSeries()
653 for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling) in pci_dma_bus_setup_pSeries()
658 /* Calculate amount of DMA window per slot. Each window must be in pci_dma_bus_setup_pSeries()
665 /* No ISA/IDE - just set window size and return */ in pci_dma_bus_setup_pSeries()
666 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */ in pci_dma_bus_setup_pSeries()
668 while (pci->phb->dma_window_size * children > 0x80000000ul) in pci_dma_bus_setup_pSeries()
669 pci->phb->dma_window_size >>= 1; in pci_dma_bus_setup_pSeries()
670 pr_debug("No ISA/IDE, window size is 0x%llx\n", in pci_dma_bus_setup_pSeries()
671 pci->phb->dma_window_size); in pci_dma_bus_setup_pSeries()
672 pci->phb->dma_window_base_cur = 0; in pci_dma_bus_setup_pSeries()
682 pci->phb->dma_window_size = 0x8000000ul; in pci_dma_bus_setup_pSeries()
683 pci->phb->dma_window_base_cur = 0x8000000ul; in pci_dma_bus_setup_pSeries()
685 pci->table_group = iommu_pseries_alloc_group(pci->phb->node); in pci_dma_bus_setup_pSeries()
686 tbl = pci->table_group->tables[0]; in pci_dma_bus_setup_pSeries()
688 iommu_table_setparms(pci->phb, dn, tbl); in pci_dma_bus_setup_pSeries()
690 if (!iommu_init_table(tbl, pci->phb->node, 0, 0)) in pci_dma_bus_setup_pSeries()
694 pci->phb->dma_window_size = 0x80000000ul; in pci_dma_bus_setup_pSeries()
695 while (pci->phb->dma_window_size * children > 0x70000000ul) in pci_dma_bus_setup_pSeries()
696 pci->phb->dma_window_size >>= 1; in pci_dma_bus_setup_pSeries()
698 pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size); in pci_dma_bus_setup_pSeries()
706 unsigned long ioba = (unsigned long) index << tbl->it_page_shift; in tce_exchange_pseries()
711 spin_lock_irqsave(&tbl->large_pool.lock, flags); in tce_exchange_pseries()
713 rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce); in tce_exchange_pseries()
715 rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce); in tce_exchange_pseries()
722 spin_unlock_irqrestore(&tbl->large_pool.lock, flags); in tce_exchange_pseries()
730 return tbl->it_userspace ? &tbl->it_userspace[index - tbl->it_offset] : NULL; in tce_useraddr_pSeriesLP()
747 * When the DMA window properties might have been removed,
756 for (; dn && PCI_DN(dn); dn = dn->parent) { in pci_dma_find_parent_node()
759 if (table_group == rpdn->table_group) in pci_dma_find_parent_node()
768 * Find nearest ibm,dma-window (default DMA window) or direct DMA window or
769 * dynamic 64bit DMA window, walking up the device tree.
779 for ( ; dn && PCI_DN(dn); dn = dn->parent) { in pci_dma_find()
780 default_prop = of_get_property(dn, "ibm,dma-window", NULL); in pci_dma_find()
798 /* At least found default window, which is the case for normal boot */ in pci_dma_find()
803 /* For PCI devices there will always be a DMA window, either on the device in pci_dma_find()
808 /* caller doesn't want to get DMA window property */ in pci_dma_find()
812 /* parse DMA window property. During normal system boot, only default in pci_dma_find()
813 * DMA window is passed in OF. But, for kdump, a dedicated adapter might in pci_dma_find()
815 * over default window. in pci_dma_find()
821 prop->liobn = p->liobn; in pci_dma_find()
822 prop->dma_base = p->dma_base; in pci_dma_find()
823 prop->tce_shift = p->tce_shift; in pci_dma_find()
824 prop->window_shift = p->window_shift; in pci_dma_find()
830 prop->liobn = cpu_to_be32((u32)liobn); in pci_dma_find()
831 prop->dma_base = cpu_to_be64(offset); in pci_dma_find()
832 prop->tce_shift = cpu_to_be32(IOMMU_PAGE_SHIFT_4K); in pci_dma_find()
833 prop->window_shift = cpu_to_be32(order_base_2(size)); in pci_dma_find()
853 /* In PPC architecture, there will always be DMA window on bus or one of the in pci_dma_bus_setup_pSeriesLP()
854 * parent bus. During reboot, there will be ibm,dma-window property to in pci_dma_bus_setup_pSeriesLP()
855 * define DMA window. For kdump, there will at least be default window or DDW in pci_dma_bus_setup_pSeriesLP()
858 * state, firmware may not provide ibm,dma-window property at the time in pci_dma_bus_setup_pSeriesLP()
863 pr_debug(" no ibm,dma-window property !\n"); in pci_dma_bus_setup_pSeriesLP()
870 pdn, ppci->table_group); in pci_dma_bus_setup_pSeriesLP()
872 if (!ppci->table_group) { in pci_dma_bus_setup_pSeriesLP()
873 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node); in pci_dma_bus_setup_pSeriesLP()
874 tbl = ppci->table_group->tables[0]; in pci_dma_bus_setup_pSeriesLP()
876 iommu_table_setparms_common(tbl, ppci->phb->bus->number, in pci_dma_bus_setup_pSeriesLP()
883 if (!iommu_init_table(tbl, ppci->phb->node, 0, 0)) in pci_dma_bus_setup_pSeriesLP()
886 iommu_register_group(ppci->table_group, in pci_dma_bus_setup_pSeriesLP()
888 pr_debug(" created table: %p\n", ppci->table_group); in pci_dma_bus_setup_pSeriesLP()
900 dn = dev->dev.of_node; in pci_dma_dev_setup_pSeries()
904 * the window sizes already. in pci_dma_dev_setup_pSeries()
906 if (!dev->bus->self) { in pci_dma_dev_setup_pSeries()
907 struct pci_controller *phb = PCI_DN(dn)->phb; in pci_dma_dev_setup_pSeries()
909 pr_debug(" --> first child, no bridge. Allocating iommu table.\n"); in pci_dma_dev_setup_pSeries()
910 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node); in pci_dma_dev_setup_pSeries()
911 tbl = PCI_DN(dn)->table_group->tables[0]; in pci_dma_dev_setup_pSeries()
914 if (!iommu_init_table(tbl, phb->node, 0, 0)) in pci_dma_dev_setup_pSeries()
917 set_iommu_table_base(&dev->dev, tbl); in pci_dma_dev_setup_pSeries()
925 while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL) in pci_dma_dev_setup_pSeries()
926 dn = dn->parent; in pci_dma_dev_setup_pSeries()
929 set_iommu_table_base(&dev->dev, in pci_dma_dev_setup_pSeries()
930 PCI_DN(dn)->table_group->tables[0]); in pci_dma_dev_setup_pSeries()
953 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp); in clean_dma_window()
955 pr_warn("%pOF failed to clear tces in window.\n", in clean_dma_window()
958 pr_debug("%pOF successfully cleared tces in window.\n", in clean_dma_window()
963 * Call only if DMA window is clean.
971 pr_warn("%pOF: failed to remove DMA window: rtas returned " in __remove_dma_window()
972 "%d to ibm,remove-pe-dma-window(%x) %llx\n", in __remove_dma_window()
975 pr_debug("%pOF: successfully removed DMA window: rtas returned " in __remove_dma_window()
976 "%d to ibm,remove-pe-dma-window(%x) %llx\n", in __remove_dma_window()
986 dwp = win->value; in remove_dma_window()
987 liobn = (u64)be32_to_cpu(dwp->liobn); in remove_dma_window()
1006 dst->name = kstrdup(to, GFP_KERNEL); in copy_property()
1007 dst->value = kmemdup(src->value, src->length, GFP_KERNEL); in copy_property()
1008 dst->length = src->length; in copy_property()
1009 if (!dst->name || !dst->value) in copy_property()
1013 pr_err("Unable to add DMA window property for %pOF", pdn); in copy_property()
1020 kfree(dst->name); in copy_property()
1021 kfree(dst->value); in copy_property()
1034 return -EINVAL; in remove_dma_window_named()
1036 ret = of_property_read_u32_array(np, "ibm,ddw-applicable", in remove_dma_window_named()
1041 if (win->length >= sizeof(struct dynamic_dma_window_prop)) in remove_dma_window_named()
1047 /* Default window property if removed is lost as reset-pe doesn't restore it. in remove_dma_window_named()
1051 if ((strcmp(win_name, "ibm,dma-window") == 0) && in remove_dma_window_named()
1052 !of_find_property(np, "ibm,dma-window-saved", NULL)) in remove_dma_window_named()
1053 copy_property(np, win_name, "ibm,dma-window-saved"); in remove_dma_window_named()
1057 pr_warn("%pOF: failed to remove DMA window property: %d\n", in remove_dma_window_named()
1065 struct dma_win *window; in find_existing_ddw() local
1070 /* check if we already created a window and dupe that config if so */ in find_existing_ddw()
1071 list_for_each_entry(window, &dma_win_list, list) { in find_existing_ddw()
1072 if (window->device == pdn) { in find_existing_ddw()
1073 dma64 = window->prop; in find_existing_ddw()
1074 *dma_addr = be64_to_cpu(dma64->dma_base); in find_existing_ddw()
1075 *window_shift = be32_to_cpu(dma64->window_shift); in find_existing_ddw()
1076 *direct_mapping = window->direct; in find_existing_ddw()
1089 struct dma_win *window; in ddw_list_new_entry() local
1091 window = kzalloc(sizeof(*window), GFP_KERNEL); in ddw_list_new_entry()
1092 if (!window) in ddw_list_new_entry()
1095 window->device = pdn; in ddw_list_new_entry()
1096 window->prop = dma64; in ddw_list_new_entry()
1097 window->direct = false; in ddw_list_new_entry()
1099 return window; in ddw_list_new_entry()
1106 struct dma_win *window; in find_existing_ddw_windows_named() local
1120 * set window->direct = false. in find_existing_ddw_windows_named()
1122 window = ddw_list_new_entry(pdn, dma64); in find_existing_ddw_windows_named()
1123 if (!window) { in find_existing_ddw_windows_named()
1129 list_add(&window->list, &dma_win_list); in find_existing_ddw_windows_named()
1147 * ddw_read_ext - Get the value of an DDW extension
1152 * Checks if "ibm,ddw-extensions" exists for this node, and get the value
1158 * -EINVAL if the "ibm,ddw-extensions" does not exist,
1159 * -ENODATA if "ibm,ddw-extensions" does not have a value, and
1160 * -EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
1165 static const char propname[] = "ibm,ddw-extensions"; in ddw_read_ext()
1174 return -EOVERFLOW; in ddw_read_ext()
1193 * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many in query_ddw()
1194 * output parameters ibm,query-pe-dma-windows will have, ranging from in query_ddw()
1204 * Get the config address and phb buid of the PE window. in query_ddw()
1207 * dma-window property in query_ddw()
1211 buid = pdn->phb->buid; in query_ddw()
1212 cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8)); in query_ddw()
1219 query->windows_available = query_out[0]; in query_ddw()
1220 query->largest_available_block = query_out[1]; in query_ddw()
1221 query->page_size = query_out[2]; in query_ddw()
1222 query->migration_capable = query_out[3]; in query_ddw()
1225 query->windows_available = query_out[0]; in query_ddw()
1226 query->largest_available_block = ((u64)query_out[1] << 32) | in query_ddw()
1228 query->page_size = query_out[3]; in query_ddw()
1229 query->migration_capable = query_out[4]; in query_ddw()
1233 dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d, lb=%llx ps=%x wn=%d\n", in query_ddw()
1235 BUID_LO(buid), ret, query->largest_available_block, in query_ddw()
1236 query->page_size, query->windows_available); in query_ddw()
1252 * Get the config address and phb buid of the PE window. in create_ddw()
1255 * dma-window property in create_ddw()
1259 buid = pdn->phb->buid; in create_ddw()
1260 cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8)); in create_ddw()
1263 /* extra outputs are LIOBN and dma-addr (hi, lo) */ in create_ddw()
1268 dev_info(&dev->dev, in create_ddw()
1269 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d " in create_ddw()
1272 BUID_LO(buid), page_shift, window_shift, ret, create->liobn, in create_ddw()
1273 create->addr_hi, create->addr_lo); in create_ddw()
1304 * ibm,ddw-extensions, which carries the rtas token for
1305 * ibm,reset-pe-dma-windows.
1306 * That rtas-call can be used to restore the default DMA window for the device.
1322 buid = pdn->phb->buid; in reset_dma_window()
1323 cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8); in reset_dma_window()
1328 dev_info(&dev->dev, in reset_dma_window()
1329 "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ", in reset_dma_window()
1334 /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
1337 /* Supported IO page-sizes according to LoPAR, note that 2M is out of order */ in iommu_get_page_shift()
1344 int i = ARRAY_SIZE(shift) - 1; in iommu_get_page_shift()
1348 * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field: in iommu_get_page_shift()
1349 * - bit 31 means 4k pages are supported, in iommu_get_page_shift()
1350 * - bit 30 means 64k pages are supported, and so on. in iommu_get_page_shift()
1353 for (; i >= 0 ; i--) { in iommu_get_page_shift()
1371 win64->name = kstrdup(propname, GFP_KERNEL); in ddw_property_create()
1373 win64->value = ddwprop; in ddw_property_create()
1374 win64->length = sizeof(*ddwprop); in ddw_property_create()
1375 if (!win64->name || !win64->value) { in ddw_property_create()
1376 kfree(win64->name); in ddw_property_create()
1377 kfree(win64->value); in ddw_property_create()
1382 ddwprop->liobn = cpu_to_be32(liobn); in ddw_property_create()
1383 ddwprop->dma_base = cpu_to_be64(dma_addr); in ddw_property_create()
1384 ddwprop->tce_shift = cpu_to_be32(page_shift); in ddw_property_create()
1385 ddwprop->window_shift = cpu_to_be32(window_shift); in ddw_property_create()
1391 * If the PE supports dynamic dma windows, and there is space for a table
1393 * and record the dma-offset in the struct device.
1397 * Future: also check if we can remap the base window for our base page size
1412 struct dma_win *window; in enable_ddw() local
1427 if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len, &direct_mapping)) in enable_ddw()
1433 * DMA window again, as it will race with in-flight operations in enable_ddw()
1438 if (fpdn->pdn == pdn) in enable_ddw()
1443 * the ibm,ddw-applicable property holds the tokens for: in enable_ddw()
1444 * ibm,query-pe-dma-window in enable_ddw()
1445 * ibm,create-pe-dma-window in enable_ddw()
1446 * ibm,remove-pe-dma-window in enable_ddw()
1450 ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable", in enable_ddw()
1456 * Query if there is a second window of size to map the in enable_ddw()
1459 * of page sizes: supported and supported for migrate-dma. in enable_ddw()
1467 * If there is no window available, remove the default DMA window, in enable_ddw()
1469 * new DDW window. in enable_ddw()
1476 /* DDW + IOMMU on single window may fail if there is any allocation */ in enable_ddw()
1477 if (iommu_table_in_use(pci->table_group->tables[0])) { in enable_ddw()
1478 dev_warn(&dev->dev, "current IOMMU table in use, can't be replaced.\n"); in enable_ddw()
1482 default_win = of_find_property(pdn, "ibm,dma-window", NULL); in enable_ddw()
1493 /* Query again, to check if the window is available */ in enable_ddw()
1500 dev_dbg(&dev->dev, "no free dynamic windows"); in enable_ddw()
1507 dev_dbg(&dev->dev, "no supported page size in mask %x", in enable_ddw()
1521 (1ULL << (MAX_PHYSMEM_BITS - page_shift))) in enable_ddw()
1524 dev_info(&dev->dev, "Skipping ibm,pmemory"); in enable_ddw()
1528 if (query.largest_available_block < (1ULL << (len - page_shift))) { in enable_ddw()
1529 dev_dbg(&dev->dev, in enable_ddw()
1530 "can't map partition max 0x%llx with %llu %llu-sized pages\n", in enable_ddw()
1546 * is only for SR-IOV devices. in enable_ddw()
1576 dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n", in enable_ddw()
1583 dev_info(&dev->dev, in enable_ddw()
1590 dev_err(&dev->dev, "unable to add DMA window property for %pOF: %d", in enable_ddw()
1595 window = ddw_list_new_entry(pdn, win64->value); in enable_ddw()
1596 if (!window) in enable_ddw()
1599 window->direct = direct_mapping; in enable_ddw()
1602 /* DDW maps the whole partition, so enable direct DMA mapping */ in enable_ddw()
1604 win64->value, tce_setrange_multi_pSeriesLP_walk); in enable_ddw()
1606 dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n", in enable_ddw()
1610 clean_dma_window(pdn, win64->value); in enable_ddw()
1614 iommu_tce_table_put(pci->table_group->tables[0]); in enable_ddw()
1615 pci->table_group->tables[0] = NULL; in enable_ddw()
1616 set_iommu_table_base(&dev->dev, NULL); in enable_ddw()
1626 for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) { in enable_ddw()
1630 if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) { in enable_ddw()
1631 start = pci->phb->mem_resources[i].start; in enable_ddw()
1632 end = pci->phb->mem_resources[i].end; in enable_ddw()
1637 /* New table for using DDW instead of the default DMA window */ in enable_ddw()
1638 newtbl = iommu_pseries_alloc_table(pci->phb->node); in enable_ddw()
1640 dev_dbg(&dev->dev, "couldn't create new IOMMU table\n"); in enable_ddw()
1649 dynamic_len = (1UL << len) - dynamic_offset; in enable_ddw()
1650 iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, in enable_ddw()
1653 iommu_init_table(newtbl, pci->phb->node, start, end); in enable_ddw()
1655 pci->table_group->tables[default_win_removed ? 0 : 1] = newtbl; in enable_ddw()
1657 set_iommu_table_base(&dev->dev, newtbl); in enable_ddw()
1662 if (!of_find_property(pdn, "ibm,dma-window-saved", NULL)) in enable_ddw()
1663 copy_property(pdn, "ibm,dma-window", "ibm,dma-window-saved"); in enable_ddw()
1665 dev_info(&dev->dev, "Removed default DMA window for %pOF\n", pdn); in enable_ddw()
1669 list_add(&window->list, &dma_win_list); in enable_ddw()
1672 dev->dev.archdata.dma_offset = win_addr; in enable_ddw()
1676 kfree(window); in enable_ddw()
1682 kfree(win64->name); in enable_ddw()
1683 kfree(win64->value); in enable_ddw()
1697 fpdn->pdn = pdn; in enable_ddw()
1698 list_add(&fpdn->list, &failed_ddw_pdn_list); in enable_ddw()
1703 /* If we have persistent memory and the window size is not big enough in enable_ddw()
1704 * to directly map both RAM and vPMEM, then we need to set DMA limit. in enable_ddw()
1707 dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + in enable_ddw()
1734 struct iommu_table_group *table_group = PCI_DN(pdn)->table_group; in spapr_tce_init_table_group()
1740 /* Only for normal boot with default window. Doesn't matter during in spapr_tce_init_table_group()
1746 if (table_group->max_dynamic_windows_supported != 0) in spapr_tce_init_table_group()
1749 table_group->tce32_start = be64_to_cpu(prop.dma_base); in spapr_tce_init_table_group()
1750 table_group->tce32_size = 1 << be32_to_cpu(prop.window_shift); in spapr_tce_init_table_group()
1752 if (!of_find_property(pdn, "ibm,dma-window", NULL)) in spapr_tce_init_table_group()
1753 dev_err(&pdev->dev, "default dma window missing!\n"); in spapr_tce_init_table_group()
1755 ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable", in spapr_tce_init_table_group()
1758 table_group->max_dynamic_windows_supported = -1; in spapr_tce_init_table_group()
1764 dev_err(&pdev->dev, "%s: query_ddw failed\n", __func__); in spapr_tce_init_table_group()
1765 table_group->max_dynamic_windows_supported = -1; in spapr_tce_init_table_group()
1770 table_group->max_dynamic_windows_supported = 1; in spapr_tce_init_table_group()
1772 table_group->max_dynamic_windows_supported = IOMMU_TABLE_GROUP_MAX_TABLES; in spapr_tce_init_table_group()
1774 table_group->max_levels = 1; in spapr_tce_init_table_group()
1775 table_group->pgsizes |= query_page_size_to_mask(query.page_size); in spapr_tce_init_table_group()
1788 * contain the dma-window properties per-device and not necessarily in pci_dma_dev_setup_pSeriesLP()
1790 * either hit a dma-window property, OR find a parent with a table in pci_dma_dev_setup_pSeriesLP()
1799 "no DMA window found for pci dev=%s dn=%pOF\n", in pci_dma_dev_setup_pSeriesLP()
1806 if (!pci->table_group) { in pci_dma_dev_setup_pSeriesLP()
1807 pci->table_group = iommu_pseries_alloc_group(pci->phb->node); in pci_dma_dev_setup_pSeriesLP()
1808 tbl = pci->table_group->tables[0]; in pci_dma_dev_setup_pSeriesLP()
1810 iommu_table_setparms_common(tbl, pci->phb->bus->number, in pci_dma_dev_setup_pSeriesLP()
1817 iommu_init_table(tbl, pci->phb->node, 0, 0); in pci_dma_dev_setup_pSeriesLP()
1818 iommu_register_group(pci->table_group, in pci_dma_dev_setup_pSeriesLP()
1819 pci_domain_nr(pci->phb->bus), 0); in pci_dma_dev_setup_pSeriesLP()
1820 pr_debug(" created table: %p\n", pci->table_group); in pci_dma_dev_setup_pSeriesLP()
1822 pr_debug(" found DMA window, table: %p\n", pci->table_group); in pci_dma_dev_setup_pSeriesLP()
1827 set_iommu_table_base(&dev->dev, pci->table_group->tables[0]); in pci_dma_dev_setup_pSeriesLP()
1828 iommu_add_device(pci->table_group, &dev->dev); in pci_dma_dev_setup_pSeriesLP()
1835 /* only attempt to use a new window if 64-bit DMA is requested */ in iommu_bypass_supported_pSeriesLP()
1839 dev_dbg(&pdev->dev, "node is %pOF\n", dn); in iommu_bypass_supported_pSeriesLP()
1842 * the device tree might contain the dma-window properties in iommu_bypass_supported_pSeriesLP()
1843 * per-device and not necessarily for the bus. So we need to in iommu_bypass_supported_pSeriesLP()
1844 * search upwards in the tree until we either hit a dma-window in iommu_bypass_supported_pSeriesLP()
1859 * one if it matches table_group->tce32_start/tce32_size/page_shift.
1868 size = window_size >> (page_shift - 3); in spapr_tce_get_table_size()
1890 copy_property(pdn, "ibm,dma-window-saved", "ibm,dma-window"); in restore_default_dma_window()
1896 struct dma_win *window; in remove_dynamic_dma_windows() local
1900 if (find_existing_ddw(pdn, &pdev->dev.archdata.dma_offset, &len, &direct_mapping)) { in remove_dynamic_dma_windows()
1904 WARN_ON(!pci->table_group->tables[0] && !pci->table_group->tables[1]); in remove_dynamic_dma_windows()
1906 if (pci->table_group->tables[1]) { in remove_dynamic_dma_windows()
1907 iommu_tce_table_put(pci->table_group->tables[1]); in remove_dynamic_dma_windows()
1908 pci->table_group->tables[1] = NULL; in remove_dynamic_dma_windows()
1909 } else if (pci->table_group->tables[0]) { in remove_dynamic_dma_windows()
1910 /* Default window was removed and only the DDW exists */ in remove_dynamic_dma_windows()
1911 iommu_tce_table_put(pci->table_group->tables[0]); in remove_dynamic_dma_windows()
1912 pci->table_group->tables[0] = NULL; in remove_dynamic_dma_windows()
1916 list_for_each_entry(window, &dma_win_list, list) { in remove_dynamic_dma_windows()
1917 if (window->device == pdn) { in remove_dynamic_dma_windows()
1918 list_del(&window->list); in remove_dynamic_dma_windows()
1919 kfree(window); in remove_dynamic_dma_windows()
1941 dev_warn(&pdev->dev, "No table_group configured for the node %pOF\n", pdn); in pseries_setup_default_iommu_config()
1942 return -1; in pseries_setup_default_iommu_config()
1946 /* The default window is restored if not present already on removal of DDW. in pseries_setup_default_iommu_config()
1950 * So, restore the default window with reset-pe-dma call explicitly. in pseries_setup_default_iommu_config()
1954 default_prop = of_get_property(pdn, "ibm,dma-window", NULL); in pseries_setup_default_iommu_config()
1956 tbl = iommu_pseries_alloc_table(pci->phb->node); in pseries_setup_default_iommu_config()
1958 dev_err(&pdev->dev, "couldn't create new IOMMU table\n"); in pseries_setup_default_iommu_config()
1959 return -1; in pseries_setup_default_iommu_config()
1962 iommu_table_setparms_common(tbl, pci->phb->bus->number, liobn, offset, in pseries_setup_default_iommu_config()
1965 iommu_init_table(tbl, pci->phb->node, 0, 0); in pseries_setup_default_iommu_config()
1967 pci->table_group->tables[0] = tbl; in pseries_setup_default_iommu_config()
1968 set_iommu_table_base(&pdev->dev, tbl); in pseries_setup_default_iommu_config()
1976 if ((window_size <= table_group->tce32_size) && in is_default_window_request()
1987 struct pci_dev *pdev = iommu_group_get_first_pci_dev(table_group->group); in spapr_tce_create_table()
1998 struct dma_win *window; in spapr_tce_create_table() local
2006 return -EINVAL; in spapr_tce_create_table()
2012 ret = -ENODEV; in spapr_tce_create_table()
2015 if (!pdn || !PCI_DN(pdn)) { /* Niether of 32s|64-bit exist! */ in spapr_tce_create_table()
2016 dev_warn(&pdev->dev, "No dma-windows exist for the node %pOF\n", pdn); in spapr_tce_create_table()
2023 if (fpdn->pdn == pdn) { in spapr_tce_create_table()
2024 dev_info(&pdev->dev, "%pOF in failed DDW device list\n", pdn); in spapr_tce_create_table()
2029 tbl = iommu_pseries_alloc_table(pci->phb->node); in spapr_tce_create_table()
2031 dev_dbg(&pdev->dev, "couldn't create new IOMMU table\n"); in spapr_tce_create_table()
2037 /* The request is not for default window? Ensure there is no DDW window already */ in spapr_tce_create_table()
2039 if (find_existing_ddw(pdn, &pdev->dev.archdata.dma_offset, &len, in spapr_tce_create_table()
2041 dev_warn(&pdev->dev, "%pOF: 64-bit window already present.", pdn); in spapr_tce_create_table()
2042 ret = -EPERM; in spapr_tce_create_table()
2046 /* Request is for Default window, ensure there is no DDW if there is a in spapr_tce_create_table()
2047 * need to reset. reset-pe otherwise removes the DDW also in spapr_tce_create_table()
2049 default_prop = of_get_property(pdn, "ibm,dma-window", NULL); in spapr_tce_create_table()
2051 if (find_existing_ddw(pdn, &pdev->dev.archdata.dma_offset, &len, in spapr_tce_create_table()
2053 …arn(&pdev->dev, "%pOF: Attempt to create window#0 when 64-bit window is present. Preventing the at… in spapr_tce_create_table()
2055 ret = -EPERM; in spapr_tce_create_table()
2061 default_prop = of_get_property(pdn, "ibm,dma-window", NULL); in spapr_tce_create_table()
2063 /* Limit the default window size to window_size */ in spapr_tce_create_table()
2064 iommu_table_setparms_common(tbl, pci->phb->bus->number, liobn, in spapr_tce_create_table()
2068 iommu_init_table(tbl, pci->phb->node, start, end); in spapr_tce_create_table()
2070 table_group->tables[0] = tbl; in spapr_tce_create_table()
2079 ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable", in spapr_tce_create_table()
2082 dev_info(&pdev->dev, "ibm,ddw-applicable not found\n"); in spapr_tce_create_table()
2085 ret = -ENODEV; in spapr_tce_create_table()
2091 ret = -ENODEV; in spapr_tce_create_table()
2094 if (query.largest_available_block < (1ULL << (len - page_shift))) { in spapr_tce_create_table()
2095 dev_dbg(&pdev->dev, "can't map window 0x%llx with %llu %llu-sized pages\n", in spapr_tce_create_table()
2098 ret = -EINVAL; /* Retry with smaller window size */ in spapr_tce_create_table()
2114 dev_err(&pdev->dev, "unable to add DMA window property for %pOF: %ld", pdn, ret); in spapr_tce_create_table()
2117 ret = -ENODEV; in spapr_tce_create_table()
2119 window = ddw_list_new_entry(pdn, win64->value); in spapr_tce_create_table()
2120 if (!window) in spapr_tce_create_table()
2123 window->direct = false; in spapr_tce_create_table()
2125 for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) { in spapr_tce_create_table()
2129 if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) { in spapr_tce_create_table()
2130 start = pci->phb->mem_resources[i].start; in spapr_tce_create_table()
2131 end = pci->phb->mem_resources[i].end; in spapr_tce_create_table()
2136 /* New table for using DDW instead of the default DMA window */ in spapr_tce_create_table()
2137 iommu_table_setparms_common(tbl, pci->phb->bus->number, create.liobn, win_addr, in spapr_tce_create_table()
2139 iommu_init_table(tbl, pci->phb->node, start, end); in spapr_tce_create_table()
2141 pci->table_group->tables[num] = tbl; in spapr_tce_create_table()
2142 set_iommu_table_base(&pdev->dev, tbl); in spapr_tce_create_table()
2143 pdev->dev.archdata.dma_offset = win_addr; in spapr_tce_create_table()
2146 list_add(&window->list, &dma_win_list); in spapr_tce_create_table()
2156 kfree(win64->name); in spapr_tce_create_table()
2157 kfree(win64->value); in spapr_tce_create_table()
2166 fpdn->pdn = pdn; in spapr_tce_create_table()
2167 list_add(&fpdn->list, &failed_ddw_pdn_list); in spapr_tce_create_table()
2176 tbl->it_allocated_size = spapr_tce_get_table_size(page_shift, window_size, levels); in spapr_tce_create_table()
2185 if (((tbl->it_size << tbl->it_page_shift) <= table_group->tce32_size) && in is_default_window_table()
2186 (tbl->it_page_shift == IOMMU_PAGE_SHIFT_4K)) in is_default_window_table()
2195 return tbl == table_group->tables[num] ? 0 : -EPERM; in spapr_tce_set_window()
2200 struct pci_dev *pdev = iommu_group_get_first_pci_dev(table_group->group); in spapr_tce_unset_window()
2202 struct iommu_table *tbl = table_group->tables[num]; in spapr_tce_unset_window()
2204 struct dma_win *window; in spapr_tce_unset_window() local
2206 int ret = -ENODEV; in spapr_tce_unset_window()
2211 win_name = "ibm,dma-window"; in spapr_tce_unset_window()
2216 if (!pdn || !PCI_DN(pdn)) { /* Niether of 32s|64-bit exist! */ in spapr_tce_unset_window()
2217 dev_warn(&pdev->dev, "No dma-windows exist for the node %pOF\n", pdn); in spapr_tce_unset_window()
2229 list_for_each_entry(window, &dma_win_list, list) { in spapr_tce_unset_window()
2230 if (window->device == pdn) { in spapr_tce_unset_window()
2231 list_del(&window->list); in spapr_tce_unset_window()
2232 kfree(window); in spapr_tce_unset_window()
2239 iommu_tce_table_put(table_group->tables[num]); in spapr_tce_unset_window()
2240 table_group->tables[num] = NULL; in spapr_tce_unset_window()
2250 fpdn->pdn = pdn; in spapr_tce_unset_window()
2251 list_add(&fpdn->list, &failed_ddw_pdn_list); in spapr_tce_unset_window()
2261 struct iommu_table *tbl = table_group->tables[0]; in spapr_tce_take_ownership()
2269 if (!tbl && (table_group->max_dynamic_windows_supported != 1)) in spapr_tce_take_ownership()
2275 if (!pdn || !PCI_DN(pdn)) { /* Niether of 32s|64-bit exist! */ in spapr_tce_take_ownership()
2276 dev_warn(&pdev->dev, "No dma-windows exist for the node %pOF\n", pdn); in spapr_tce_take_ownership()
2278 return -1; in spapr_tce_take_ownership()
2282 * Though rtas call reset-pe removes the DDW, it doesn't clear the entries on the table in spapr_tce_take_ownership()
2284 * is fine for PEs with 2 DMA windows where the second window is created with create-pe in spapr_tce_take_ownership()
2285 * at which point the table is cleared. However, on VFs having only one DMA window, the in spapr_tce_take_ownership()
2286 * default window would end up seeing the entries left over from the direct map done in spapr_tce_take_ownership()
2287 * on the second window. So, remove the ddw explicitly so that clean_dma_window() in spapr_tce_take_ownership()
2291 dev_warn(&pdev->dev, "The existing DDW removal failed for node %pOF\n", pdn); in spapr_tce_take_ownership()
2293 return -1; in spapr_tce_take_ownership()
2296 /* The table_group->tables[0] is not null now, it must be the default window in spapr_tce_take_ownership()
2299 if (table_group->tables[0]) { in spapr_tce_take_ownership()
2300 remove_dma_window_named(pdn, true, "ibm,dma-window", true); in spapr_tce_take_ownership()
2302 table_group->tables[0] = NULL; in spapr_tce_take_ownership()
2313 struct iommu_table *tbl = table_group->tables[0]; in spapr_tce_release_ownership()
2315 if (tbl) { /* Default window already restored */ in spapr_tce_release_ownership()
2321 /* Restore the default window */ in spapr_tce_release_ownership()
2342 struct dma_win *window; in iommu_mem_notifier() local
2349 list_for_each_entry(window, &dma_win_list, list) { in iommu_mem_notifier()
2350 if (window->direct) { in iommu_mem_notifier()
2351 ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, in iommu_mem_notifier()
2352 arg->nr_pages, window->prop); in iommu_mem_notifier()
2361 list_for_each_entry(window, &dma_win_list, list) { in iommu_mem_notifier()
2362 if (window->direct) { in iommu_mem_notifier()
2363 ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, in iommu_mem_notifier()
2364 arg->nr_pages, window->prop); in iommu_mem_notifier()
2387 struct device_node *np = rd->dn; in iommu_reconfig_notifier()
2389 struct dma_win *window; in iommu_reconfig_notifier() local
2395 * notifier again, which causes dead-lock on the in iommu_reconfig_notifier()
2396 * read-write semaphore of the notifier chain. So in iommu_reconfig_notifier()
2403 if (pci && pci->table_group) in iommu_reconfig_notifier()
2404 iommu_pseries_free_group(pci->table_group, in iommu_reconfig_notifier()
2405 np->full_name); in iommu_reconfig_notifier()
2408 list_for_each_entry(window, &dma_win_list, list) { in iommu_reconfig_notifier()
2409 if (window->device == np) { in iommu_reconfig_notifier()
2410 list_del(&window->list); in iommu_reconfig_notifier()
2411 kfree(window); in iommu_reconfig_notifier()
2431 if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) in iommu_init_early_pSeries()
2471 struct device_node *pdn, *dn = pdev->dev.of_node; in pSeries_pci_device_group()
2477 return ERR_PTR(-ENODEV); in pSeries_pci_device_group()
2480 if (!pci->table_group) in pSeries_pci_device_group()
2481 return ERR_PTR(-ENODEV); in pSeries_pci_device_group()
2483 grp = pci->table_group->group; in pSeries_pci_device_group()
2485 return ERR_PTR(-ENODEV); in pSeries_pci_device_group()