Lines Matching +full:not +full:- +full:swapped
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2020 HabanaLabs, Ltd.
15 * hl_mmu_v2_ctx_init() - initialize a context for using the MMU module.
20 * Return: 0 on success, non-zero otherwise.
24 hash_init(ctx->mmu_shadow_hash); in hl_mmu_v2_ctx_init()
30 * hl_mmu_v2_ctx_fini - disable a ctx from using the mmu module
35 * - Free any pgts which were not freed yet
36 * - Free the mutex
37 * - Free DRAM default page mapping hops
41 struct hl_device *hdev = ctx->hdev; in hl_mmu_v2_ctx_fini()
46 if (!hash_empty(ctx->mmu_shadow_hash)) in hl_mmu_v2_ctx_fini()
47 dev_err(hdev->dev, "ctx %d is freed while it has pgts in use\n", in hl_mmu_v2_ctx_fini()
48 ctx->asid); in hl_mmu_v2_ctx_fini()
50 hash_for_each_safe(ctx->mmu_shadow_hash, i, tmp, pgt_info, node) { in hl_mmu_v2_ctx_fini()
51 dev_err_ratelimited(hdev->dev, in hl_mmu_v2_ctx_fini()
52 "pgt_info of addr 0x%llx of asid %d was not destroyed, num_ptes: %d\n", in hl_mmu_v2_ctx_fini()
53 pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes); in hl_mmu_v2_ctx_fini()
62 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_mmu_v2_unmap()
63 struct hl_device *hdev = ctx->hdev; in hl_mmu_v2_unmap()
70 return -EINVAL; in hl_mmu_v2_unmap()
72 mmu_prop = &prop->dmmu; in hl_mmu_v2_unmap()
74 hop_last = mmu_prop->num_hops - 1; in hl_mmu_v2_unmap()
76 scrambled_virt_addr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in hl_mmu_v2_unmap()
82 return -EFAULT; in hl_mmu_v2_unmap()
86 for (i = 1 ; i < mmu_prop->num_hops ; i++) { in hl_mmu_v2_unmap()
94 return -EFAULT; in hl_mmu_v2_unmap()
98 if ((i <= hop_last) && (curr_pte & mmu_prop->last_mask)) { in hl_mmu_v2_unmap()
106 dev_err(hdev->dev, "DRAM unmapping should use huge pages only\n"); in hl_mmu_v2_unmap()
107 return -EFAULT; in hl_mmu_v2_unmap()
113 for (i = hop_last ; i > 0 ; i--) { in hl_mmu_v2_unmap()
124 dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n", in hl_mmu_v2_unmap()
127 return -EINVAL; in hl_mmu_v2_unmap()
135 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_mmu_v2_map()
137 struct hl_device *hdev = ctx->hdev; in hl_mmu_v2_map()
143 return -EINVAL; in hl_mmu_v2_map()
145 mmu_prop = &prop->dmmu; in hl_mmu_v2_map()
147 hop_last = mmu_prop->num_hops - 1; in hl_mmu_v2_map()
149 scrambled_virt_addr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in hl_mmu_v2_map()
150 scrambled_phys_addr = hdev->asic_funcs->scramble_addr(hdev, phys_addr); in hl_mmu_v2_map()
162 rc = -ENOMEM; in hl_mmu_v2_map()
169 rc = -EINVAL; in hl_mmu_v2_map()
174 rc = -EINVAL; in hl_mmu_v2_map()
182 dev_err(hdev->dev, in hl_mmu_v2_map()
187 dev_dbg(hdev->dev, "hop%d pte: 0x%llx (0x%llx)\n", in hl_mmu_v2_map()
191 rc = -EINVAL; in hl_mmu_v2_map()
196 | mmu_prop->last_mask | PAGE_PRESENT_MASK; in hl_mmu_v2_map()
201 /* for each new hop, add its address to the table of previous-hop */ in hl_mmu_v2_map()
205 hl_mmu_dr_write_pte(ctx, hop_pte_addr[i - 1], curr_pte); in hl_mmu_v2_map()
207 if (i - 1) in hl_mmu_v2_map()
208 hl_mmu_dr_get_pte(ctx, hop_addr[i - 1]); in hl_mmu_v2_map()
224 * hl_mmu_v2_swap_out - marks all mapping of the given ctx as swapped out
235 * hl_mmu_v2_swap_in - marks all mapping of the given ctx as swapped in
247 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_mmu_v2_get_tlb_info()
248 struct hl_device *hdev = ctx->hdev; in hl_mmu_v2_get_tlb_info()
253 is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_mmu_v2_get_tlb_info()
254 prop->dmmu.start_addr, in hl_mmu_v2_get_tlb_info()
255 prop->dmmu.end_addr); in hl_mmu_v2_get_tlb_info()
259 return -EINVAL; in hl_mmu_v2_get_tlb_info()
261 mmu_prop = &prop->dmmu; in hl_mmu_v2_get_tlb_info()
262 hops->range_type = HL_VA_RANGE_TYPE_DRAM; in hl_mmu_v2_get_tlb_info()
264 hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in hl_mmu_v2_get_tlb_info()
266 hops->hop_info[0].hop_addr = hl_mmu_dr_get_phys_hop0_addr(ctx); in hl_mmu_v2_get_tlb_info()
267 hops->hop_info[0].hop_pte_addr = hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, 0, in hl_mmu_v2_get_tlb_info()
268 hops->hop_info[0].hop_addr, in hl_mmu_v2_get_tlb_info()
269 hops->scrambled_vaddr); in hl_mmu_v2_get_tlb_info()
270 if (hops->hop_info[0].hop_pte_addr == U64_MAX) in hl_mmu_v2_get_tlb_info()
271 return -EFAULT; in hl_mmu_v2_get_tlb_info()
273 hops->hop_info[0].hop_pte_val = hdev->asic_funcs->read_pte(hdev, in hl_mmu_v2_get_tlb_info()
274 hops->hop_info[0].hop_pte_addr); in hl_mmu_v2_get_tlb_info()
275 if (hops->hop_info[0].hop_pte_val == U64_MAX) in hl_mmu_v2_get_tlb_info()
276 return -EFAULT; in hl_mmu_v2_get_tlb_info()
278 for (i = 1 ; i < mmu_prop->num_hops ; i++) { in hl_mmu_v2_get_tlb_info()
279 hops->hop_info[i].hop_addr = in hl_mmu_v2_get_tlb_info()
280 hl_mmu_get_next_hop_addr(ctx, hops->hop_info[i - 1].hop_pte_val); in hl_mmu_v2_get_tlb_info()
281 if (hops->hop_info[i].hop_addr == ULLONG_MAX) in hl_mmu_v2_get_tlb_info()
282 return -EFAULT; in hl_mmu_v2_get_tlb_info()
284 hops->hop_info[i].hop_pte_addr = in hl_mmu_v2_get_tlb_info()
286 hops->hop_info[i].hop_addr, in hl_mmu_v2_get_tlb_info()
287 hops->scrambled_vaddr); in hl_mmu_v2_get_tlb_info()
288 if (hops->hop_info[i].hop_pte_addr == U64_MAX) in hl_mmu_v2_get_tlb_info()
289 return -EFAULT; in hl_mmu_v2_get_tlb_info()
291 hops->hop_info[i].hop_pte_val = in hl_mmu_v2_get_tlb_info()
292 hdev->asic_funcs->read_pte(hdev, in hl_mmu_v2_get_tlb_info()
293 hops->hop_info[i].hop_pte_addr); in hl_mmu_v2_get_tlb_info()
295 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK)) in hl_mmu_v2_get_tlb_info()
296 return -EFAULT; in hl_mmu_v2_get_tlb_info()
298 if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask) in hl_mmu_v2_get_tlb_info()
303 if (i == mmu_prop->num_hops) in hl_mmu_v2_get_tlb_info()
304 return -EFAULT; in hl_mmu_v2_get_tlb_info()
306 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK)) in hl_mmu_v2_get_tlb_info()
307 return -EFAULT; in hl_mmu_v2_get_tlb_info()
309 if (hops->scrambled_vaddr != virt_addr) in hl_mmu_v2_get_tlb_info()
310 hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr in hl_mmu_v2_get_tlb_info()
311 (hdev, hops->hop_info[i].hop_pte_val); in hl_mmu_v2_get_tlb_info()
313 hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val; in hl_mmu_v2_get_tlb_info()
315 hops->used_hops = i + 1; in hl_mmu_v2_get_tlb_info()
321 * hl_mmu_v2_prepare - prepare mmu_if for working with mmu v2
328 mmu->init = hl_mmu_dr_init; in hl_mmu_v2_set_funcs()
329 mmu->fini = hl_mmu_dr_fini; in hl_mmu_v2_set_funcs()
330 mmu->ctx_init = hl_mmu_v2_ctx_init; in hl_mmu_v2_set_funcs()
331 mmu->ctx_fini = hl_mmu_v2_ctx_fini; in hl_mmu_v2_set_funcs()
332 mmu->map = hl_mmu_v2_map; in hl_mmu_v2_set_funcs()
333 mmu->unmap = hl_mmu_v2_unmap; in hl_mmu_v2_set_funcs()
334 mmu->flush = hl_mmu_dr_flush; in hl_mmu_v2_set_funcs()
335 mmu->swap_out = hl_mmu_v2_swap_out; in hl_mmu_v2_set_funcs()
336 mmu->swap_in = hl_mmu_v2_swap_in; in hl_mmu_v2_set_funcs()
337 mmu->get_tlb_info = hl_mmu_v2_get_tlb_info; in hl_mmu_v2_set_funcs()