15ce3bf3cSBen Skeggs /*
25ce3bf3cSBen Skeggs * Copyright 2010 Red Hat Inc.
35ce3bf3cSBen Skeggs *
45ce3bf3cSBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a
55ce3bf3cSBen Skeggs * copy of this software and associated documentation files (the "Software"),
65ce3bf3cSBen Skeggs * to deal in the Software without restriction, including without limitation
75ce3bf3cSBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense,
85ce3bf3cSBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the
95ce3bf3cSBen Skeggs * Software is furnished to do so, subject to the following conditions:
105ce3bf3cSBen Skeggs *
115ce3bf3cSBen Skeggs * The above copyright notice and this permission notice shall be included in
125ce3bf3cSBen Skeggs * all copies or substantial portions of the Software.
135ce3bf3cSBen Skeggs *
145ce3bf3cSBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
155ce3bf3cSBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
165ce3bf3cSBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
175ce3bf3cSBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
185ce3bf3cSBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
195ce3bf3cSBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
205ce3bf3cSBen Skeggs * OTHER DEALINGS IN THE SOFTWARE.
215ce3bf3cSBen Skeggs *
225ce3bf3cSBen Skeggs * Authors: Ben Skeggs
235ce3bf3cSBen Skeggs */
24eea5cf0fSBen Skeggs #include "ummu.h"
25806a7335SBen Skeggs #include "vmm.h"
265ce3bf3cSBen Skeggs
2751645eb7SBen Skeggs #include <subdev/bar.h>
28c9582455SBen Skeggs #include <subdev/fb.h>
295ce3bf3cSBen Skeggs
30fd542a3eSBen Skeggs #include <nvif/if500d.h>
31b77791daSBen Skeggs #include <nvif/if900d.h>
32fd542a3eSBen Skeggs
33f1280394SBen Skeggs struct nvkm_mmu_ptp {
34f1280394SBen Skeggs struct nvkm_mmu_pt *pt;
35f1280394SBen Skeggs struct list_head head;
36f1280394SBen Skeggs u8 shift;
37f1280394SBen Skeggs u16 mask;
38f1280394SBen Skeggs u16 free;
39f1280394SBen Skeggs };
40f1280394SBen Skeggs
41f1280394SBen Skeggs static void
nvkm_mmu_ptp_put(struct nvkm_mmu * mmu,bool force,struct nvkm_mmu_pt * pt)42f1280394SBen Skeggs nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt)
43f1280394SBen Skeggs {
44f1280394SBen Skeggs const int slot = pt->base >> pt->ptp->shift;
45f1280394SBen Skeggs struct nvkm_mmu_ptp *ptp = pt->ptp;
46f1280394SBen Skeggs
47f1280394SBen Skeggs /* If there were no free slots in the parent allocation before,
48f1280394SBen Skeggs * there will be now, so return PTP to the cache.
49f1280394SBen Skeggs */
50f1280394SBen Skeggs if (!ptp->free)
51f1280394SBen Skeggs list_add(&ptp->head, &mmu->ptp.list);
52f1280394SBen Skeggs ptp->free |= BIT(slot);
53f1280394SBen Skeggs
54f1280394SBen Skeggs /* If there's no more sub-allocations, destroy PTP. */
55f1280394SBen Skeggs if (ptp->free == ptp->mask) {
56f1280394SBen Skeggs nvkm_mmu_ptc_put(mmu, force, &ptp->pt);
57f1280394SBen Skeggs list_del(&ptp->head);
58f1280394SBen Skeggs kfree(ptp);
59f1280394SBen Skeggs }
60f1280394SBen Skeggs
61f1280394SBen Skeggs kfree(pt);
62f1280394SBen Skeggs }
63f1280394SBen Skeggs
643b54befdSBen Skeggs static struct nvkm_mmu_pt *
nvkm_mmu_ptp_get(struct nvkm_mmu * mmu,u32 size,bool zero)65f1280394SBen Skeggs nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero)
66f1280394SBen Skeggs {
67f1280394SBen Skeggs struct nvkm_mmu_pt *pt;
68f1280394SBen Skeggs struct nvkm_mmu_ptp *ptp;
69f1280394SBen Skeggs int slot;
70f1280394SBen Skeggs
71f1280394SBen Skeggs if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL)))
72f1280394SBen Skeggs return NULL;
73f1280394SBen Skeggs
74f1280394SBen Skeggs ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head);
75f1280394SBen Skeggs if (!ptp) {
76f1280394SBen Skeggs /* Need to allocate a new parent to sub-allocate from. */
77f1280394SBen Skeggs if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) {
78f1280394SBen Skeggs kfree(pt);
79f1280394SBen Skeggs return NULL;
80f1280394SBen Skeggs }
81f1280394SBen Skeggs
82f1280394SBen Skeggs ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false);
83f1280394SBen Skeggs if (!ptp->pt) {
84f1280394SBen Skeggs kfree(ptp);
85f1280394SBen Skeggs kfree(pt);
86f1280394SBen Skeggs return NULL;
87f1280394SBen Skeggs }
88f1280394SBen Skeggs
89f1280394SBen Skeggs ptp->shift = order_base_2(size);
90f1280394SBen Skeggs slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift;
91f1280394SBen Skeggs ptp->mask = (1 << slot) - 1;
92f1280394SBen Skeggs ptp->free = ptp->mask;
93f1280394SBen Skeggs list_add(&ptp->head, &mmu->ptp.list);
94f1280394SBen Skeggs }
95f1280394SBen Skeggs pt->ptp = ptp;
96f1280394SBen Skeggs pt->sub = true;
97f1280394SBen Skeggs
98f1280394SBen Skeggs /* Sub-allocate from parent object, removing PTP from cache
99f1280394SBen Skeggs * if there's no more free slots left.
100f1280394SBen Skeggs */
101f1280394SBen Skeggs slot = __ffs(ptp->free);
102f1280394SBen Skeggs ptp->free &= ~BIT(slot);
103f1280394SBen Skeggs if (!ptp->free)
104f1280394SBen Skeggs list_del(&ptp->head);
105f1280394SBen Skeggs
106f1280394SBen Skeggs pt->memory = pt->ptp->pt->memory;
107f1280394SBen Skeggs pt->base = slot << ptp->shift;
108f1280394SBen Skeggs pt->addr = pt->ptp->pt->addr + pt->base;
109f1280394SBen Skeggs return pt;
110f1280394SBen Skeggs }
111f1280394SBen Skeggs
1129a45ddaaSBen Skeggs struct nvkm_mmu_ptc {
1139a45ddaaSBen Skeggs struct list_head head;
1149a45ddaaSBen Skeggs struct list_head item;
1159a45ddaaSBen Skeggs u32 size;
1169a45ddaaSBen Skeggs u32 refs;
1179a45ddaaSBen Skeggs };
1189a45ddaaSBen Skeggs
1199a45ddaaSBen Skeggs static inline struct nvkm_mmu_ptc *
nvkm_mmu_ptc_find(struct nvkm_mmu * mmu,u32 size)1209a45ddaaSBen Skeggs nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size)
1219a45ddaaSBen Skeggs {
1229a45ddaaSBen Skeggs struct nvkm_mmu_ptc *ptc;
1239a45ddaaSBen Skeggs
1249a45ddaaSBen Skeggs list_for_each_entry(ptc, &mmu->ptc.list, head) {
1259a45ddaaSBen Skeggs if (ptc->size == size)
1269a45ddaaSBen Skeggs return ptc;
1279a45ddaaSBen Skeggs }
1289a45ddaaSBen Skeggs
1299a45ddaaSBen Skeggs ptc = kmalloc(sizeof(*ptc), GFP_KERNEL);
1309a45ddaaSBen Skeggs if (ptc) {
1319a45ddaaSBen Skeggs INIT_LIST_HEAD(&ptc->item);
1329a45ddaaSBen Skeggs ptc->size = size;
1339a45ddaaSBen Skeggs ptc->refs = 0;
1349a45ddaaSBen Skeggs list_add(&ptc->head, &mmu->ptc.list);
1359a45ddaaSBen Skeggs }
1369a45ddaaSBen Skeggs
1379a45ddaaSBen Skeggs return ptc;
1389a45ddaaSBen Skeggs }
1399a45ddaaSBen Skeggs
1409a45ddaaSBen Skeggs void
nvkm_mmu_ptc_put(struct nvkm_mmu * mmu,bool force,struct nvkm_mmu_pt ** ppt)1419a45ddaaSBen Skeggs nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt)
1429a45ddaaSBen Skeggs {
1439a45ddaaSBen Skeggs struct nvkm_mmu_pt *pt = *ppt;
1449a45ddaaSBen Skeggs if (pt) {
145f1280394SBen Skeggs /* Handle sub-allocated page tables. */
146f1280394SBen Skeggs if (pt->sub) {
147f1280394SBen Skeggs mutex_lock(&mmu->ptp.mutex);
148f1280394SBen Skeggs nvkm_mmu_ptp_put(mmu, force, pt);
149f1280394SBen Skeggs mutex_unlock(&mmu->ptp.mutex);
150f1280394SBen Skeggs return;
151f1280394SBen Skeggs }
152f1280394SBen Skeggs
1539a45ddaaSBen Skeggs /* Either cache or free the object. */
1549a45ddaaSBen Skeggs mutex_lock(&mmu->ptc.mutex);
1559a45ddaaSBen Skeggs if (pt->ptc->refs < 8 /* Heuristic. */ && !force) {
1569a45ddaaSBen Skeggs list_add_tail(&pt->head, &pt->ptc->item);
1579a45ddaaSBen Skeggs pt->ptc->refs++;
1589a45ddaaSBen Skeggs } else {
1599a45ddaaSBen Skeggs nvkm_memory_unref(&pt->memory);
1609a45ddaaSBen Skeggs kfree(pt);
1619a45ddaaSBen Skeggs }
1629a45ddaaSBen Skeggs mutex_unlock(&mmu->ptc.mutex);
1639a45ddaaSBen Skeggs }
1649a45ddaaSBen Skeggs }
1659a45ddaaSBen Skeggs
1669a45ddaaSBen Skeggs struct nvkm_mmu_pt *
nvkm_mmu_ptc_get(struct nvkm_mmu * mmu,u32 size,u32 align,bool zero)1679a45ddaaSBen Skeggs nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero)
1689a45ddaaSBen Skeggs {
1699a45ddaaSBen Skeggs struct nvkm_mmu_ptc *ptc;
1709a45ddaaSBen Skeggs struct nvkm_mmu_pt *pt;
1719a45ddaaSBen Skeggs int ret;
1729a45ddaaSBen Skeggs
173f1280394SBen Skeggs /* Sub-allocated page table (ie. GP100 LPT). */
174f1280394SBen Skeggs if (align < 0x1000) {
175f1280394SBen Skeggs mutex_lock(&mmu->ptp.mutex);
176f1280394SBen Skeggs pt = nvkm_mmu_ptp_get(mmu, align, zero);
177f1280394SBen Skeggs mutex_unlock(&mmu->ptp.mutex);
178f1280394SBen Skeggs return pt;
179f1280394SBen Skeggs }
180f1280394SBen Skeggs
1819a45ddaaSBen Skeggs /* Lookup cache for this page table size. */
1829a45ddaaSBen Skeggs mutex_lock(&mmu->ptc.mutex);
1839a45ddaaSBen Skeggs ptc = nvkm_mmu_ptc_find(mmu, size);
1849a45ddaaSBen Skeggs if (!ptc) {
1859a45ddaaSBen Skeggs mutex_unlock(&mmu->ptc.mutex);
1869a45ddaaSBen Skeggs return NULL;
1879a45ddaaSBen Skeggs }
1889a45ddaaSBen Skeggs
1899a45ddaaSBen Skeggs /* If there's a free PT in the cache, reuse it. */
1909a45ddaaSBen Skeggs pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head);
1919a45ddaaSBen Skeggs if (pt) {
1929a45ddaaSBen Skeggs if (zero)
1939a45ddaaSBen Skeggs nvkm_fo64(pt->memory, 0, 0, size >> 3);
1949a45ddaaSBen Skeggs list_del(&pt->head);
1959a45ddaaSBen Skeggs ptc->refs--;
1969a45ddaaSBen Skeggs mutex_unlock(&mmu->ptc.mutex);
1979a45ddaaSBen Skeggs return pt;
1989a45ddaaSBen Skeggs }
1999a45ddaaSBen Skeggs mutex_unlock(&mmu->ptc.mutex);
2009a45ddaaSBen Skeggs
2019a45ddaaSBen Skeggs /* No such luck, we need to allocate. */
2029a45ddaaSBen Skeggs if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL)))
2039a45ddaaSBen Skeggs return NULL;
2049a45ddaaSBen Skeggs pt->ptc = ptc;
205f1280394SBen Skeggs pt->sub = false;
2069a45ddaaSBen Skeggs
2079a45ddaaSBen Skeggs ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
2089a45ddaaSBen Skeggs size, align, zero, &pt->memory);
2099a45ddaaSBen Skeggs if (ret) {
2109a45ddaaSBen Skeggs kfree(pt);
2119a45ddaaSBen Skeggs return NULL;
2129a45ddaaSBen Skeggs }
2139a45ddaaSBen Skeggs
2149a45ddaaSBen Skeggs pt->base = 0;
2159a45ddaaSBen Skeggs pt->addr = nvkm_memory_addr(pt->memory);
2169a45ddaaSBen Skeggs return pt;
2179a45ddaaSBen Skeggs }
2189a45ddaaSBen Skeggs
2199a45ddaaSBen Skeggs void
nvkm_mmu_ptc_dump(struct nvkm_mmu * mmu)2209a45ddaaSBen Skeggs nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu)
2219a45ddaaSBen Skeggs {
2229a45ddaaSBen Skeggs struct nvkm_mmu_ptc *ptc;
2239a45ddaaSBen Skeggs list_for_each_entry(ptc, &mmu->ptc.list, head) {
2249a45ddaaSBen Skeggs struct nvkm_mmu_pt *pt, *tt;
2259a45ddaaSBen Skeggs list_for_each_entry_safe(pt, tt, &ptc->item, head) {
2269a45ddaaSBen Skeggs nvkm_memory_unref(&pt->memory);
2279a45ddaaSBen Skeggs list_del(&pt->head);
2289a45ddaaSBen Skeggs kfree(pt);
2299a45ddaaSBen Skeggs }
2309a45ddaaSBen Skeggs }
2319a45ddaaSBen Skeggs }
2329a45ddaaSBen Skeggs
2339a45ddaaSBen Skeggs static void
nvkm_mmu_ptc_fini(struct nvkm_mmu * mmu)2349a45ddaaSBen Skeggs nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu)
2359a45ddaaSBen Skeggs {
2369a45ddaaSBen Skeggs struct nvkm_mmu_ptc *ptc, *ptct;
2379a45ddaaSBen Skeggs
2389a45ddaaSBen Skeggs list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) {
2399a45ddaaSBen Skeggs WARN_ON(!list_empty(&ptc->item));
2409a45ddaaSBen Skeggs list_del(&ptc->head);
2419a45ddaaSBen Skeggs kfree(ptc);
2429a45ddaaSBen Skeggs }
2439a45ddaaSBen Skeggs }
2449a45ddaaSBen Skeggs
2459a45ddaaSBen Skeggs static void
nvkm_mmu_ptc_init(struct nvkm_mmu * mmu)2469a45ddaaSBen Skeggs nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
2479a45ddaaSBen Skeggs {
2489a45ddaaSBen Skeggs mutex_init(&mmu->ptc.mutex);
2499a45ddaaSBen Skeggs INIT_LIST_HEAD(&mmu->ptc.list);
250f1280394SBen Skeggs mutex_init(&mmu->ptp.mutex);
251f1280394SBen Skeggs INIT_LIST_HEAD(&mmu->ptp.list);
2529a45ddaaSBen Skeggs }
2539a45ddaaSBen Skeggs
25451645eb7SBen Skeggs static void
nvkm_mmu_type(struct nvkm_mmu * mmu,int heap,u8 type)25551645eb7SBen Skeggs nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type)
25651645eb7SBen Skeggs {
25751645eb7SBen Skeggs if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) {
25851645eb7SBen Skeggs mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type;
25951645eb7SBen Skeggs mmu->type[mmu->type_nr].heap = heap;
26051645eb7SBen Skeggs mmu->type_nr++;
26151645eb7SBen Skeggs }
26251645eb7SBen Skeggs }
26351645eb7SBen Skeggs
26451645eb7SBen Skeggs static int
nvkm_mmu_heap(struct nvkm_mmu * mmu,u8 type,u64 size)26551645eb7SBen Skeggs nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size)
26651645eb7SBen Skeggs {
26751645eb7SBen Skeggs if (size) {
26851645eb7SBen Skeggs if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) {
26951645eb7SBen Skeggs mmu->heap[mmu->heap_nr].type = type;
27051645eb7SBen Skeggs mmu->heap[mmu->heap_nr].size = size;
27151645eb7SBen Skeggs return mmu->heap_nr++;
27251645eb7SBen Skeggs }
27351645eb7SBen Skeggs }
27451645eb7SBen Skeggs return -EINVAL;
27551645eb7SBen Skeggs }
27651645eb7SBen Skeggs
27751645eb7SBen Skeggs static void
nvkm_mmu_host(struct nvkm_mmu * mmu)27851645eb7SBen Skeggs nvkm_mmu_host(struct nvkm_mmu *mmu)
27951645eb7SBen Skeggs {
28051645eb7SBen Skeggs struct nvkm_device *device = mmu->subdev.device;
28151645eb7SBen Skeggs u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys;
28251645eb7SBen Skeggs int heap;
28351645eb7SBen Skeggs
28451645eb7SBen Skeggs /* Non-mappable system memory. */
28551645eb7SBen Skeggs heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL);
28651645eb7SBen Skeggs nvkm_mmu_type(mmu, heap, type);
28751645eb7SBen Skeggs
28851645eb7SBen Skeggs /* Non-coherent, cached, system memory.
28951645eb7SBen Skeggs *
29051645eb7SBen Skeggs * Block-linear mappings of system memory must be done through
29151645eb7SBen Skeggs * BAR1, and cannot be supported on systems where we're unable
29251645eb7SBen Skeggs * to map BAR1 with write-combining.
29351645eb7SBen Skeggs */
29451645eb7SBen Skeggs type |= NVKM_MEM_MAPPABLE;
29551645eb7SBen Skeggs if (!device->bar || device->bar->iomap_uncached)
29651645eb7SBen Skeggs nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
29751645eb7SBen Skeggs else
29851645eb7SBen Skeggs nvkm_mmu_type(mmu, heap, type);
29951645eb7SBen Skeggs
30051645eb7SBen Skeggs /* Coherent, cached, system memory.
30151645eb7SBen Skeggs *
30251645eb7SBen Skeggs * Unsupported on systems that aren't able to support snooped
30351645eb7SBen Skeggs * mappings, and also for block-linear mappings which must be
30451645eb7SBen Skeggs * done through BAR1.
30551645eb7SBen Skeggs */
30651645eb7SBen Skeggs type |= NVKM_MEM_COHERENT;
30751645eb7SBen Skeggs if (device->func->cpu_coherent)
30851645eb7SBen Skeggs nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
30951645eb7SBen Skeggs
31051645eb7SBen Skeggs /* Uncached system memory. */
31151645eb7SBen Skeggs nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED);
31251645eb7SBen Skeggs }
31351645eb7SBen Skeggs
31451645eb7SBen Skeggs static void
nvkm_mmu_vram(struct nvkm_mmu * mmu)31551645eb7SBen Skeggs nvkm_mmu_vram(struct nvkm_mmu *mmu)
31651645eb7SBen Skeggs {
31751645eb7SBen Skeggs struct nvkm_device *device = mmu->subdev.device;
31851645eb7SBen Skeggs struct nvkm_mm *mm = &device->fb->ram->vram;
319add42781SBen Skeggs const u64 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
320add42781SBen Skeggs const u64 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
321add42781SBen Skeggs const u64 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
32251645eb7SBen Skeggs u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
32351645eb7SBen Skeggs u8 heap = NVKM_MEM_VRAM;
32451645eb7SBen Skeggs int heapM, heapN, heapU;
32551645eb7SBen Skeggs
32651645eb7SBen Skeggs /* Mixed-memory doesn't support compression or display. */
32751645eb7SBen Skeggs heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT);
32851645eb7SBen Skeggs
32951645eb7SBen Skeggs heap |= NVKM_MEM_COMP;
33051645eb7SBen Skeggs heap |= NVKM_MEM_DISP;
33151645eb7SBen Skeggs heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT);
33251645eb7SBen Skeggs heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT);
33351645eb7SBen Skeggs
33451645eb7SBen Skeggs /* Add non-mappable VRAM types first so that they're preferred
33551645eb7SBen Skeggs * over anything else. Mixed-memory will be slower than other
33651645eb7SBen Skeggs * heaps, it's prioritised last.
33751645eb7SBen Skeggs */
33851645eb7SBen Skeggs nvkm_mmu_type(mmu, heapU, type);
33951645eb7SBen Skeggs nvkm_mmu_type(mmu, heapN, type);
34051645eb7SBen Skeggs nvkm_mmu_type(mmu, heapM, type);
34151645eb7SBen Skeggs
34251645eb7SBen Skeggs /* Add host memory types next, under the assumption that users
34351645eb7SBen Skeggs * wanting mappable memory want to use them as staging buffers
34451645eb7SBen Skeggs * or the like.
34551645eb7SBen Skeggs */
34651645eb7SBen Skeggs nvkm_mmu_host(mmu);
34751645eb7SBen Skeggs
34851645eb7SBen Skeggs /* Mappable VRAM types go last, as they're basically the worst
34951645eb7SBen Skeggs * possible type to ask for unless there's no other choice.
35051645eb7SBen Skeggs */
35151645eb7SBen Skeggs if (device->bar) {
35251645eb7SBen Skeggs /* Write-combined BAR1 access. */
35351645eb7SBen Skeggs type |= NVKM_MEM_MAPPABLE;
35451645eb7SBen Skeggs if (!device->bar->iomap_uncached) {
35551645eb7SBen Skeggs nvkm_mmu_type(mmu, heapN, type);
35651645eb7SBen Skeggs nvkm_mmu_type(mmu, heapM, type);
35751645eb7SBen Skeggs }
35851645eb7SBen Skeggs
35951645eb7SBen Skeggs /* Uncached BAR1 access. */
36051645eb7SBen Skeggs type |= NVKM_MEM_COHERENT;
36151645eb7SBen Skeggs type |= NVKM_MEM_UNCACHED;
36251645eb7SBen Skeggs nvkm_mmu_type(mmu, heapN, type);
36351645eb7SBen Skeggs nvkm_mmu_type(mmu, heapM, type);
36451645eb7SBen Skeggs }
36551645eb7SBen Skeggs }
36651645eb7SBen Skeggs
367c9582455SBen Skeggs static int
nvkm_mmu_oneinit(struct nvkm_subdev * subdev)368c9582455SBen Skeggs nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
369c9582455SBen Skeggs {
370c9582455SBen Skeggs struct nvkm_mmu *mmu = nvkm_mmu(subdev);
371806a7335SBen Skeggs
37251645eb7SBen Skeggs /* Determine available memory types. */
37351645eb7SBen Skeggs if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram)
37451645eb7SBen Skeggs nvkm_mmu_vram(mmu);
37551645eb7SBen Skeggs else
37651645eb7SBen Skeggs nvkm_mmu_host(mmu);
37751645eb7SBen Skeggs
378806a7335SBen Skeggs if (mmu->func->vmm.global) {
379f9463a4bSBen Skeggs int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
380f9463a4bSBen Skeggs "gart", &mmu->vmm);
381806a7335SBen Skeggs if (ret)
382806a7335SBen Skeggs return ret;
383806a7335SBen Skeggs }
384806a7335SBen Skeggs
385c9582455SBen Skeggs return 0;
386c9582455SBen Skeggs }
387c9582455SBen Skeggs
388c9582455SBen Skeggs static int
nvkm_mmu_init(struct nvkm_subdev * subdev)389c9582455SBen Skeggs nvkm_mmu_init(struct nvkm_subdev *subdev)
390c9582455SBen Skeggs {
391c9582455SBen Skeggs struct nvkm_mmu *mmu = nvkm_mmu(subdev);
392c9582455SBen Skeggs if (mmu->func->init)
393c9582455SBen Skeggs mmu->func->init(mmu);
394c9582455SBen Skeggs return 0;
395c9582455SBen Skeggs }
396c9582455SBen Skeggs
397c9582455SBen Skeggs static void *
nvkm_mmu_dtor(struct nvkm_subdev * subdev)398c9582455SBen Skeggs nvkm_mmu_dtor(struct nvkm_subdev *subdev)
399c9582455SBen Skeggs {
400c9582455SBen Skeggs struct nvkm_mmu *mmu = nvkm_mmu(subdev);
4019a45ddaaSBen Skeggs
402f9463a4bSBen Skeggs nvkm_vmm_unref(&mmu->vmm);
4039a45ddaaSBen Skeggs
4049a45ddaaSBen Skeggs nvkm_mmu_ptc_fini(mmu);
4055ec69c91SBen Skeggs mutex_destroy(&mmu->mutex);
406*5bf02571SBen Skeggs
407*5bf02571SBen Skeggs if (mmu->func->dtor)
408*5bf02571SBen Skeggs mmu->func->dtor(mmu);
409*5bf02571SBen Skeggs
41003b0ba7bSBen Skeggs return mmu;
411c9582455SBen Skeggs }
412c9582455SBen Skeggs
413c9582455SBen Skeggs static const struct nvkm_subdev_func
414c9582455SBen Skeggs nvkm_mmu = {
415c9582455SBen Skeggs .dtor = nvkm_mmu_dtor,
416c9582455SBen Skeggs .oneinit = nvkm_mmu_oneinit,
417c9582455SBen Skeggs .init = nvkm_mmu_init,
418c9582455SBen Skeggs };
419c9582455SBen Skeggs
420c9582455SBen Skeggs void
nvkm_mmu_ctor(const struct nvkm_mmu_func * func,struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_mmu * mmu)421c9582455SBen Skeggs nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
4226dd123baSBen Skeggs enum nvkm_subdev_type type, int inst, struct nvkm_mmu *mmu)
423c9582455SBen Skeggs {
4246dd123baSBen Skeggs nvkm_subdev_ctor(&nvkm_mmu, device, type, inst, &mmu->subdev);
425c9582455SBen Skeggs mmu->func = func;
426c9582455SBen Skeggs mmu->dma_bits = func->dma_bits;
4279a45ddaaSBen Skeggs nvkm_mmu_ptc_init(mmu);
4285ec69c91SBen Skeggs mutex_init(&mmu->mutex);
429eea5cf0fSBen Skeggs mmu->user.ctor = nvkm_ummu_new;
430eea5cf0fSBen Skeggs mmu->user.base = func->mmu.user;
431c9582455SBen Skeggs }
432c9582455SBen Skeggs
433c9582455SBen Skeggs int
nvkm_mmu_new_(const struct nvkm_mmu_func * func,struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_mmu ** pmmu)434c9582455SBen Skeggs nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
4356dd123baSBen Skeggs enum nvkm_subdev_type type, int inst, struct nvkm_mmu **pmmu)
436c9582455SBen Skeggs {
437c9582455SBen Skeggs if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL)))
438c9582455SBen Skeggs return -ENOMEM;
4396dd123baSBen Skeggs nvkm_mmu_ctor(func, device, type, inst, *pmmu);
440c9582455SBen Skeggs return 0;
441c9582455SBen Skeggs }
442