1*d8a3f0a0SChristian Koenig // SPDX-License-Identifier: GPL-2.0 OR MIT
2*d8a3f0a0SChristian Koenig /*
3*d8a3f0a0SChristian Koenig * Copyright 2023 Advanced Micro Devices, Inc.
4*d8a3f0a0SChristian Koenig *
5*d8a3f0a0SChristian Koenig * Permission is hereby granted, free of charge, to any person obtaining a
6*d8a3f0a0SChristian Koenig * copy of this software and associated documentation files (the "Software"),
7*d8a3f0a0SChristian Koenig * to deal in the Software without restriction, including without limitation
8*d8a3f0a0SChristian Koenig * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9*d8a3f0a0SChristian Koenig * and/or sell copies of the Software, and to permit persons to whom the
10*d8a3f0a0SChristian Koenig * Software is furnished to do so, subject to the following conditions:
11*d8a3f0a0SChristian Koenig *
12*d8a3f0a0SChristian Koenig * The above copyright notice and this permission notice shall be included in
13*d8a3f0a0SChristian Koenig * all copies or substantial portions of the Software.
14*d8a3f0a0SChristian Koenig *
15*d8a3f0a0SChristian Koenig * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16*d8a3f0a0SChristian Koenig * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17*d8a3f0a0SChristian Koenig * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18*d8a3f0a0SChristian Koenig * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19*d8a3f0a0SChristian Koenig * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20*d8a3f0a0SChristian Koenig * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21*d8a3f0a0SChristian Koenig * OTHER DEALINGS IN THE SOFTWARE.
22*d8a3f0a0SChristian Koenig */
23*d8a3f0a0SChristian Koenig
24*d8a3f0a0SChristian Koenig #include <linux/dma-fence.h>
25*d8a3f0a0SChristian Koenig #include <linux/workqueue.h>
26*d8a3f0a0SChristian Koenig
27*d8a3f0a0SChristian Koenig #include "amdgpu.h"
28*d8a3f0a0SChristian Koenig #include "amdgpu_vm.h"
29*d8a3f0a0SChristian Koenig #include "amdgpu_gmc.h"
30*d8a3f0a0SChristian Koenig
31*d8a3f0a0SChristian Koenig struct amdgpu_tlb_fence {
32*d8a3f0a0SChristian Koenig struct dma_fence base;
33*d8a3f0a0SChristian Koenig struct amdgpu_device *adev;
34*d8a3f0a0SChristian Koenig struct dma_fence *dependency;
35*d8a3f0a0SChristian Koenig struct work_struct work;
36*d8a3f0a0SChristian Koenig spinlock_t lock;
37*d8a3f0a0SChristian Koenig uint16_t pasid;
38*d8a3f0a0SChristian Koenig
39*d8a3f0a0SChristian Koenig };
40*d8a3f0a0SChristian Koenig
amdgpu_tlb_fence_get_driver_name(struct dma_fence * fence)41*d8a3f0a0SChristian Koenig static const char *amdgpu_tlb_fence_get_driver_name(struct dma_fence *fence)
42*d8a3f0a0SChristian Koenig {
43*d8a3f0a0SChristian Koenig return "amdgpu tlb fence";
44*d8a3f0a0SChristian Koenig }
45*d8a3f0a0SChristian Koenig
amdgpu_tlb_fence_get_timeline_name(struct dma_fence * f)46*d8a3f0a0SChristian Koenig static const char *amdgpu_tlb_fence_get_timeline_name(struct dma_fence *f)
47*d8a3f0a0SChristian Koenig {
48*d8a3f0a0SChristian Koenig return "amdgpu tlb timeline";
49*d8a3f0a0SChristian Koenig }
50*d8a3f0a0SChristian Koenig
amdgpu_tlb_fence_work(struct work_struct * work)51*d8a3f0a0SChristian Koenig static void amdgpu_tlb_fence_work(struct work_struct *work)
52*d8a3f0a0SChristian Koenig {
53*d8a3f0a0SChristian Koenig struct amdgpu_tlb_fence *f = container_of(work, typeof(*f), work);
54*d8a3f0a0SChristian Koenig int r;
55*d8a3f0a0SChristian Koenig
56*d8a3f0a0SChristian Koenig if (f->dependency) {
57*d8a3f0a0SChristian Koenig dma_fence_wait(f->dependency, false);
58*d8a3f0a0SChristian Koenig dma_fence_put(f->dependency);
59*d8a3f0a0SChristian Koenig f->dependency = NULL;
60*d8a3f0a0SChristian Koenig }
61*d8a3f0a0SChristian Koenig
62*d8a3f0a0SChristian Koenig r = amdgpu_gmc_flush_gpu_tlb_pasid(f->adev, f->pasid, 2, true, 0);
63*d8a3f0a0SChristian Koenig if (r) {
64*d8a3f0a0SChristian Koenig dev_err(f->adev->dev, "TLB flush failed for PASID %d.\n",
65*d8a3f0a0SChristian Koenig f->pasid);
66*d8a3f0a0SChristian Koenig dma_fence_set_error(&f->base, r);
67*d8a3f0a0SChristian Koenig }
68*d8a3f0a0SChristian Koenig
69*d8a3f0a0SChristian Koenig dma_fence_signal(&f->base);
70*d8a3f0a0SChristian Koenig dma_fence_put(&f->base);
71*d8a3f0a0SChristian Koenig }
72*d8a3f0a0SChristian Koenig
73*d8a3f0a0SChristian Koenig static const struct dma_fence_ops amdgpu_tlb_fence_ops = {
74*d8a3f0a0SChristian Koenig .use_64bit_seqno = true,
75*d8a3f0a0SChristian Koenig .get_driver_name = amdgpu_tlb_fence_get_driver_name,
76*d8a3f0a0SChristian Koenig .get_timeline_name = amdgpu_tlb_fence_get_timeline_name
77*d8a3f0a0SChristian Koenig };
78*d8a3f0a0SChristian Koenig
amdgpu_vm_tlb_fence_create(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct dma_fence ** fence)79*d8a3f0a0SChristian Koenig void amdgpu_vm_tlb_fence_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
80*d8a3f0a0SChristian Koenig struct dma_fence **fence)
81*d8a3f0a0SChristian Koenig {
82*d8a3f0a0SChristian Koenig struct amdgpu_tlb_fence *f;
83*d8a3f0a0SChristian Koenig
84*d8a3f0a0SChristian Koenig f = kmalloc(sizeof(*f), GFP_KERNEL);
85*d8a3f0a0SChristian Koenig if (!f) {
86*d8a3f0a0SChristian Koenig /*
87*d8a3f0a0SChristian Koenig * We can't fail since the PDEs and PTEs are already updated, so
88*d8a3f0a0SChristian Koenig * just block for the dependency and execute the TLB flush
89*d8a3f0a0SChristian Koenig */
90*d8a3f0a0SChristian Koenig if (*fence)
91*d8a3f0a0SChristian Koenig dma_fence_wait(*fence, false);
92*d8a3f0a0SChristian Koenig
93*d8a3f0a0SChristian Koenig amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, 2, true, 0);
94*d8a3f0a0SChristian Koenig *fence = dma_fence_get_stub();
95*d8a3f0a0SChristian Koenig return;
96*d8a3f0a0SChristian Koenig }
97*d8a3f0a0SChristian Koenig
98*d8a3f0a0SChristian Koenig f->adev = adev;
99*d8a3f0a0SChristian Koenig f->dependency = *fence;
100*d8a3f0a0SChristian Koenig f->pasid = vm->pasid;
101*d8a3f0a0SChristian Koenig INIT_WORK(&f->work, amdgpu_tlb_fence_work);
102*d8a3f0a0SChristian Koenig spin_lock_init(&f->lock);
103*d8a3f0a0SChristian Koenig
104*d8a3f0a0SChristian Koenig dma_fence_init(&f->base, &amdgpu_tlb_fence_ops, &f->lock,
105*d8a3f0a0SChristian Koenig vm->tlb_fence_context, atomic64_read(&vm->tlb_seq));
106*d8a3f0a0SChristian Koenig
107*d8a3f0a0SChristian Koenig /* TODO: We probably need a separate wq here */
108*d8a3f0a0SChristian Koenig dma_fence_get(&f->base);
109*d8a3f0a0SChristian Koenig schedule_work(&f->work);
110*d8a3f0a0SChristian Koenig
111*d8a3f0a0SChristian Koenig *fence = &f->base;
112*d8a3f0a0SChristian Koenig }
113