1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KVM guest fault handling.
4 *
5 * Copyright IBM Corp. 2025
6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
7 */
8 #include <linux/kvm_types.h>
9 #include <linux/kvm_host.h>
10
11 #include "gmap.h"
12 #include "trace.h"
13 #include "faultin.h"
14
15 bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu);
16
17 /*
18 * kvm_s390_faultin_gfn() - handle a dat fault.
19 * @vcpu: The vCPU whose gmap is to be fixed up, or NULL if operating on the VM.
20 * @kvm: The VM whose gmap is to be fixed up, or NULL if operating on a vCPU.
21 * @f: The guest fault that needs to be resolved.
22 *
23 * Return:
24 * * 0 on success
25 * * < 0 in case of error
26 * * > 0 in case of guest exceptions
27 *
28 * Context:
29 * * The mm lock must not be held before calling
30 * * kvm->srcu must be held
31 * * may sleep
32 */
kvm_s390_faultin_gfn(struct kvm_vcpu * vcpu,struct kvm * kvm,struct guest_fault * f)33 int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fault *f)
34 {
35 struct kvm_s390_mmu_cache *local_mc __free(kvm_s390_mmu_cache) = NULL;
36 struct kvm_s390_mmu_cache *mc = NULL;
37 struct kvm_memory_slot *slot;
38 unsigned long inv_seq;
39 int rc = -EAGAIN;
40 int foll;
41
42 foll = f->write_attempt ? FOLL_WRITE : 0;
43 foll |= f->attempt_pfault ? FOLL_NOWAIT : 0;
44
45 if (vcpu) {
46 kvm = vcpu->kvm;
47 mc = vcpu->arch.mc;
48 }
49
50 lockdep_assert_held(&kvm->srcu);
51
52 scoped_guard(read_lock, &kvm->mmu_lock) {
53 if (gmap_try_fixup_minor(kvm->arch.gmap, f) == 0)
54 return 0;
55 }
56
57 if (!mc) {
58 local_mc = kvm_s390_new_mmu_cache();
59 if (!local_mc)
60 return -ENOMEM;
61 mc = local_mc;
62 }
63
64 while (rc == -EAGAIN) {
65 f->valid = false;
66 inv_seq = kvm->mmu_invalidate_seq;
67 /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */
68 smp_rmb();
69
70 if (vcpu)
71 slot = kvm_vcpu_gfn_to_memslot(vcpu, f->gfn);
72 else
73 slot = gfn_to_memslot(kvm, f->gfn);
74 f->pfn = __kvm_faultin_pfn(slot, f->gfn, foll, &f->writable, &f->page);
75
76 /* Needs I/O, try to setup async pfault (only possible with FOLL_NOWAIT). */
77 if (f->pfn == KVM_PFN_ERR_NEEDS_IO) {
78 if (unlikely(!f->attempt_pfault))
79 return -EAGAIN;
80 if (unlikely(!vcpu))
81 return -EINVAL;
82 trace_kvm_s390_major_guest_pfault(vcpu);
83 if (kvm_arch_setup_async_pf(vcpu))
84 return 0;
85 vcpu->stat.pfault_sync++;
86 /* Could not setup async pfault, try again synchronously. */
87 foll &= ~FOLL_NOWAIT;
88 f->pfn = __kvm_faultin_pfn(slot, f->gfn, foll, &f->writable, &f->page);
89 }
90
91 /* Access outside memory, addressing exception. */
92 if (is_noslot_pfn(f->pfn))
93 return PGM_ADDRESSING;
94 /* Signal pending: try again. */
95 if (f->pfn == KVM_PFN_ERR_SIGPENDING)
96 return -EAGAIN;
97 /* Check if it's read-only memory; don't try to actually handle that case. */
98 if (f->pfn == KVM_PFN_ERR_RO_FAULT)
99 return -EOPNOTSUPP;
100 /* Any other error. */
101 if (is_error_pfn(f->pfn))
102 return -EFAULT;
103
104 /* Loop, release the faulted page. */
105 if (mmu_invalidate_retry_gfn_unsafe(kvm, inv_seq, f->gfn)) {
106 kvm_release_faultin_page(kvm, f->page, true, false);
107 continue;
108 }
109
110 scoped_guard(read_lock, &kvm->mmu_lock) {
111 if (!mmu_invalidate_retry_gfn(kvm, inv_seq, f->gfn)) {
112 f->valid = true;
113 rc = gmap_link(mc, kvm->arch.gmap, f, slot);
114 }
115 kvm_release_faultin_page(kvm, f->page, !!rc, f->write_attempt);
116 }
117
118 if (rc == -ENOMEM) {
119 rc = kvm_s390_mmu_cache_topup(mc);
120 if (rc)
121 return rc;
122 rc = -EAGAIN;
123 }
124 }
125
126 return rc;
127 }
128
kvm_s390_get_guest_page(struct kvm * kvm,struct guest_fault * f,gfn_t gfn,bool w)129 int kvm_s390_get_guest_page(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool w)
130 {
131 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
132 int foll = w ? FOLL_WRITE : 0;
133
134 f->write_attempt = w;
135 f->gfn = gfn;
136 f->pfn = __kvm_faultin_pfn(slot, gfn, foll, &f->writable, &f->page);
137 if (is_noslot_pfn(f->pfn))
138 return PGM_ADDRESSING;
139 if (is_sigpending_pfn(f->pfn))
140 return -EINTR;
141 if (f->pfn == KVM_PFN_ERR_NEEDS_IO)
142 return -EAGAIN;
143 if (is_error_pfn(f->pfn))
144 return -EFAULT;
145
146 f->valid = true;
147 return 0;
148 }
149