1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * VMID allocator. 4 * 5 * Based on Arm64 ASID allocator algorithm. 6 * Please refer arch/arm64/mm/context.c for detailed 7 * comments on algorithm. 8 * 9 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved. 10 * Copyright (C) 2012 ARM Ltd. 11 */ 12 13 #include <linux/bitfield.h> 14 #include <linux/bitops.h> 15 16 #include <asm/kvm_asm.h> 17 #include <asm/kvm_mmu.h> 18 19 unsigned int __ro_after_init kvm_arm_vmid_bits; 20 static DEFINE_RAW_SPINLOCK(cpu_vmid_lock); 21 22 static atomic64_t vmid_generation; 23 static unsigned long *vmid_map; 24 25 static DEFINE_PER_CPU(atomic64_t, active_vmids); 26 static DEFINE_PER_CPU(u64, reserved_vmids); 27 28 #define VMID_MASK (~GENMASK(kvm_arm_vmid_bits - 1, 0)) 29 #define VMID_FIRST_VERSION (1UL << kvm_arm_vmid_bits) 30 31 #define NUM_USER_VMIDS VMID_FIRST_VERSION 32 #define vmid2idx(vmid) ((vmid) & ~VMID_MASK) 33 #define idx2vmid(idx) vmid2idx(idx) 34 35 /* 36 * As vmid #0 is always reserved, we will never allocate one 37 * as below and can be treated as invalid. This is used to 38 * set the active_vmids on vCPU schedule out. 39 */ 40 #define VMID_ACTIVE_INVALID VMID_FIRST_VERSION 41 42 #define vmid_gen_match(vmid) \ 43 (!(((vmid) ^ atomic64_read(&vmid_generation)) >> kvm_arm_vmid_bits)) 44 45 static void flush_context(void) 46 { 47 int cpu; 48 u64 vmid; 49 50 bitmap_zero(vmid_map, NUM_USER_VMIDS); 51 52 for_each_possible_cpu(cpu) { 53 vmid = atomic64_xchg_relaxed(&per_cpu(active_vmids, cpu), 0); 54 55 /* Preserve reserved VMID */ 56 if (vmid == 0) 57 vmid = per_cpu(reserved_vmids, cpu); 58 __set_bit(vmid2idx(vmid), vmid_map); 59 per_cpu(reserved_vmids, cpu) = vmid; 60 } 61 62 /* 63 * Unlike ASID allocator, we expect less frequent rollover in 64 * case of VMIDs. Hence, instead of marking the CPU as 65 * flush_pending and issuing a local context invalidation on 66 * the next context-switch, we broadcast TLB flush + I-cache 67 * invalidation over the inner shareable domain on rollover. 68 */ 69 kvm_call_hyp(__kvm_flush_vm_context); 70 } 71 72 static bool check_update_reserved_vmid(u64 vmid, u64 newvmid) 73 { 74 int cpu; 75 bool hit = false; 76 77 /* 78 * Iterate over the set of reserved VMIDs looking for a match 79 * and update to use newvmid (i.e. the same VMID in the current 80 * generation). 81 */ 82 for_each_possible_cpu(cpu) { 83 if (per_cpu(reserved_vmids, cpu) == vmid) { 84 hit = true; 85 per_cpu(reserved_vmids, cpu) = newvmid; 86 } 87 } 88 89 return hit; 90 } 91 92 static u64 new_vmid(struct kvm_vmid *kvm_vmid) 93 { 94 static u32 cur_idx = 1; 95 u64 vmid = atomic64_read(&kvm_vmid->id); 96 u64 generation = atomic64_read(&vmid_generation); 97 98 if (vmid != 0) { 99 u64 newvmid = generation | (vmid & ~VMID_MASK); 100 101 if (check_update_reserved_vmid(vmid, newvmid)) { 102 atomic64_set(&kvm_vmid->id, newvmid); 103 return newvmid; 104 } 105 106 if (!__test_and_set_bit(vmid2idx(vmid), vmid_map)) { 107 atomic64_set(&kvm_vmid->id, newvmid); 108 return newvmid; 109 } 110 } 111 112 vmid = find_next_zero_bit(vmid_map, NUM_USER_VMIDS, cur_idx); 113 if (vmid != NUM_USER_VMIDS) 114 goto set_vmid; 115 116 /* We're out of VMIDs, so increment the global generation count */ 117 generation = atomic64_add_return_relaxed(VMID_FIRST_VERSION, 118 &vmid_generation); 119 flush_context(); 120 121 /* We have more VMIDs than CPUs, so this will always succeed */ 122 vmid = find_next_zero_bit(vmid_map, NUM_USER_VMIDS, 1); 123 124 set_vmid: 125 __set_bit(vmid, vmid_map); 126 cur_idx = vmid; 127 vmid = idx2vmid(vmid) | generation; 128 atomic64_set(&kvm_vmid->id, vmid); 129 return vmid; 130 } 131 132 /* Called from vCPU sched out with preemption disabled */ 133 void kvm_arm_vmid_clear_active(void) 134 { 135 atomic64_set(this_cpu_ptr(&active_vmids), VMID_ACTIVE_INVALID); 136 } 137 138 bool kvm_arm_vmid_update(struct kvm_vmid *kvm_vmid) 139 { 140 unsigned long flags; 141 u64 vmid, old_active_vmid; 142 bool updated = false; 143 144 vmid = atomic64_read(&kvm_vmid->id); 145 146 /* 147 * Please refer comments in check_and_switch_context() in 148 * arch/arm64/mm/context.c. 149 * 150 * Unlike ASID allocator, we set the active_vmids to 151 * VMID_ACTIVE_INVALID on vCPU schedule out to avoid 152 * reserving the VMID space needlessly on rollover. 153 * Hence explicitly check here for a "!= 0" to 154 * handle the sync with a concurrent rollover. 155 */ 156 old_active_vmid = atomic64_read(this_cpu_ptr(&active_vmids)); 157 if (old_active_vmid != 0 && vmid_gen_match(vmid) && 158 0 != atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_vmids), 159 old_active_vmid, vmid)) 160 return false; 161 162 raw_spin_lock_irqsave(&cpu_vmid_lock, flags); 163 164 /* Check that our VMID belongs to the current generation. */ 165 vmid = atomic64_read(&kvm_vmid->id); 166 if (!vmid_gen_match(vmid)) { 167 vmid = new_vmid(kvm_vmid); 168 updated = true; 169 } 170 171 atomic64_set(this_cpu_ptr(&active_vmids), vmid); 172 raw_spin_unlock_irqrestore(&cpu_vmid_lock, flags); 173 174 return updated; 175 } 176 177 /* 178 * Initialize the VMID allocator 179 */ 180 int __init kvm_arm_vmid_alloc_init(void) 181 { 182 kvm_arm_vmid_bits = kvm_get_vmid_bits(); 183 184 /* 185 * Expect allocation after rollover to fail if we don't have 186 * at least one more VMID than CPUs. VMID #0 is always reserved. 187 */ 188 WARN_ON(NUM_USER_VMIDS - 1 <= num_possible_cpus()); 189 atomic64_set(&vmid_generation, VMID_FIRST_VERSION); 190 vmid_map = bitmap_zalloc(NUM_USER_VMIDS, GFP_KERNEL); 191 if (!vmid_map) 192 return -ENOMEM; 193 194 return 0; 195 } 196 197 void __init kvm_arm_vmid_alloc_free(void) 198 { 199 bitmap_free(vmid_map); 200 } 201