1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2023 Ventana Micro Systems Inc. 4 */ 5 6 #define pr_fmt(fmt) "riscv-pv: " fmt 7 8 #include <linux/cpuhotplug.h> 9 #include <linux/compiler.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/jump_label.h> 13 #include <linux/kconfig.h> 14 #include <linux/kernel.h> 15 #include <linux/percpu-defs.h> 16 #include <linux/printk.h> 17 #include <linux/static_call.h> 18 #include <linux/types.h> 19 20 #include <asm/barrier.h> 21 #include <asm/page.h> 22 #include <asm/paravirt.h> 23 #include <asm/sbi.h> 24 25 struct static_key paravirt_steal_enabled; 26 struct static_key paravirt_steal_rq_enabled; 27 28 static u64 native_steal_clock(int cpu) 29 { 30 return 0; 31 } 32 33 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 34 35 static bool steal_acc = true; 36 static int __init parse_no_stealacc(char *arg) 37 { 38 steal_acc = false; 39 return 0; 40 } 41 42 early_param("no-steal-acc", parse_no_stealacc); 43 44 static DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64); 45 46 static bool __init has_pv_steal_clock(void) 47 { 48 if (sbi_spec_version >= sbi_mk_version(2, 0) && 49 sbi_probe_extension(SBI_EXT_STA) > 0) { 50 pr_info("SBI STA extension detected\n"); 51 return true; 52 } 53 54 return false; 55 } 56 57 static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi, 58 unsigned long flags) 59 { 60 struct sbiret ret; 61 62 ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM, 63 lo, hi, flags, 0, 0, 0); 64 if (ret.error) { 65 if (lo == SBI_STA_SHMEM_DISABLE && hi == SBI_STA_SHMEM_DISABLE) 66 pr_warn("Failed to disable steal-time shmem"); 67 else 68 pr_warn("Failed to set steal-time shmem"); 69 return sbi_err_map_linux_errno(ret.error); 70 } 71 72 return 0; 73 } 74 75 static int pv_time_cpu_online(unsigned int cpu) 76 { 77 struct sbi_sta_struct *st = this_cpu_ptr(&steal_time); 78 phys_addr_t pa = __pa(st); 79 unsigned long lo = (unsigned long)pa; 80 unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0; 81 82 return sbi_sta_steal_time_set_shmem(lo, hi, 0); 83 } 84 85 static int pv_time_cpu_down_prepare(unsigned int cpu) 86 { 87 return sbi_sta_steal_time_set_shmem(SBI_STA_SHMEM_DISABLE, 88 SBI_STA_SHMEM_DISABLE, 0); 89 } 90 91 static u64 pv_time_steal_clock(int cpu) 92 { 93 struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu); 94 __le32 sequence; 95 __le64 steal; 96 97 /* 98 * Check the sequence field before and after reading the steal 99 * field. Repeat the read if it is different or odd. 100 */ 101 do { 102 sequence = READ_ONCE(st->sequence); 103 virt_rmb(); 104 steal = READ_ONCE(st->steal); 105 virt_rmb(); 106 } while ((le32_to_cpu(sequence) & 1) || 107 sequence != READ_ONCE(st->sequence)); 108 109 return le64_to_cpu(steal); 110 } 111 112 int __init pv_time_init(void) 113 { 114 int ret; 115 116 if (!has_pv_steal_clock()) 117 return 0; 118 119 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 120 "riscv/pv_time:online", 121 pv_time_cpu_online, 122 pv_time_cpu_down_prepare); 123 if (ret < 0) 124 return ret; 125 126 static_call_update(pv_steal_clock, pv_time_steal_clock); 127 128 static_key_slow_inc(¶virt_steal_enabled); 129 if (steal_acc) 130 static_key_slow_inc(¶virt_steal_rq_enabled); 131 132 pr_info("Computing paravirt steal-time\n"); 133 134 return 0; 135 } 136