1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2023 Ventana Micro Systems Inc. 4 */ 5 6 #define pr_fmt(fmt) "riscv-pv: " fmt 7 8 #include <linux/cpuhotplug.h> 9 #include <linux/compiler.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/jump_label.h> 13 #include <linux/kconfig.h> 14 #include <linux/kernel.h> 15 #include <linux/percpu-defs.h> 16 #include <linux/printk.h> 17 #include <linux/static_call.h> 18 #include <linux/types.h> 19 #include <linux/sched/cputime.h> 20 21 #include <asm/barrier.h> 22 #include <asm/page.h> 23 #include <asm/paravirt.h> 24 #include <asm/sbi.h> 25 26 static bool steal_acc = true; 27 static int __init parse_no_stealacc(char *arg) 28 { 29 steal_acc = false; 30 return 0; 31 } 32 33 early_param("no-steal-acc", parse_no_stealacc); 34 35 static DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64); 36 37 static bool __init has_pv_steal_clock(void) 38 { 39 if (sbi_spec_version >= sbi_mk_version(2, 0) && 40 sbi_probe_extension(SBI_EXT_STA) > 0) { 41 pr_info("SBI STA extension detected\n"); 42 return true; 43 } 44 45 return false; 46 } 47 48 static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi, 49 unsigned long flags) 50 { 51 struct sbiret ret; 52 53 ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM, 54 lo, hi, flags, 0, 0, 0); 55 if (ret.error) { 56 if (lo == SBI_SHMEM_DISABLE && hi == SBI_SHMEM_DISABLE) 57 pr_warn("Failed to disable steal-time shmem"); 58 else 59 pr_warn("Failed to set steal-time shmem"); 60 return sbi_err_map_linux_errno(ret.error); 61 } 62 63 return 0; 64 } 65 66 static int pv_time_cpu_online(unsigned int cpu) 67 { 68 struct sbi_sta_struct *st = this_cpu_ptr(&steal_time); 69 phys_addr_t pa = __pa(st); 70 unsigned long lo = (unsigned long)pa; 71 unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0; 72 73 return sbi_sta_steal_time_set_shmem(lo, hi, 0); 74 } 75 76 static int pv_time_cpu_down_prepare(unsigned int cpu) 77 { 78 return sbi_sta_steal_time_set_shmem(SBI_SHMEM_DISABLE, 79 SBI_SHMEM_DISABLE, 0); 80 } 81 82 static u64 pv_time_steal_clock(int cpu) 83 { 84 struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu); 85 __le32 sequence; 86 __le64 steal; 87 88 /* 89 * Check the sequence field before and after reading the steal 90 * field. Repeat the read if it is different or odd. 91 */ 92 do { 93 sequence = READ_ONCE(st->sequence); 94 virt_rmb(); 95 steal = READ_ONCE(st->steal); 96 virt_rmb(); 97 } while ((le32_to_cpu(sequence) & 1) || 98 sequence != READ_ONCE(st->sequence)); 99 100 return le64_to_cpu(steal); 101 } 102 103 int __init pv_time_init(void) 104 { 105 int ret; 106 107 if (!has_pv_steal_clock()) 108 return 0; 109 110 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 111 "riscv/pv_time:online", 112 pv_time_cpu_online, 113 pv_time_cpu_down_prepare); 114 if (ret < 0) 115 return ret; 116 117 static_call_update(pv_steal_clock, pv_time_steal_clock); 118 119 static_key_slow_inc(¶virt_steal_enabled); 120 if (steal_acc) 121 static_key_slow_inc(¶virt_steal_rq_enabled); 122 123 pr_info("Computing paravirt steal-time\n"); 124 125 return 0; 126 } 127