1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2016 Red Hat, Inc. 4 * Author: Michael S. Tsirkin <mst@redhat.com> 5 * 6 * Common macros and functions for ring benchmarking. 7 */ 8 #ifndef MAIN_H 9 #define MAIN_H 10 11 #include <assert.h> 12 #include <stdbool.h> 13 14 extern int param; 15 16 extern bool do_exit; 17 18 #if defined(__x86_64__) || defined(__i386__) 19 #include "x86intrin.h" 20 21 static inline void wait_cycles(unsigned long long cycles) 22 { 23 unsigned long long t; 24 25 t = __rdtsc(); 26 while (__rdtsc() - t < cycles) {} 27 } 28 29 #define VMEXIT_CYCLES 500 30 #define VMENTRY_CYCLES 500 31 32 #elif defined(__s390x__) 33 static inline void wait_cycles(unsigned long long cycles) 34 { 35 asm volatile("0: brctg %0,0b" : : "d" (cycles)); 36 } 37 38 /* tweak me */ 39 #define VMEXIT_CYCLES 200 40 #define VMENTRY_CYCLES 200 41 42 #else 43 static inline void wait_cycles(unsigned long long cycles) 44 { 45 _Exit(5); 46 } 47 #define VMEXIT_CYCLES 0 48 #define VMENTRY_CYCLES 0 49 #endif 50 51 static inline void vmexit(void) 52 { 53 if (!do_exit) 54 return; 55 56 wait_cycles(VMEXIT_CYCLES); 57 } 58 static inline void vmentry(void) 59 { 60 if (!do_exit) 61 return; 62 63 wait_cycles(VMENTRY_CYCLES); 64 } 65 66 /* implemented by ring */ 67 void alloc_ring(void); 68 /* guest side */ 69 int add_inbuf(unsigned, void *, void *); 70 void *get_buf(unsigned *, void **); 71 void disable_call(); 72 bool used_empty(); 73 bool enable_call(); 74 void kick_available(); 75 /* host side */ 76 void disable_kick(); 77 bool avail_empty(); 78 bool enable_kick(); 79 bool use_buf(unsigned *, void **); 80 void call_used(); 81 82 /* implemented by main */ 83 extern bool do_sleep; 84 void kick(void); 85 void wait_for_kick(void); 86 void call(void); 87 void wait_for_call(void); 88 89 extern unsigned ring_size; 90 91 /* Compiler barrier - similar to what Linux uses */ 92 #define barrier() asm volatile("" ::: "memory") 93 94 /* Is there a portable way to do this? */ 95 #if defined(__x86_64__) || defined(__i386__) 96 #define cpu_relax() asm ("rep; nop" ::: "memory") 97 #elif defined(__s390x__) 98 #define cpu_relax() barrier() 99 #elif defined(__aarch64__) 100 #define cpu_relax() asm ("yield" ::: "memory") 101 #else 102 #define cpu_relax() assert(0) 103 #endif 104 105 extern bool do_relax; 106 107 static inline void busy_wait(void) 108 { 109 if (do_relax) 110 cpu_relax(); 111 else 112 /* prevent compiler from removing busy loops */ 113 barrier(); 114 } 115 116 #if defined(__x86_64__) || defined(__i386__) 117 #define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc") 118 #elif defined(__aarch64__) 119 #define smp_mb() asm volatile("dmb ish" ::: "memory") 120 #else 121 /* 122 * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized 123 * with other __ATOMIC_SEQ_CST calls. 124 */ 125 #define smp_mb() __sync_synchronize() 126 #endif 127 128 /* 129 * This abuses the atomic builtins for thread fences, and 130 * adds a compiler barrier. 131 */ 132 #define smp_release() do { \ 133 barrier(); \ 134 __atomic_thread_fence(__ATOMIC_RELEASE); \ 135 } while (0) 136 137 #define smp_acquire() do { \ 138 __atomic_thread_fence(__ATOMIC_ACQUIRE); \ 139 barrier(); \ 140 } while (0) 141 142 #if defined(__i386__) || defined(__x86_64__) || defined(__s390x__) 143 #define smp_wmb() barrier() 144 #elif defined(__aarch64__) 145 #define smp_wmb() asm volatile("dmb ishst" ::: "memory") 146 #else 147 #define smp_wmb() smp_release() 148 #endif 149 150 #ifndef __always_inline 151 #define __always_inline inline __attribute__((always_inline)) 152 #endif 153 154 static __always_inline 155 void __read_once_size(const volatile void *p, void *res, int size) 156 { 157 switch (size) { 158 case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break; 159 case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break; 160 case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break; 161 case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break; 162 default: 163 barrier(); 164 __builtin_memcpy((void *)res, (const void *)p, size); 165 barrier(); 166 } 167 } 168 169 static __always_inline void __write_once_size(volatile void *p, void *res, int size) 170 { 171 switch (size) { 172 case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break; 173 case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break; 174 case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break; 175 case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break; 176 default: 177 barrier(); 178 __builtin_memcpy((void *)p, (const void *)res, size); 179 barrier(); 180 } 181 } 182 183 #ifdef __alpha__ 184 #define READ_ONCE(x) \ 185 ({ \ 186 union { typeof(x) __val; char __c[1]; } __u; \ 187 __read_once_size(&(x), __u.__c, sizeof(x)); \ 188 smp_mb(); /* Enforce dependency ordering from x */ \ 189 __u.__val; \ 190 }) 191 #else 192 #define READ_ONCE(x) \ 193 ({ \ 194 union { typeof(x) __val; char __c[1]; } __u; \ 195 __read_once_size(&(x), __u.__c, sizeof(x)); \ 196 __u.__val; \ 197 }) 198 #endif 199 200 #define WRITE_ONCE(x, val) \ 201 ({ \ 202 union { typeof(x) __val; char __c[1]; } __u = \ 203 { .__val = (typeof(x)) (val) }; \ 204 __write_once_size(&(x), __u.__c, sizeof(x)); \ 205 __u.__val; \ 206 }) 207 208 #endif 209