1 /* 2 * Copyright (C) 2016 Red Hat, Inc. 3 * Author: Michael S. Tsirkin <mst@redhat.com> 4 * This work is licensed under the terms of the GNU GPL, version 2. 5 * 6 * Partial implementation of virtio 0.9. event index is used for signalling, 7 * unconditionally. Design roughly follows linux kernel implementation in order 8 * to be able to judge its performance. 9 */ 10 #define _GNU_SOURCE 11 #include "main.h" 12 #include <stdlib.h> 13 #include <stdio.h> 14 #include <assert.h> 15 #include <string.h> 16 #include <linux/virtio_ring.h> 17 18 struct data { 19 void *data; 20 } *data; 21 22 struct vring ring; 23 24 /* enabling the below activates experimental ring polling code 25 * (which skips index reads on consumer in favor of looking at 26 * high bits of ring id ^ 0x8000). 27 */ 28 /* #ifdef RING_POLL */ 29 30 /* how much padding is needed to avoid false cache sharing */ 31 #define HOST_GUEST_PADDING 0x80 32 33 struct guest { 34 unsigned short avail_idx; 35 unsigned short last_used_idx; 36 unsigned short num_free; 37 unsigned short kicked_avail_idx; 38 unsigned short free_head; 39 unsigned char reserved[HOST_GUEST_PADDING - 10]; 40 } guest; 41 42 struct host { 43 /* we do not need to track last avail index 44 * unless we have more than one in flight. 45 */ 46 unsigned short used_idx; 47 unsigned short called_used_idx; 48 unsigned char reserved[HOST_GUEST_PADDING - 4]; 49 } host; 50 51 /* implemented by ring */ 52 void alloc_ring(void) 53 { 54 int ret; 55 int i; 56 void *p; 57 58 ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000)); 59 if (ret) { 60 perror("Unable to allocate ring buffer.\n"); 61 exit(3); 62 } 63 memset(p, 0, vring_size(ring_size, 0x1000)); 64 vring_init(&ring, ring_size, p, 0x1000); 65 66 guest.avail_idx = 0; 67 guest.kicked_avail_idx = -1; 68 guest.last_used_idx = 0; 69 /* Put everything in free lists. */ 70 guest.free_head = 0; 71 for (i = 0; i < ring_size - 1; i++) 72 ring.desc[i].next = i + 1; 73 host.used_idx = 0; 74 host.called_used_idx = -1; 75 guest.num_free = ring_size; 76 data = malloc(ring_size * sizeof *data); 77 if (!data) { 78 perror("Unable to allocate data buffer.\n"); 79 exit(3); 80 } 81 memset(data, 0, ring_size * sizeof *data); 82 } 83 84 /* guest side */ 85 int add_inbuf(unsigned len, void *buf, void *datap) 86 { 87 unsigned head, avail; 88 struct vring_desc *desc; 89 90 if (!guest.num_free) 91 return -1; 92 93 head = guest.free_head; 94 guest.num_free--; 95 96 desc = ring.desc; 97 desc[head].flags = VRING_DESC_F_NEXT; 98 desc[head].addr = (unsigned long)(void *)buf; 99 desc[head].len = len; 100 /* We do it like this to simulate the way 101 * we'd have to flip it if we had multiple 102 * descriptors. 103 */ 104 desc[head].flags &= ~VRING_DESC_F_NEXT; 105 guest.free_head = desc[head].next; 106 107 data[head].data = datap; 108 109 #ifdef RING_POLL 110 /* Barrier A (for pairing) */ 111 smp_release(); 112 avail = guest.avail_idx++; 113 ring.avail->ring[avail & (ring_size - 1)] = 114 (head | (avail & ~(ring_size - 1))) ^ 0x8000; 115 #else 116 avail = (ring_size - 1) & (guest.avail_idx++); 117 ring.avail->ring[avail] = head; 118 /* Barrier A (for pairing) */ 119 smp_release(); 120 #endif 121 ring.avail->idx = guest.avail_idx; 122 return 0; 123 } 124 125 void *get_buf(unsigned *lenp, void **bufp) 126 { 127 unsigned head; 128 unsigned index; 129 void *datap; 130 131 #ifdef RING_POLL 132 head = (ring_size - 1) & guest.last_used_idx; 133 index = ring.used->ring[head].id; 134 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1)) 135 return NULL; 136 /* Barrier B (for pairing) */ 137 smp_acquire(); 138 index &= ring_size - 1; 139 #else 140 if (ring.used->idx == guest.last_used_idx) 141 return NULL; 142 /* Barrier B (for pairing) */ 143 smp_acquire(); 144 head = (ring_size - 1) & guest.last_used_idx; 145 index = ring.used->ring[head].id; 146 #endif 147 *lenp = ring.used->ring[head].len; 148 datap = data[index].data; 149 *bufp = (void*)(unsigned long)ring.desc[index].addr; 150 data[index].data = NULL; 151 ring.desc[index].next = guest.free_head; 152 guest.free_head = index; 153 guest.num_free++; 154 guest.last_used_idx++; 155 return datap; 156 } 157 158 void poll_used(void) 159 { 160 #ifdef RING_POLL 161 unsigned head = (ring_size - 1) & guest.last_used_idx; 162 163 for (;;) { 164 unsigned index = ring.used->ring[head].id; 165 166 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1)) 167 busy_wait(); 168 else 169 break; 170 } 171 #else 172 unsigned head = guest.last_used_idx; 173 174 while (ring.used->idx == head) 175 busy_wait(); 176 #endif 177 } 178 179 void disable_call() 180 { 181 /* Doing nothing to disable calls might cause 182 * extra interrupts, but reduces the number of cache misses. 183 */ 184 } 185 186 bool enable_call() 187 { 188 unsigned short last_used_idx; 189 190 vring_used_event(&ring) = (last_used_idx = guest.last_used_idx); 191 /* Flush call index write */ 192 /* Barrier D (for pairing) */ 193 smp_mb(); 194 #ifdef RING_POLL 195 { 196 unsigned short head = last_used_idx & (ring_size - 1); 197 unsigned index = ring.used->ring[head].id; 198 199 return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1); 200 } 201 #else 202 return ring.used->idx == last_used_idx; 203 #endif 204 } 205 206 void kick_available(void) 207 { 208 /* Flush in previous flags write */ 209 /* Barrier C (for pairing) */ 210 smp_mb(); 211 if (!vring_need_event(vring_avail_event(&ring), 212 guest.avail_idx, 213 guest.kicked_avail_idx)) 214 return; 215 216 guest.kicked_avail_idx = guest.avail_idx; 217 kick(); 218 } 219 220 /* host side */ 221 void disable_kick() 222 { 223 /* Doing nothing to disable kicks might cause 224 * extra interrupts, but reduces the number of cache misses. 225 */ 226 } 227 228 bool enable_kick() 229 { 230 unsigned head = host.used_idx; 231 232 vring_avail_event(&ring) = head; 233 /* Barrier C (for pairing) */ 234 smp_mb(); 235 #ifdef RING_POLL 236 { 237 unsigned index = ring.avail->ring[head & (ring_size - 1)]; 238 239 return (index ^ head ^ 0x8000) & ~(ring_size - 1); 240 } 241 #else 242 return head == ring.avail->idx; 243 #endif 244 } 245 246 void poll_avail(void) 247 { 248 unsigned head = host.used_idx; 249 #ifdef RING_POLL 250 for (;;) { 251 unsigned index = ring.avail->ring[head & (ring_size - 1)]; 252 if ((index ^ head ^ 0x8000) & ~(ring_size - 1)) 253 busy_wait(); 254 else 255 break; 256 } 257 #else 258 while (ring.avail->idx == head) 259 busy_wait(); 260 #endif 261 } 262 263 bool use_buf(unsigned *lenp, void **bufp) 264 { 265 unsigned used_idx = host.used_idx; 266 struct vring_desc *desc; 267 unsigned head; 268 269 #ifdef RING_POLL 270 head = ring.avail->ring[used_idx & (ring_size - 1)]; 271 if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1)) 272 return false; 273 /* Barrier A (for pairing) */ 274 smp_acquire(); 275 276 used_idx &= ring_size - 1; 277 desc = &ring.desc[head & (ring_size - 1)]; 278 #else 279 if (used_idx == ring.avail->idx) 280 return false; 281 282 /* Barrier A (for pairing) */ 283 smp_acquire(); 284 285 used_idx &= ring_size - 1; 286 head = ring.avail->ring[used_idx]; 287 desc = &ring.desc[head]; 288 #endif 289 290 *lenp = desc->len; 291 *bufp = (void *)(unsigned long)desc->addr; 292 293 /* now update used ring */ 294 ring.used->ring[used_idx].id = head; 295 ring.used->ring[used_idx].len = desc->len - 1; 296 /* Barrier B (for pairing) */ 297 smp_release(); 298 host.used_idx++; 299 ring.used->idx = host.used_idx; 300 301 return true; 302 } 303 304 void call_used(void) 305 { 306 /* Flush in previous flags write */ 307 /* Barrier D (for pairing) */ 308 smp_mb(); 309 if (!vring_need_event(vring_used_event(&ring), 310 host.used_idx, 311 host.called_used_idx)) 312 return; 313 314 host.called_used_idx = host.used_idx; 315 call(); 316 } 317