1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2020-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/poll.h> 25 #include <linux/wait.h> 26 #include <linux/anon_inodes.h> 27 #include <uapi/linux/kfd_ioctl.h> 28 #include "amdgpu.h" 29 #include "amdgpu_vm.h" 30 #include "kfd_priv.h" 31 #include "kfd_smi_events.h" 32 33 struct kfd_smi_client { 34 struct list_head list; 35 struct kfifo fifo; 36 wait_queue_head_t wait_queue; 37 /* events enabled */ 38 uint64_t events; 39 struct kfd_node *dev; 40 spinlock_t lock; 41 struct rcu_head rcu; 42 pid_t pid; 43 bool suser; 44 }; 45 46 #define MAX_KFIFO_SIZE 1024 47 48 static __poll_t kfd_smi_ev_poll(struct file *, struct poll_table_struct *); 49 static ssize_t kfd_smi_ev_read(struct file *, char __user *, size_t, loff_t *); 50 static ssize_t kfd_smi_ev_write(struct file *, const char __user *, size_t, 51 loff_t *); 52 static int kfd_smi_ev_release(struct inode *, struct file *); 53 54 static const char kfd_smi_name[] = "kfd_smi_ev"; 55 56 static const struct file_operations kfd_smi_ev_fops = { 57 .owner = THIS_MODULE, 58 .poll = kfd_smi_ev_poll, 59 .read = kfd_smi_ev_read, 60 .write = kfd_smi_ev_write, 61 .release = kfd_smi_ev_release 62 }; 63 64 static __poll_t kfd_smi_ev_poll(struct file *filep, 65 struct poll_table_struct *wait) 66 { 67 struct kfd_smi_client *client = filep->private_data; 68 __poll_t mask = 0; 69 70 poll_wait(filep, &client->wait_queue, wait); 71 72 spin_lock(&client->lock); 73 if (!kfifo_is_empty(&client->fifo)) 74 mask = EPOLLIN | EPOLLRDNORM; 75 spin_unlock(&client->lock); 76 77 return mask; 78 } 79 80 static ssize_t kfd_smi_ev_read(struct file *filep, char __user *user, 81 size_t size, loff_t *offset) 82 { 83 int ret; 84 size_t to_copy; 85 struct kfd_smi_client *client = filep->private_data; 86 unsigned char *buf; 87 88 size = min_t(size_t, size, MAX_KFIFO_SIZE); 89 buf = kmalloc(size, GFP_KERNEL); 90 if (!buf) 91 return -ENOMEM; 92 93 /* kfifo_to_user can sleep so we can't use spinlock protection around 94 * it. Instead, we kfifo out as spinlocked then copy them to the user. 95 */ 96 spin_lock(&client->lock); 97 to_copy = kfifo_len(&client->fifo); 98 if (!to_copy) { 99 spin_unlock(&client->lock); 100 ret = -EAGAIN; 101 goto ret_err; 102 } 103 to_copy = min(size, to_copy); 104 ret = kfifo_out(&client->fifo, buf, to_copy); 105 spin_unlock(&client->lock); 106 if (ret <= 0) { 107 ret = -EAGAIN; 108 goto ret_err; 109 } 110 111 ret = copy_to_user(user, buf, to_copy); 112 if (ret) { 113 ret = -EFAULT; 114 goto ret_err; 115 } 116 117 kfree(buf); 118 return to_copy; 119 120 ret_err: 121 kfree(buf); 122 return ret; 123 } 124 125 static ssize_t kfd_smi_ev_write(struct file *filep, const char __user *user, 126 size_t size, loff_t *offset) 127 { 128 struct kfd_smi_client *client = filep->private_data; 129 uint64_t events; 130 131 if (!access_ok(user, size) || size < sizeof(events)) 132 return -EFAULT; 133 if (copy_from_user(&events, user, sizeof(events))) 134 return -EFAULT; 135 136 WRITE_ONCE(client->events, events); 137 138 return sizeof(events); 139 } 140 141 static void kfd_smi_ev_client_free(struct rcu_head *p) 142 { 143 struct kfd_smi_client *ev = container_of(p, struct kfd_smi_client, rcu); 144 145 kfifo_free(&ev->fifo); 146 kfree(ev); 147 } 148 149 static int kfd_smi_ev_release(struct inode *inode, struct file *filep) 150 { 151 struct kfd_smi_client *client = filep->private_data; 152 struct kfd_node *dev = client->dev; 153 154 spin_lock(&dev->smi_lock); 155 list_del_rcu(&client->list); 156 spin_unlock(&dev->smi_lock); 157 158 call_rcu(&client->rcu, kfd_smi_ev_client_free); 159 return 0; 160 } 161 162 static bool kfd_smi_ev_enabled(pid_t pid, struct kfd_smi_client *client, 163 unsigned int event) 164 { 165 uint64_t all = KFD_SMI_EVENT_MASK_FROM_INDEX(KFD_SMI_EVENT_ALL_PROCESS); 166 uint64_t events = READ_ONCE(client->events); 167 168 if (pid && client->pid != pid && !(client->suser && (events & all))) 169 return false; 170 171 return events & KFD_SMI_EVENT_MASK_FROM_INDEX(event); 172 } 173 174 static void add_event_to_kfifo(pid_t pid, struct kfd_node *dev, 175 unsigned int smi_event, char *event_msg, int len) 176 { 177 struct kfd_smi_client *client; 178 179 rcu_read_lock(); 180 181 list_for_each_entry_rcu(client, &dev->smi_clients, list) { 182 if (!kfd_smi_ev_enabled(pid, client, smi_event)) 183 continue; 184 spin_lock(&client->lock); 185 if (kfifo_avail(&client->fifo) >= len) { 186 kfifo_in(&client->fifo, event_msg, len); 187 wake_up_all(&client->wait_queue); 188 } else { 189 pr_debug("smi_event(EventID: %u): no space left\n", 190 smi_event); 191 } 192 spin_unlock(&client->lock); 193 } 194 195 rcu_read_unlock(); 196 } 197 198 __printf(4, 5) 199 static void kfd_smi_event_add(pid_t pid, struct kfd_node *dev, 200 unsigned int event, char *fmt, ...) 201 { 202 char fifo_in[KFD_SMI_EVENT_MSG_SIZE]; 203 int len; 204 va_list args; 205 206 if (list_empty(&dev->smi_clients)) 207 return; 208 209 len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event); 210 211 va_start(args, fmt); 212 len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args); 213 va_end(args); 214 215 add_event_to_kfifo(pid, dev, event, fifo_in, len); 216 } 217 218 void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset) 219 { 220 unsigned int event; 221 222 if (post_reset) { 223 event = KFD_SMI_EVENT_GPU_POST_RESET; 224 } else { 225 event = KFD_SMI_EVENT_GPU_PRE_RESET; 226 ++(dev->reset_seq_num); 227 } 228 kfd_smi_event_add(0, dev, event, "%x\n", dev->reset_seq_num); 229 } 230 231 void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev, 232 uint64_t throttle_bitmask) 233 { 234 kfd_smi_event_add(0, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, "%llx:%llx\n", 235 throttle_bitmask, 236 amdgpu_dpm_get_thermal_throttling_counter(dev->adev)); 237 } 238 239 void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid) 240 { 241 struct amdgpu_task_info task_info; 242 243 memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 244 amdgpu_vm_get_task_info(dev->adev, pasid, &task_info); 245 /* Report VM faults from user applications, not retry from kernel */ 246 if (!task_info.pid) 247 return; 248 249 kfd_smi_event_add(0, dev, KFD_SMI_EVENT_VMFAULT, "%x:%s\n", 250 task_info.pid, task_info.task_name); 251 } 252 253 void kfd_smi_event_page_fault_start(struct kfd_node *node, pid_t pid, 254 unsigned long address, bool write_fault, 255 ktime_t ts) 256 { 257 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_PAGE_FAULT_START, 258 "%lld -%d @%lx(%x) %c\n", ktime_to_ns(ts), pid, 259 address, node->id, write_fault ? 'W' : 'R'); 260 } 261 262 void kfd_smi_event_page_fault_end(struct kfd_node *node, pid_t pid, 263 unsigned long address, bool migration) 264 { 265 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_PAGE_FAULT_END, 266 "%lld -%d @%lx(%x) %c\n", ktime_get_boottime_ns(), 267 pid, address, node->id, migration ? 'M' : 'U'); 268 } 269 270 void kfd_smi_event_migration_start(struct kfd_node *node, pid_t pid, 271 unsigned long start, unsigned long end, 272 uint32_t from, uint32_t to, 273 uint32_t prefetch_loc, uint32_t preferred_loc, 274 uint32_t trigger) 275 { 276 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_MIGRATE_START, 277 "%lld -%d @%lx(%lx) %x->%x %x:%x %d\n", 278 ktime_get_boottime_ns(), pid, start, end - start, 279 from, to, prefetch_loc, preferred_loc, trigger); 280 } 281 282 void kfd_smi_event_migration_end(struct kfd_node *node, pid_t pid, 283 unsigned long start, unsigned long end, 284 uint32_t from, uint32_t to, uint32_t trigger) 285 { 286 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_MIGRATE_END, 287 "%lld -%d @%lx(%lx) %x->%x %d\n", 288 ktime_get_boottime_ns(), pid, start, end - start, 289 from, to, trigger); 290 } 291 292 void kfd_smi_event_queue_eviction(struct kfd_node *node, pid_t pid, 293 uint32_t trigger) 294 { 295 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_EVICTION, 296 "%lld -%d %x %d\n", ktime_get_boottime_ns(), pid, 297 node->id, trigger); 298 } 299 300 void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid) 301 { 302 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_RESTORE, 303 "%lld -%d %x\n", ktime_get_boottime_ns(), pid, 304 node->id); 305 } 306 307 void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) 308 { 309 struct kfd_process *p; 310 int i; 311 312 p = kfd_lookup_process_by_mm(mm); 313 if (!p) 314 return; 315 316 for (i = 0; i < p->n_pdds; i++) { 317 struct kfd_process_device *pdd = p->pdds[i]; 318 319 kfd_smi_event_add(p->lead_thread->pid, pdd->dev, 320 KFD_SMI_EVENT_QUEUE_RESTORE, 321 "%lld -%d %x %c\n", ktime_get_boottime_ns(), 322 p->lead_thread->pid, pdd->dev->id, 'R'); 323 } 324 kfd_unref_process(p); 325 } 326 327 void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, pid_t pid, 328 unsigned long address, unsigned long last, 329 uint32_t trigger) 330 { 331 kfd_smi_event_add(pid, node, KFD_SMI_EVENT_UNMAP_FROM_GPU, 332 "%lld -%d @%lx(%lx) %x %d\n", ktime_get_boottime_ns(), 333 pid, address, last - address + 1, node->id, trigger); 334 } 335 336 int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd) 337 { 338 struct kfd_smi_client *client; 339 int ret; 340 341 client = kzalloc(sizeof(struct kfd_smi_client), GFP_KERNEL); 342 if (!client) 343 return -ENOMEM; 344 INIT_LIST_HEAD(&client->list); 345 346 ret = kfifo_alloc(&client->fifo, MAX_KFIFO_SIZE, GFP_KERNEL); 347 if (ret) { 348 kfree(client); 349 return ret; 350 } 351 352 init_waitqueue_head(&client->wait_queue); 353 spin_lock_init(&client->lock); 354 client->events = 0; 355 client->dev = dev; 356 client->pid = current->tgid; 357 client->suser = capable(CAP_SYS_ADMIN); 358 359 spin_lock(&dev->smi_lock); 360 list_add_rcu(&client->list, &dev->smi_clients); 361 spin_unlock(&dev->smi_lock); 362 363 ret = anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops, (void *)client, 364 O_RDWR); 365 if (ret < 0) { 366 spin_lock(&dev->smi_lock); 367 list_del_rcu(&client->list); 368 spin_unlock(&dev->smi_lock); 369 370 synchronize_rcu(); 371 372 kfifo_free(&client->fifo); 373 kfree(client); 374 return ret; 375 } 376 *fd = ret; 377 378 return 0; 379 } 380