1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2020-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/poll.h> 25 #include <linux/wait.h> 26 #include <linux/anon_inodes.h> 27 #include <uapi/linux/kfd_ioctl.h> 28 #include "amdgpu.h" 29 #include "amdgpu_vm.h" 30 #include "kfd_priv.h" 31 #include "kfd_smi_events.h" 32 #include "amdgpu_reset.h" 33 34 struct kfd_smi_client { 35 struct list_head list; 36 struct kfifo fifo; 37 wait_queue_head_t wait_queue; 38 /* events enabled */ 39 uint64_t events; 40 struct kfd_node *dev; 41 spinlock_t lock; 42 struct rcu_head rcu; 43 pid_t pid; 44 bool suser; 45 }; 46 47 #define KFD_MAX_KFIFO_SIZE 8192 48 49 static __poll_t kfd_smi_ev_poll(struct file *, struct poll_table_struct *); 50 static ssize_t kfd_smi_ev_read(struct file *, char __user *, size_t, loff_t *); 51 static ssize_t kfd_smi_ev_write(struct file *, const char __user *, size_t, 52 loff_t *); 53 static int kfd_smi_ev_release(struct inode *, struct file *); 54 55 static const char kfd_smi_name[] = "kfd_smi_ev"; 56 57 static const struct file_operations kfd_smi_ev_fops = { 58 .owner = THIS_MODULE, 59 .poll = kfd_smi_ev_poll, 60 .read = kfd_smi_ev_read, 61 .write = kfd_smi_ev_write, 62 .release = kfd_smi_ev_release 63 }; 64 65 static __poll_t kfd_smi_ev_poll(struct file *filep, 66 struct poll_table_struct *wait) 67 { 68 struct kfd_smi_client *client = filep->private_data; 69 __poll_t mask = 0; 70 71 poll_wait(filep, &client->wait_queue, wait); 72 73 spin_lock(&client->lock); 74 if (!kfifo_is_empty(&client->fifo)) 75 mask = EPOLLIN | EPOLLRDNORM; 76 spin_unlock(&client->lock); 77 78 return mask; 79 } 80 81 static ssize_t kfd_smi_ev_read(struct file *filep, char __user *user, 82 size_t size, loff_t *offset) 83 { 84 int ret; 85 size_t to_copy; 86 struct kfd_smi_client *client = filep->private_data; 87 unsigned char *buf; 88 89 size = min_t(size_t, size, KFD_MAX_KFIFO_SIZE); 90 buf = kmalloc(size, GFP_KERNEL); 91 if (!buf) 92 return -ENOMEM; 93 94 /* kfifo_to_user can sleep so we can't use spinlock protection around 95 * it. Instead, we kfifo out as spinlocked then copy them to the user. 96 */ 97 spin_lock(&client->lock); 98 to_copy = kfifo_len(&client->fifo); 99 if (!to_copy) { 100 spin_unlock(&client->lock); 101 ret = -EAGAIN; 102 goto ret_err; 103 } 104 to_copy = min(size, to_copy); 105 ret = kfifo_out(&client->fifo, buf, to_copy); 106 spin_unlock(&client->lock); 107 if (ret <= 0) { 108 ret = -EAGAIN; 109 goto ret_err; 110 } 111 112 ret = copy_to_user(user, buf, to_copy); 113 if (ret) { 114 ret = -EFAULT; 115 goto ret_err; 116 } 117 118 kfree(buf); 119 return to_copy; 120 121 ret_err: 122 kfree(buf); 123 return ret; 124 } 125 126 static ssize_t kfd_smi_ev_write(struct file *filep, const char __user *user, 127 size_t size, loff_t *offset) 128 { 129 struct kfd_smi_client *client = filep->private_data; 130 uint64_t events; 131 132 if (!access_ok(user, size) || size < sizeof(events)) 133 return -EFAULT; 134 if (copy_from_user(&events, user, sizeof(events))) 135 return -EFAULT; 136 137 WRITE_ONCE(client->events, events); 138 139 return sizeof(events); 140 } 141 142 static void kfd_smi_ev_client_free(struct rcu_head *p) 143 { 144 struct kfd_smi_client *ev = container_of(p, struct kfd_smi_client, rcu); 145 146 kfifo_free(&ev->fifo); 147 kfree(ev); 148 } 149 150 static int kfd_smi_ev_release(struct inode *inode, struct file *filep) 151 { 152 struct kfd_smi_client *client = filep->private_data; 153 struct kfd_node *dev = client->dev; 154 155 spin_lock(&dev->smi_lock); 156 list_del_rcu(&client->list); 157 spin_unlock(&dev->smi_lock); 158 159 call_rcu(&client->rcu, kfd_smi_ev_client_free); 160 return 0; 161 } 162 163 static bool kfd_smi_ev_enabled(pid_t pid, struct kfd_smi_client *client, 164 unsigned int event) 165 { 166 uint64_t events = READ_ONCE(client->events); 167 168 if (pid && client->pid != pid && !client->suser) 169 return false; 170 171 return events & KFD_SMI_EVENT_MASK_FROM_INDEX(event); 172 } 173 174 static void add_event_to_kfifo(pid_t pid, struct kfd_node *dev, 175 unsigned int smi_event, char *event_msg, int len) 176 { 177 struct kfd_smi_client *client; 178 179 rcu_read_lock(); 180 181 list_for_each_entry_rcu(client, &dev->smi_clients, list) { 182 if (!kfd_smi_ev_enabled(pid, client, smi_event)) 183 continue; 184 spin_lock(&client->lock); 185 if (kfifo_avail(&client->fifo) >= len) { 186 kfifo_in(&client->fifo, event_msg, len); 187 wake_up_all(&client->wait_queue); 188 } else { 189 pr_debug("smi_event(EventID: %u): no space left\n", 190 smi_event); 191 } 192 spin_unlock(&client->lock); 193 } 194 195 rcu_read_unlock(); 196 } 197 198 /** 199 * kfd_smi_task_to_pid - Convert task to namespace-aware PID 200 * @task: task_struct pointer (typically p->lead_thread) 201 * 202 * Returns the PID as it appears in the task's own PID namespace. 203 * For containerized processes, this returns the container-local PID 204 * (what getpid() returns), not the global host PID. 205 * 206 * Returns 0 if task is NULL. 207 */ 208 static inline pid_t kfd_smi_task_to_pid(struct task_struct *task) 209 { 210 return task ? task_tgid_nr_ns(task, task_active_pid_ns(task)) : 0; 211 } 212 213 __printf(4, 5) 214 static void kfd_smi_event_add(struct task_struct *task, struct kfd_node *dev, 215 unsigned int event, char *fmt, ...) 216 { 217 char fifo_in[KFD_SMI_EVENT_MSG_SIZE]; 218 int len; 219 va_list args; 220 pid_t pid; 221 222 if (list_empty(&dev->smi_clients)) 223 return; 224 225 pid = kfd_smi_task_to_pid(task); 226 227 len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event); 228 229 va_start(args, fmt); 230 len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args); 231 va_end(args); 232 233 add_event_to_kfifo(pid, dev, event, fifo_in, len); 234 } 235 236 void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset, 237 struct amdgpu_reset_context *reset_context) 238 { 239 unsigned int event; 240 char reset_cause[64]; 241 242 if (post_reset) { 243 event = KFD_SMI_EVENT_GPU_POST_RESET; 244 } else { 245 event = KFD_SMI_EVENT_GPU_PRE_RESET; 246 ++(dev->reset_seq_num); 247 } 248 249 memset(reset_cause, 0, sizeof(reset_cause)); 250 251 if (reset_context) 252 amdgpu_reset_get_desc(reset_context, reset_cause, 253 sizeof(reset_cause)); 254 255 kfd_smi_event_add(NULL, dev, event, KFD_EVENT_FMT_UPDATE_GPU_RESET( 256 dev->reset_seq_num, reset_cause)); 257 } 258 259 void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev, 260 uint64_t throttle_bitmask) 261 { 262 kfd_smi_event_add(NULL, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, 263 KFD_EVENT_FMT_THERMAL_THROTTLING( 264 throttle_bitmask, 265 amdgpu_dpm_get_thermal_throttling_counter(dev->adev))); 266 } 267 268 void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid) 269 { 270 struct amdgpu_task_info *task_info; 271 272 task_info = amdgpu_vm_get_task_info_pasid(dev->adev, pasid); 273 if (task_info) { 274 /* Report VM faults from user applications, not retry from kernel */ 275 if (task_info->task.pid) 276 kfd_smi_event_add(NULL, dev, KFD_SMI_EVENT_VMFAULT, KFD_EVENT_FMT_VMFAULT( 277 task_info->task.pid, task_info->task.comm)); 278 amdgpu_vm_put_task_info(task_info); 279 } 280 } 281 282 void kfd_smi_event_page_fault_start(struct kfd_node *node, struct task_struct *task, 283 unsigned long address, bool write_fault, 284 ktime_t ts) 285 { 286 kfd_smi_event_add(task, node, KFD_SMI_EVENT_PAGE_FAULT_START, 287 KFD_EVENT_FMT_PAGEFAULT_START(ktime_to_ns(ts), 288 kfd_smi_task_to_pid(task), address, node->id, 289 write_fault ? 'W' : 'R')); 290 } 291 292 void kfd_smi_event_page_fault_end(struct kfd_node *node, struct task_struct *task, 293 unsigned long address, bool migration) 294 { 295 kfd_smi_event_add(task, node, KFD_SMI_EVENT_PAGE_FAULT_END, 296 KFD_EVENT_FMT_PAGEFAULT_END(ktime_get_boottime_ns(), 297 kfd_smi_task_to_pid(task), address, node->id, 298 migration ? 'M' : 'U')); 299 } 300 301 void kfd_smi_event_migration_start(struct kfd_node *node, struct task_struct *task, 302 unsigned long start, unsigned long end, 303 uint32_t from, uint32_t to, 304 uint32_t prefetch_loc, uint32_t preferred_loc, 305 uint32_t trigger) 306 { 307 kfd_smi_event_add(task, node, KFD_SMI_EVENT_MIGRATE_START, 308 KFD_EVENT_FMT_MIGRATE_START(ktime_get_boottime_ns(), 309 kfd_smi_task_to_pid(task), start, end - start, from, 310 to, prefetch_loc, preferred_loc, trigger)); 311 } 312 313 void kfd_smi_event_migration_end(struct kfd_node *node, struct task_struct *task, 314 unsigned long start, unsigned long end, 315 uint32_t from, uint32_t to, uint32_t trigger, 316 int error_code) 317 { 318 kfd_smi_event_add(task, node, KFD_SMI_EVENT_MIGRATE_END, 319 KFD_EVENT_FMT_MIGRATE_END(ktime_get_boottime_ns(), 320 kfd_smi_task_to_pid(task), start, end - start, from, 321 to, trigger, error_code)); 322 } 323 324 void kfd_smi_event_queue_eviction(struct kfd_node *node, struct task_struct *task, 325 uint32_t trigger) 326 { 327 kfd_smi_event_add(task, node, KFD_SMI_EVENT_QUEUE_EVICTION, 328 KFD_EVENT_FMT_QUEUE_EVICTION(ktime_get_boottime_ns(), 329 kfd_smi_task_to_pid(task), node->id, trigger)); 330 } 331 332 void kfd_smi_event_queue_restore(struct kfd_node *node, struct task_struct *task) 333 { 334 kfd_smi_event_add(task, node, KFD_SMI_EVENT_QUEUE_RESTORE, 335 KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), 336 kfd_smi_task_to_pid(task), node->id, '0')); 337 } 338 339 void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) 340 { 341 struct kfd_process *p; 342 int i; 343 344 p = kfd_lookup_process_by_mm(mm); 345 if (!p) 346 return; 347 348 for (i = 0; i < p->n_pdds; i++) { 349 struct kfd_process_device *pdd = p->pdds[i]; 350 351 kfd_smi_event_add(p->lead_thread, pdd->dev, 352 KFD_SMI_EVENT_QUEUE_RESTORE, 353 KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), 354 kfd_smi_task_to_pid(p->lead_thread), 355 pdd->dev->id, 'R')); 356 } 357 kfd_unref_process(p); 358 } 359 360 void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, struct task_struct *task, 361 unsigned long address, unsigned long last, 362 uint32_t trigger) 363 { 364 kfd_smi_event_add(task, node, KFD_SMI_EVENT_UNMAP_FROM_GPU, 365 KFD_EVENT_FMT_UNMAP_FROM_GPU(ktime_get_boottime_ns(), 366 kfd_smi_task_to_pid(task), address, 367 last - address + 1, node->id, trigger)); 368 } 369 370 void kfd_smi_event_process(struct kfd_process_device *pdd, bool start) 371 { 372 struct amdgpu_task_info *task_info; 373 struct amdgpu_vm *avm; 374 375 if (!pdd->drm_priv) 376 return; 377 378 avm = drm_priv_to_vm(pdd->drm_priv); 379 task_info = amdgpu_vm_get_task_info_vm(avm); 380 381 if (task_info) { 382 kfd_smi_event_add(NULL, pdd->dev, 383 start ? KFD_SMI_EVENT_PROCESS_START : 384 KFD_SMI_EVENT_PROCESS_END, 385 KFD_EVENT_FMT_PROCESS(task_info->task.pid, 386 task_info->task.comm)); 387 amdgpu_vm_put_task_info(task_info); 388 } 389 } 390 391 int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd) 392 { 393 struct kfd_smi_client *client; 394 int ret; 395 396 client = kzalloc_obj(struct kfd_smi_client); 397 if (!client) 398 return -ENOMEM; 399 INIT_LIST_HEAD(&client->list); 400 401 ret = kfifo_alloc(&client->fifo, KFD_MAX_KFIFO_SIZE, GFP_KERNEL); 402 if (ret) { 403 kfree(client); 404 return ret; 405 } 406 407 init_waitqueue_head(&client->wait_queue); 408 spin_lock_init(&client->lock); 409 client->events = 0; 410 client->dev = dev; 411 client->pid = kfd_smi_task_to_pid(current); 412 client->suser = capable(CAP_SYS_ADMIN); 413 414 spin_lock(&dev->smi_lock); 415 list_add_rcu(&client->list, &dev->smi_clients); 416 spin_unlock(&dev->smi_lock); 417 418 ret = anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops, (void *)client, 419 O_RDWR); 420 if (ret < 0) { 421 spin_lock(&dev->smi_lock); 422 list_del_rcu(&client->list); 423 spin_unlock(&dev->smi_lock); 424 425 synchronize_rcu(); 426 427 kfifo_free(&client->fifo); 428 kfree(client); 429 return ret; 430 } 431 *fd = ret; 432 433 return 0; 434 } 435