xref: /linux/drivers/gpu/drm/amd/amdkfd/kfd_process.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/mutex.h>
25 #include <linux/log2.h>
26 #include <linux/sched.h>
27 #include <linux/sched/mm.h>
28 #include <linux/sched/task.h>
29 #include <linux/mmu_context.h>
30 #include <linux/slab.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include <drm/ttm/ttm_bo.h>
37 #include "amdgpu_amdkfd.h"
38 #include "amdgpu.h"
39 #include "amdgpu_reset.h"
40 
41 struct mm_struct;
42 
43 #include "kfd_priv.h"
44 #include "kfd_device_queue_manager.h"
45 #include "kfd_svm.h"
46 #include "kfd_smi_events.h"
47 #include "kfd_debug.h"
48 
49 /*
50  * List of struct kfd_process (field kfd_process).
51  * Unique/indexed by mm_struct*
52  */
53 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
54 DEFINE_MUTEX(kfd_processes_mutex);
55 
56 DEFINE_SRCU(kfd_processes_srcu);
57 
58 /* For process termination handling */
59 static struct workqueue_struct *kfd_process_wq;
60 
61 /* Ordered, single-threaded workqueue for restoring evicted
62  * processes. Restoring multiple processes concurrently under memory
63  * pressure can lead to processes blocking each other from validating
64  * their BOs and result in a live-lock situation where processes
65  * remain evicted indefinitely.
66  */
67 static struct workqueue_struct *kfd_restore_wq;
68 
69 static struct kfd_process *find_process(const struct task_struct *thread,
70 					bool ref);
71 static void kfd_process_ref_release(struct kref *ref);
72 
73 static void evict_process_worker(struct work_struct *work);
74 static void restore_process_worker(struct work_struct *work);
75 
76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77 
78 struct kfd_procfs_tree {
79 	struct kobject *kobj;
80 };
81 
82 static struct kfd_procfs_tree procfs;
83 
84 /*
85  * Structure for SDMA activity tracking
86  */
87 struct kfd_sdma_activity_handler_workarea {
88 	struct work_struct sdma_activity_work;
89 	struct kfd_process_device *pdd;
90 	uint64_t sdma_activity_counter;
91 };
92 
93 struct temp_sdma_queue_list {
94 	uint64_t __user *rptr;
95 	uint64_t sdma_val;
96 	unsigned int queue_id;
97 	struct list_head list;
98 };
99 
100 static void kfd_sdma_activity_worker(struct work_struct *work)
101 {
102 	struct kfd_sdma_activity_handler_workarea *workarea;
103 	struct kfd_process_device *pdd;
104 	uint64_t val;
105 	struct mm_struct *mm;
106 	struct queue *q;
107 	struct qcm_process_device *qpd;
108 	struct device_queue_manager *dqm;
109 	int ret = 0;
110 	struct temp_sdma_queue_list sdma_q_list;
111 	struct temp_sdma_queue_list *sdma_q, *next;
112 
113 	workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114 				sdma_activity_work);
115 
116 	pdd = workarea->pdd;
117 	if (!pdd)
118 		return;
119 	dqm = pdd->dev->dqm;
120 	qpd = &pdd->qpd;
121 	if (!dqm || !qpd)
122 		return;
123 	/*
124 	 * Total SDMA activity is current SDMA activity + past SDMA activity
125 	 * Past SDMA count is stored in pdd.
126 	 * To get the current activity counters for all active SDMA queues,
127 	 * we loop over all SDMA queues and get their counts from user-space.
128 	 *
129 	 * We cannot call get_user() with dqm_lock held as it can cause
130 	 * a circular lock dependency situation. To read the SDMA stats,
131 	 * we need to do the following:
132 	 *
133 	 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134 	 *    with dqm_lock/dqm_unlock().
135 	 * 2. Call get_user() for each node in temporary list without dqm_lock.
136 	 *    Save the SDMA count for each node and also add the count to the total
137 	 *    SDMA count counter.
138 	 *    Its possible, during this step, a few SDMA queue nodes got deleted
139 	 *    from the qpd->queues_list.
140 	 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141 	 *    If any node got deleted, its SDMA count would be captured in the sdma
142 	 *    past activity counter. So subtract the SDMA counter stored in step 2
143 	 *    for this node from the total SDMA count.
144 	 */
145 	INIT_LIST_HEAD(&sdma_q_list.list);
146 
147 	/*
148 	 * Create the temp list of all SDMA queues
149 	 */
150 	dqm_lock(dqm);
151 
152 	list_for_each_entry(q, &qpd->queues_list, list) {
153 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155 			continue;
156 
157 		if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) {
158 			val = 0;
159 			ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
160 					dqm->dev->adev, q->mqd,
161 					dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
162 					&val);
163 
164 			if (ret)
165 				pr_debug("Failed to read SDMA queue active counter %i\n", ret);
166 			else
167 				workarea->sdma_activity_counter += val;
168 
169 			continue;
170 		}
171 
172 		sdma_q = kzalloc_obj(struct temp_sdma_queue_list);
173 		if (!sdma_q) {
174 			dqm_unlock(dqm);
175 			goto cleanup;
176 		}
177 
178 		INIT_LIST_HEAD(&sdma_q->list);
179 		sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
180 		sdma_q->queue_id = q->properties.queue_id;
181 		list_add_tail(&sdma_q->list, &sdma_q_list.list);
182 	}
183 
184 	/*
185 	 * If the temp list is empty, then no SDMA queues nodes were found in
186 	 * qpd->queues_list. Return the past activity count as the total sdma
187 	 * count
188 	 */
189 	if (list_empty(&sdma_q_list.list)) {
190 		workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
191 		dqm_unlock(dqm);
192 		return;
193 	}
194 
195 	dqm_unlock(dqm);
196 
197 	/*
198 	 * Get the usage count for each SDMA queue in temp_list.
199 	 */
200 	mm = get_task_mm(pdd->process->lead_thread);
201 	if (!mm)
202 		goto cleanup;
203 
204 	kthread_use_mm(mm);
205 
206 	list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
207 		val = 0;
208 		ret = read_sdma_queue_counter(sdma_q->rptr, &val);
209 		if (ret) {
210 			pr_debug("Failed to read SDMA queue active counter for queue id: %d",
211 				 sdma_q->queue_id);
212 		} else {
213 			sdma_q->sdma_val = val;
214 			workarea->sdma_activity_counter += val;
215 		}
216 	}
217 
218 	kthread_unuse_mm(mm);
219 	mmput(mm);
220 
221 	/*
222 	 * Do a second iteration over qpd_queues_list to check if any SDMA
223 	 * nodes got deleted while fetching SDMA counter.
224 	 */
225 	dqm_lock(dqm);
226 
227 	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
228 
229 	list_for_each_entry(q, &qpd->queues_list, list) {
230 		if (list_empty(&sdma_q_list.list))
231 			break;
232 
233 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
234 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
235 			continue;
236 
237 		list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
238 			if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
239 			     (sdma_q->queue_id == q->properties.queue_id)) {
240 				list_del(&sdma_q->list);
241 				kfree(sdma_q);
242 				break;
243 			}
244 		}
245 	}
246 
247 	dqm_unlock(dqm);
248 
249 	/*
250 	 * If temp list is not empty, it implies some queues got deleted
251 	 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
252 	 * count for each node from the total SDMA count.
253 	 */
254 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
255 		workarea->sdma_activity_counter -= sdma_q->sdma_val;
256 		list_del(&sdma_q->list);
257 		kfree(sdma_q);
258 	}
259 
260 	return;
261 
262 cleanup:
263 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
264 		list_del(&sdma_q->list);
265 		kfree(sdma_q);
266 	}
267 }
268 
269 /**
270  * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
271  * by current process. Translates acquired wave count into number of compute units
272  * that are occupied.
273  *
274  * @attr: Handle of attribute that allows reporting of wave count. The attribute
275  * handle encapsulates GPU device it is associated with, thereby allowing collection
276  * of waves in flight, etc
277  * @buffer: Handle of user provided buffer updated with wave count
278  *
279  * Return: Number of bytes written to user buffer or an error value
280  */
281 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
282 {
283 	int cu_cnt;
284 	int wave_cnt;
285 	int max_waves_per_cu;
286 	struct kfd_node *dev = NULL;
287 	struct kfd_process *proc = NULL;
288 	struct kfd_process_device *pdd = NULL;
289 	int i;
290 	struct kfd_cu_occupancy *cu_occupancy;
291 	u32 queue_format;
292 
293 	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
294 	dev = pdd->dev;
295 	if (dev->kfd2kgd->get_cu_occupancy == NULL)
296 		return -EINVAL;
297 
298 	cu_cnt = 0;
299 	proc = pdd->process;
300 	if (pdd->qpd.queue_count == 0) {
301 		pr_debug("Gpu-Id: %d has no active queues for process pid %d\n",
302 			 dev->id, (int)proc->lead_thread->pid);
303 		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
304 	}
305 
306 	/* Collect wave count from device if it supports */
307 	wave_cnt = 0;
308 	max_waves_per_cu = 0;
309 
310 	cu_occupancy = kzalloc_objs(*cu_occupancy, AMDGPU_MAX_QUEUES);
311 	if (!cu_occupancy)
312 		return -ENOMEM;
313 
314 	/*
315 	 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.
316 	 * For AQL queues, because of cooperative dispatch we multiply the wave count
317 	 * by number of XCCs in the partition to get the total wave counts across all
318 	 * XCCs in the partition.
319 	 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
320 	 */
321 	amdgpu_gfx_off_ctrl(dev->adev, false);
322 	dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
323 			&max_waves_per_cu, ffs(dev->xcc_mask) - 1);
324 	amdgpu_gfx_off_ctrl(dev->adev, true);
325 
326 	for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
327 		if (cu_occupancy[i].wave_cnt != 0 &&
328 		    kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,
329 						cu_occupancy[i].doorbell_off,
330 						&queue_format)) {
331 			if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))
332 				wave_cnt += cu_occupancy[i].wave_cnt;
333 			else
334 				wave_cnt += (NUM_XCC(dev->xcc_mask) *
335 						cu_occupancy[i].wave_cnt);
336 		}
337 	}
338 
339 	/* Translate wave count to number of compute units */
340 	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
341 	kfree(cu_occupancy);
342 	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
343 }
344 
345 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
346 			       char *buffer)
347 {
348 	if (strcmp(attr->name, "pasid") == 0)
349 		return snprintf(buffer, PAGE_SIZE, "%d\n", 0);
350 	else if (strncmp(attr->name, "vram_", 5) == 0) {
351 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
352 							      attr_vram);
353 		return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));
354 	} else if (strncmp(attr->name, "sdma_", 5) == 0) {
355 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
356 							      attr_sdma);
357 		struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
358 
359 		INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
360 				  kfd_sdma_activity_worker);
361 
362 		sdma_activity_work_handler.pdd = pdd;
363 		sdma_activity_work_handler.sdma_activity_counter = 0;
364 
365 		schedule_work(&sdma_activity_work_handler.sdma_activity_work);
366 
367 		flush_work(&sdma_activity_work_handler.sdma_activity_work);
368 		destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work);
369 
370 		return snprintf(buffer, PAGE_SIZE, "%llu\n",
371 				(sdma_activity_work_handler.sdma_activity_counter)/
372 				 SDMA_ACTIVITY_DIVISOR);
373 	} else {
374 		pr_err("Invalid attribute");
375 		return -EINVAL;
376 	}
377 
378 	return 0;
379 }
380 
381 static void kfd_procfs_kobj_release(struct kobject *kobj)
382 {
383 	kfree(kobj);
384 }
385 
386 static const struct sysfs_ops kfd_procfs_ops = {
387 	.show = kfd_procfs_show,
388 };
389 
390 static const struct kobj_type procfs_type = {
391 	.release = kfd_procfs_kobj_release,
392 	.sysfs_ops = &kfd_procfs_ops,
393 };
394 
395 void kfd_procfs_init(void)
396 {
397 	int ret = 0;
398 
399 	procfs.kobj = kfd_alloc_struct(procfs.kobj);
400 	if (!procfs.kobj)
401 		return;
402 
403 	ret = kobject_init_and_add(procfs.kobj, &procfs_type,
404 				   &kfd_device->kobj, "proc");
405 	if (ret) {
406 		pr_warn("Could not create procfs proc folder");
407 		/* If we fail to create the procfs, clean up */
408 		kfd_procfs_shutdown();
409 	}
410 }
411 
412 void kfd_procfs_shutdown(void)
413 {
414 	if (procfs.kobj) {
415 		kobject_del(procfs.kobj);
416 		kobject_put(procfs.kobj);
417 		procfs.kobj = NULL;
418 	}
419 }
420 
421 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
422 				     struct attribute *attr, char *buffer)
423 {
424 	struct queue *q = container_of(kobj, struct queue, kobj);
425 
426 	if (!strcmp(attr->name, "size"))
427 		return snprintf(buffer, PAGE_SIZE, "%llu",
428 				q->properties.queue_size);
429 	else if (!strcmp(attr->name, "type"))
430 		return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
431 	else if (!strcmp(attr->name, "gpuid"))
432 		return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
433 	else
434 		pr_err("Invalid attribute");
435 
436 	return 0;
437 }
438 
439 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
440 				     struct attribute *attr, char *buffer)
441 {
442 	if (strcmp(attr->name, "evicted_ms") == 0) {
443 		struct kfd_process_device *pdd = container_of(attr,
444 				struct kfd_process_device,
445 				attr_evict);
446 		uint64_t evict_jiffies;
447 
448 		evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
449 
450 		return snprintf(buffer,
451 				PAGE_SIZE,
452 				"%llu\n",
453 				jiffies64_to_msecs(evict_jiffies));
454 
455 	/* Sysfs handle that gets CU occupancy is per device */
456 	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
457 		return kfd_get_cu_occupancy(attr, buffer);
458 	} else {
459 		pr_err("Invalid attribute");
460 	}
461 
462 	return 0;
463 }
464 
465 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
466 				       struct attribute *attr, char *buf)
467 {
468 	struct kfd_process_device *pdd;
469 
470 	if (!strcmp(attr->name, "faults")) {
471 		pdd = container_of(attr, struct kfd_process_device,
472 				   attr_faults);
473 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
474 	}
475 	if (!strcmp(attr->name, "page_in")) {
476 		pdd = container_of(attr, struct kfd_process_device,
477 				   attr_page_in);
478 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
479 	}
480 	if (!strcmp(attr->name, "page_out")) {
481 		pdd = container_of(attr, struct kfd_process_device,
482 				   attr_page_out);
483 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
484 	}
485 	return 0;
486 }
487 
488 static struct attribute attr_queue_size = {
489 	.name = "size",
490 	.mode = KFD_SYSFS_FILE_MODE
491 };
492 
493 static struct attribute attr_queue_type = {
494 	.name = "type",
495 	.mode = KFD_SYSFS_FILE_MODE
496 };
497 
498 static struct attribute attr_queue_gpuid = {
499 	.name = "gpuid",
500 	.mode = KFD_SYSFS_FILE_MODE
501 };
502 
503 static struct attribute *procfs_queue_attrs[] = {
504 	&attr_queue_size,
505 	&attr_queue_type,
506 	&attr_queue_gpuid,
507 	NULL
508 };
509 ATTRIBUTE_GROUPS(procfs_queue);
510 
511 static const struct sysfs_ops procfs_queue_ops = {
512 	.show = kfd_procfs_queue_show,
513 };
514 
515 static const struct kobj_type procfs_queue_type = {
516 	.sysfs_ops = &procfs_queue_ops,
517 	.default_groups = procfs_queue_groups,
518 };
519 
520 static const struct sysfs_ops procfs_stats_ops = {
521 	.show = kfd_procfs_stats_show,
522 };
523 
524 static const struct kobj_type procfs_stats_type = {
525 	.sysfs_ops = &procfs_stats_ops,
526 	.release = kfd_procfs_kobj_release,
527 };
528 
529 static const struct sysfs_ops sysfs_counters_ops = {
530 	.show = kfd_sysfs_counters_show,
531 };
532 
533 static const struct kobj_type sysfs_counters_type = {
534 	.sysfs_ops = &sysfs_counters_ops,
535 	.release = kfd_procfs_kobj_release,
536 };
537 
538 int kfd_procfs_add_queue(struct queue *q)
539 {
540 	struct kfd_process *proc;
541 	int ret;
542 
543 	if (!q || !q->process)
544 		return -EINVAL;
545 	proc = q->process;
546 
547 	/* Create proc/<pid>/queues/<queue id> folder */
548 	if (!proc->kobj_queues)
549 		return -EFAULT;
550 	ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
551 			proc->kobj_queues, "%u", q->properties.queue_id);
552 	if (ret < 0) {
553 		pr_warn("Creating proc/<pid>/queues/%u failed",
554 			q->properties.queue_id);
555 		kobject_put(&q->kobj);
556 		return ret;
557 	}
558 
559 	return 0;
560 }
561 
562 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
563 				 char *name)
564 {
565 	int ret;
566 
567 	if (!kobj || !attr || !name)
568 		return;
569 
570 	attr->name = name;
571 	attr->mode = KFD_SYSFS_FILE_MODE;
572 	sysfs_attr_init(attr);
573 
574 	ret = sysfs_create_file(kobj, attr);
575 	if (ret)
576 		pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
577 }
578 
579 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
580 {
581 	int ret;
582 	int i;
583 	char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
584 
585 	if (!p || !p->kobj)
586 		return;
587 
588 	/*
589 	 * Create sysfs files for each GPU:
590 	 * - proc/<pid>/stats_<gpuid>/
591 	 * - proc/<pid>/stats_<gpuid>/evicted_ms
592 	 * - proc/<pid>/stats_<gpuid>/cu_occupancy
593 	 */
594 	for (i = 0; i < p->n_pdds; i++) {
595 		struct kfd_process_device *pdd = p->pdds[i];
596 
597 		snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
598 				"stats_%u", pdd->dev->id);
599 		pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
600 		if (!pdd->kobj_stats)
601 			return;
602 
603 		ret = kobject_init_and_add(pdd->kobj_stats,
604 					   &procfs_stats_type,
605 					   p->kobj,
606 					   "%s", stats_dir_filename);
607 
608 		if (ret) {
609 			pr_warn("Creating KFD proc/stats_%s folder failed",
610 				stats_dir_filename);
611 			kobject_put(pdd->kobj_stats);
612 			pdd->kobj_stats = NULL;
613 			return;
614 		}
615 
616 		kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
617 				      "evicted_ms");
618 		/* Add sysfs file to report compute unit occupancy */
619 		if (pdd->dev->kfd2kgd->get_cu_occupancy)
620 			kfd_sysfs_create_file(pdd->kobj_stats,
621 					      &pdd->attr_cu_occupancy,
622 					      "cu_occupancy");
623 	}
624 }
625 
626 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
627 {
628 	int ret = 0;
629 	int i;
630 	char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
631 
632 	if (!p || !p->kobj)
633 		return;
634 
635 	/*
636 	 * Create sysfs files for each GPU which supports SVM
637 	 * - proc/<pid>/counters_<gpuid>/
638 	 * - proc/<pid>/counters_<gpuid>/faults
639 	 * - proc/<pid>/counters_<gpuid>/page_in
640 	 * - proc/<pid>/counters_<gpuid>/page_out
641 	 */
642 	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
643 		struct kfd_process_device *pdd = p->pdds[i];
644 		struct kobject *kobj_counters;
645 
646 		snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
647 			"counters_%u", pdd->dev->id);
648 		kobj_counters = kfd_alloc_struct(kobj_counters);
649 		if (!kobj_counters)
650 			return;
651 
652 		ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
653 					   p->kobj, "%s", counters_dir_filename);
654 		if (ret) {
655 			pr_warn("Creating KFD proc/%s folder failed",
656 				counters_dir_filename);
657 			kobject_put(kobj_counters);
658 			return;
659 		}
660 
661 		pdd->kobj_counters = kobj_counters;
662 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
663 				      "faults");
664 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
665 				      "page_in");
666 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
667 				      "page_out");
668 	}
669 }
670 
671 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
672 {
673 	int i;
674 
675 	if (!p || !p->kobj)
676 		return;
677 
678 	/*
679 	 * Create sysfs files for each GPU:
680 	 * - proc/<pid>/vram_<gpuid>
681 	 * - proc/<pid>/sdma_<gpuid>
682 	 */
683 	for (i = 0; i < p->n_pdds; i++) {
684 		struct kfd_process_device *pdd = p->pdds[i];
685 
686 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
687 			 pdd->dev->id);
688 		kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
689 				      pdd->vram_filename);
690 
691 		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
692 			 pdd->dev->id);
693 		kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
694 					    pdd->sdma_filename);
695 	}
696 }
697 
698 void kfd_procfs_del_queue(struct queue *q)
699 {
700 	if (!q || !q->process->kobj)
701 		return;
702 
703 	kobject_del(&q->kobj);
704 	kobject_put(&q->kobj);
705 }
706 
707 int kfd_process_create_wq(void)
708 {
709 	if (!kfd_process_wq)
710 		kfd_process_wq = alloc_workqueue("kfd_process_wq", WQ_UNBOUND,
711 						 0);
712 	if (!kfd_restore_wq)
713 		kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
714 							 WQ_FREEZABLE);
715 
716 	if (!kfd_process_wq || !kfd_restore_wq) {
717 		kfd_process_destroy_wq();
718 		return -ENOMEM;
719 	}
720 
721 	return 0;
722 }
723 
724 void kfd_process_destroy_wq(void)
725 {
726 	if (kfd_process_wq) {
727 		destroy_workqueue(kfd_process_wq);
728 		kfd_process_wq = NULL;
729 	}
730 	if (kfd_restore_wq) {
731 		destroy_workqueue(kfd_restore_wq);
732 		kfd_restore_wq = NULL;
733 	}
734 }
735 
736 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
737 			struct kfd_process_device *pdd, void **kptr)
738 {
739 	struct kfd_node *dev = pdd->dev;
740 
741 	if (kptr && *kptr) {
742 		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
743 		*kptr = NULL;
744 	}
745 
746 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
747 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
748 					       NULL);
749 }
750 
751 static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
752 				       struct kfd_process_device *pdd,
753 				       struct iosys_map *map)
754 {
755 	struct kfd_node *dev = pdd->dev;
756 
757 	if (map && !iosys_map_is_null(map)) {
758 		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
759 		iosys_map_clear(map);
760 	}
761 
762 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
763 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
764 					       NULL);
765 }
766 
767 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
768  *	This function should be only called right after the process
769  *	is created and when kfd_processes_mutex is still being held
770  *	to avoid concurrency. Because of that exclusiveness, we do
771  *	not need to take p->mutex.
772  */
773 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
774 				   uint64_t gpu_va, uint32_t size,
775 				   uint32_t flags, struct kgd_mem **mem, void **kptr)
776 {
777 	struct kfd_node *kdev = pdd->dev;
778 	int err;
779 
780 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
781 						 pdd->drm_priv, mem, NULL,
782 						 flags, false);
783 	if (err)
784 		goto err_alloc_mem;
785 
786 	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
787 			pdd->drm_priv);
788 	if (err)
789 		goto err_map_mem;
790 
791 	err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
792 	if (err) {
793 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
794 		goto sync_memory_failed;
795 	}
796 
797 	if (kptr) {
798 		u32 domain;
799 
800 		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
801 			domain = AMDGPU_GEM_DOMAIN_VRAM;
802 		else
803 			domain = AMDGPU_GEM_DOMAIN_GTT;
804 		err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel((struct kgd_mem *)*mem,
805 							   kptr, NULL, domain);
806 		if (err) {
807 			pr_debug("Map BO to kernel failed err %d\n", err);
808 			goto sync_memory_failed;
809 		}
810 	}
811 
812 	return err;
813 
814 sync_memory_failed:
815 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
816 
817 err_map_mem:
818 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
819 					       NULL);
820 err_alloc_mem:
821 	*mem = NULL;
822 	*kptr = NULL;
823 	return err;
824 }
825 
826 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
827  *	process for IB usage The memory reserved is for KFD to submit
828  *	IB to AMDGPU from kernel.  If the memory is reserved
829  *	successfully, ib_kaddr will have the CPU/kernel
830  *	address. Check ib_kaddr before accessing the memory.
831  */
832 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
833 {
834 	struct qcm_process_device *qpd = &pdd->qpd;
835 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
836 			KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
837 			KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
838 			KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
839 	struct kgd_mem *mem;
840 	void *kaddr;
841 	int ret;
842 
843 	if (qpd->ib_kaddr || !qpd->ib_base)
844 		return 0;
845 
846 	/* ib_base is only set for dGPU */
847 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
848 				      &mem, &kaddr);
849 	if (ret)
850 		return ret;
851 
852 	qpd->ib_mem = mem;
853 	qpd->ib_kaddr = kaddr;
854 
855 	return 0;
856 }
857 
858 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
859 {
860 	struct qcm_process_device *qpd = &pdd->qpd;
861 
862 	if (!qpd->ib_kaddr || !qpd->ib_base)
863 		return;
864 
865 	kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
866 }
867 
868 int kfd_create_process_sysfs(struct kfd_process *process)
869 {
870 	struct kfd_process *primary_process;
871 	int ret;
872 
873 	if (process->kobj) {
874 		pr_warn("kobject already exists for the kfd_process\n");
875 		return -EINVAL;
876 	}
877 
878 	process->kobj = kfd_alloc_struct(process->kobj);
879 	if (!process->kobj) {
880 		pr_warn("Creating procfs kobject failed");
881 		return -ENOMEM;
882 	}
883 
884 	if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
885 		ret = kobject_init_and_add(process->kobj, &procfs_type,
886 					   procfs.kobj, "%d",
887 					   (int)process->lead_thread->pid);
888 	else {
889 		primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
890 		if (!primary_process)
891 			return -ESRCH;
892 
893 		ret = kobject_init_and_add(process->kobj, &procfs_type,
894 					   primary_process->kobj, "context_%u",
895 					   process->context_id);
896 		kfd_unref_process(primary_process);
897 	}
898 
899 	if (ret) {
900 		pr_warn("Creating procfs pid directory failed");
901 		kobject_put(process->kobj);
902 		process->kobj = NULL;
903 		return ret;
904 	}
905 
906 	kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
907 			      "pasid");
908 
909 	process->kobj_queues = kobject_create_and_add("queues",
910 						process->kobj);
911 	if (!process->kobj_queues)
912 		pr_warn("Creating KFD proc/queues folder failed");
913 
914 	kfd_procfs_add_sysfs_stats(process);
915 	kfd_procfs_add_sysfs_files(process);
916 	kfd_procfs_add_sysfs_counters(process);
917 
918 	return 0;
919 }
920 
921 static int kfd_process_alloc_id(struct kfd_process *process)
922 {
923 	int ret;
924 	struct kfd_process *primary_process;
925 
926 	/* already assign 0xFFFF when create */
927 	if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
928 		return 0;
929 
930 	primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
931 	if (!primary_process)
932 		return -ESRCH;
933 
934 	/* id range: KFD_CONTEXT_ID_MIN to 0xFFFE */
935 	ret = ida_alloc_range(&primary_process->id_table, KFD_CONTEXT_ID_MIN,
936 	      KFD_CONTEXT_ID_PRIMARY - 1, GFP_KERNEL);
937 	if (ret < 0)
938 		goto out;
939 
940 	process->context_id = ret;
941 	ret = 0;
942 
943 out:
944 	kfd_unref_process(primary_process);
945 
946 	return ret;
947 }
948 
949 static void kfd_process_free_id(struct kfd_process *process)
950 {
951 	struct kfd_process *primary_process;
952 
953 	if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
954 		return;
955 
956 	primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
957 	if (!primary_process)
958 		return;
959 
960 	ida_free(&primary_process->id_table, process->context_id);
961 
962 	kfd_unref_process(primary_process);
963 }
964 
965 struct kfd_process *kfd_create_process(struct task_struct *thread)
966 {
967 	struct kfd_process *process;
968 	int ret;
969 
970 	if (!(thread->mm && mmget_not_zero(thread->mm)))
971 		return ERR_PTR(-EINVAL);
972 
973 	/* If the process just called exec(3), it is possible that the
974 	 * cleanup of the kfd_process (following the release of the mm
975 	 * of the old process image) is still in the cleanup work queue.
976 	 * Make sure to drain any job before trying to recreate any
977 	 * resource for this process.
978 	 */
979 	flush_workqueue(kfd_process_wq);
980 
981 	/*
982 	 * take kfd processes mutex before starting of process creation
983 	 * so there won't be a case where two threads of the same process
984 	 * create two kfd_process structures
985 	 */
986 	mutex_lock(&kfd_processes_mutex);
987 
988 	if (kfd_gpu_node_num() <= 0) {
989 		pr_warn("no gpu node! Cannot create KFD process");
990 		process = ERR_PTR(-EINVAL);
991 		goto out;
992 	}
993 
994 	if (kfd_is_locked(NULL)) {
995 		pr_debug("KFD is locked! Cannot create process");
996 		process = ERR_PTR(-EINVAL);
997 		goto out;
998 	}
999 
1000 	/* A prior open of /dev/kfd could have already created the process.
1001 	 * find_process will increase process kref in this case
1002 	 */
1003 	process = find_process(thread, true);
1004 	if (process) {
1005 		pr_debug("Process already found\n");
1006 	} else {
1007 		process = create_process(thread, true);
1008 		if (IS_ERR(process))
1009 			goto out;
1010 
1011 		if (!procfs.kobj)
1012 			goto out;
1013 
1014 		ret = kfd_create_process_sysfs(process);
1015 		if (ret)
1016 			pr_warn("Failed to create sysfs entry for the kfd_process");
1017 
1018 		ret = kfd_debugfs_add_process(process);
1019 		if (ret)
1020 			pr_warn("Failed to create debugfs entry for the kfd_process, ret = %d\n",
1021 				ret);
1022 
1023 		init_waitqueue_head(&process->wait_irq_drain);
1024 	}
1025 out:
1026 	mutex_unlock(&kfd_processes_mutex);
1027 	mmput(thread->mm);
1028 
1029 	return process;
1030 }
1031 
1032 /**
1033  * amdgpu_amdkfd_set_sigbus_delay - Set per-process KFD SIGBUS delay
1034  * @task: task in the target process
1035  * @ms:   encoded delay value (0 = immediate, 0xFFFFFFFF = suppress,
1036  *        otherwise delay in milliseconds)
1037  *
1038  * Stores the SIGBUS delivery option on the kfd_process associated with
1039  * @task. If the calling process has not opened /dev/kfd yet (no
1040  * kfd_process exists), this is a no-op - the option only applies to
1041  * processes that actually use KFD.
1042  */
1043 int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms)
1044 {
1045 	struct kfd_process *p;
1046 
1047 	if (!task->mm)
1048 		return -EINVAL;
1049 
1050 	p = kfd_lookup_process_by_mm(task->mm);
1051 	if (!p)
1052 		return 0;
1053 
1054 	atomic_set(&p->kfd_sigbus_delay_ms, ms);
1055 	kfd_unref_process(p);
1056 	return 0;
1057 }
1058 
1059 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
1060 {
1061 	struct kfd_process *process;
1062 
1063 	hash_for_each_possible_rcu(kfd_processes_table, process,
1064 					kfd_processes, (uintptr_t)mm)
1065 		if (process->mm == mm && process->context_id == KFD_CONTEXT_ID_PRIMARY)
1066 			return process;
1067 
1068 	return NULL;
1069 }
1070 
1071 static struct kfd_process *find_process(const struct task_struct *thread,
1072 					bool ref)
1073 {
1074 	struct kfd_process *p;
1075 	int idx;
1076 
1077 	idx = srcu_read_lock(&kfd_processes_srcu);
1078 	p = find_process_by_mm(thread->mm);
1079 	if (p && ref)
1080 		kref_get(&p->ref);
1081 	srcu_read_unlock(&kfd_processes_srcu, idx);
1082 
1083 	return p;
1084 }
1085 
1086 void kfd_unref_process(struct kfd_process *p)
1087 {
1088 	kref_put(&p->ref, kfd_process_ref_release);
1089 }
1090 
1091 /* This increments the process->ref counter. */
1092 struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
1093 {
1094 	struct task_struct *task = NULL;
1095 	struct kfd_process *p    = NULL;
1096 
1097 	if (!pid) {
1098 		task = current;
1099 		get_task_struct(task);
1100 	} else {
1101 		task = get_pid_task(pid, PIDTYPE_PID);
1102 	}
1103 
1104 	if (task) {
1105 		p = find_process(task, true);
1106 		put_task_struct(task);
1107 	}
1108 
1109 	return p;
1110 }
1111 
1112 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
1113 {
1114 	struct kfd_process *p = pdd->process;
1115 	void *mem;
1116 	int id;
1117 	int i;
1118 
1119 	/*
1120 	 * Remove all handles from idr and release appropriate
1121 	 * local memory object
1122 	 */
1123 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1124 
1125 		for (i = 0; i < p->n_pdds; i++) {
1126 			struct kfd_process_device *peer_pdd = p->pdds[i];
1127 
1128 			if (!peer_pdd->drm_priv)
1129 				continue;
1130 			amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1131 				peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
1132 		}
1133 
1134 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
1135 						       pdd->drm_priv, NULL);
1136 		kfd_process_device_remove_obj_handle(pdd, id);
1137 	}
1138 }
1139 
1140 /*
1141  * Just kunmap and unpin signal BO here. It will be freed in
1142  * kfd_process_free_outstanding_kfd_bos()
1143  */
1144 static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
1145 {
1146 	struct kfd_process_device *pdd;
1147 	struct kfd_node *kdev;
1148 	void *mem;
1149 
1150 	kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
1151 	if (!kdev)
1152 		return;
1153 
1154 	mutex_lock(&p->mutex);
1155 
1156 	pdd = kfd_get_process_device_data(kdev, p);
1157 	if (!pdd)
1158 		goto out;
1159 
1160 	mem = kfd_process_device_translate_handle(
1161 		pdd, GET_IDR_HANDLE(p->signal_handle));
1162 	if (!mem)
1163 		goto out;
1164 
1165 	amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
1166 
1167 out:
1168 	mutex_unlock(&p->mutex);
1169 }
1170 
1171 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1172 {
1173 	int i;
1174 
1175 	for (i = 0; i < p->n_pdds; i++)
1176 		kfd_process_device_free_bos(p->pdds[i]);
1177 }
1178 
1179 static void kfd_process_profiler_release(struct kfd_process *p, struct kfd_process_device *pdd)
1180 {
1181 	mutex_lock(&pdd->dev->kfd->profiler_lock);
1182 	if (pdd->dev->kfd->profiler_process == p) {
1183 		pdd->qpd.dqm->ops.set_perfcount(pdd->qpd.dqm, 0);
1184 		pdd->dev->kfd->profiler_process = NULL;
1185 	}
1186 	mutex_unlock(&pdd->dev->kfd->profiler_lock);
1187 }
1188 
1189 static void kfd_process_destroy_pdds(struct kfd_process *p)
1190 {
1191 	int i;
1192 
1193 	for (i = 0; i < p->n_pdds; i++) {
1194 		struct kfd_process_device *pdd = p->pdds[i];
1195 
1196 		kfd_smi_event_process(pdd, false);
1197 
1198 		pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
1199 			pdd->dev->id, p->lead_thread->pid);
1200 		kfd_process_profiler_release(p, pdd);
1201 
1202 		if (pdd->ptl_disable_req)
1203 			kfd_ptl_disable_release(pdd, p);
1204 
1205 		kfd_process_device_destroy_cwsr_dgpu(pdd);
1206 		kfd_process_device_destroy_ib_mem(pdd);
1207 
1208 		if (pdd->drm_file)
1209 			fput(pdd->drm_file);
1210 
1211 		if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
1212 			free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
1213 				   get_order(KFD_CWSR_TBA_TMA_SIZE));
1214 
1215 		idr_destroy(&pdd->alloc_idr);
1216 
1217 		kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1218 
1219 		if (pdd->dev->kfd->shared_resources.enable_mes &&
1220 			pdd->proc_ctx_cpu_ptr) {
1221 			amdgpu_mes_free_proc_ctx_index(&pdd->dev->adev->mes,
1222 						       pdd->proc_ctx_array_index);
1223 			amdgpu_amdkfd_free_kernel_mem(pdd->dev->adev,
1224 						   &pdd->proc_ctx_bo);
1225 		}
1226 		/*
1227 		 * before destroying pdd, make sure to report availability
1228 		 * for auto suspend
1229 		 */
1230 		if (pdd->runtime_inuse) {
1231 			pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1232 			pdd->runtime_inuse = false;
1233 		}
1234 
1235 		atomic_dec(&pdd->dev->kfd->kfd_processes_count);
1236 		mutex_destroy(&pdd->doorbell_mutex);
1237 
1238 		kfree(pdd);
1239 		p->pdds[i] = NULL;
1240 	}
1241 	p->n_pdds = 0;
1242 }
1243 
1244 static void kfd_process_remove_sysfs(struct kfd_process *p)
1245 {
1246 	struct kfd_process_device *pdd;
1247 	int i;
1248 
1249 	if (!p->kobj)
1250 		return;
1251 
1252 	if (p->kobj_queues) {
1253 		sysfs_remove_file(p->kobj, &p->attr_pasid);
1254 		kobject_del(p->kobj_queues);
1255 		kobject_put(p->kobj_queues);
1256 		p->kobj_queues = NULL;
1257 	}
1258 
1259 	for (i = 0; i < p->n_pdds; i++) {
1260 		pdd = p->pdds[i];
1261 
1262 		sysfs_remove_file(p->kobj, &pdd->attr_vram);
1263 		sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1264 
1265 		if (pdd->kobj_stats) {
1266 			sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1267 			if (pdd->dev->kfd2kgd->get_cu_occupancy)
1268 				sysfs_remove_file(pdd->kobj_stats,
1269 						  &pdd->attr_cu_occupancy);
1270 			kobject_del(pdd->kobj_stats);
1271 			kobject_put(pdd->kobj_stats);
1272 			pdd->kobj_stats = NULL;
1273 		}
1274 	}
1275 
1276 	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1277 		pdd = p->pdds[i];
1278 		if (!pdd->kobj_counters)
1279 			continue;
1280 
1281 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1282 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1283 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1284 		kobject_del(pdd->kobj_counters);
1285 		kobject_put(pdd->kobj_counters);
1286 		pdd->kobj_counters = NULL;
1287 	}
1288 
1289 	kobject_del(p->kobj);
1290 	kobject_put(p->kobj);
1291 	p->kobj = NULL;
1292 }
1293 
1294 /*
1295  * If any GPU is ongoing reset, wait for reset complete.
1296  */
1297 void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1298 {
1299 	int i;
1300 
1301 	for (i = 0; i < p->n_pdds; i++)
1302 		flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1303 }
1304 
1305 /* No process locking is needed in this function, because the process
1306  * is not findable any more. We must assume that no other thread is
1307  * using it any more, otherwise we couldn't safely free the process
1308  * structure in the end.
1309  */
1310 static void kfd_process_wq_release(struct work_struct *work)
1311 {
1312 	struct kfd_process *p = container_of(work, struct kfd_process,
1313 					     release_work);
1314 	struct dma_fence *ef;
1315 
1316 	/*
1317 	 * If GPU in reset, user queues may still running, wait for reset complete.
1318 	 */
1319 	kfd_process_wait_gpu_reset_complete(p);
1320 
1321 	/* Signal the eviction fence after user mode queues are
1322 	 * destroyed. This allows any BOs to be freed without
1323 	 * triggering pointless evictions or waiting for fences.
1324 	 */
1325 	synchronize_rcu();
1326 	ef = rcu_access_pointer(p->ef);
1327 	if (ef)
1328 		dma_fence_signal(ef);
1329 
1330 	if (p->context_id != KFD_CONTEXT_ID_PRIMARY)
1331 		kfd_process_free_id(p);
1332 	else
1333 		ida_destroy(&p->id_table);
1334 
1335 	kfd_debugfs_remove_process(p);
1336 
1337 	/*
1338 	 * Remove the proc/sysfs entries before destroying PDDs. The removal path
1339 	 * walks the PDD array and sysfs callbacks dereference PDD fields, so the
1340 	 * backing data must remain valid until sysfs removal has completed.
1341 	 */
1342 	kfd_process_remove_sysfs(p);
1343 
1344 	kfd_process_kunmap_signal_bo(p);
1345 	kfd_process_free_outstanding_kfd_bos(p);
1346 	svm_range_list_fini(p);
1347 
1348 	kfd_process_destroy_pdds(p);
1349 	dma_fence_put(ef);
1350 
1351 	kfd_event_free_process(p);
1352 
1353 	mutex_destroy(&p->mutex);
1354 
1355 	put_task_struct(p->lead_thread);
1356 
1357 	kfree(p);
1358 }
1359 
1360 static void kfd_process_ref_release(struct kref *ref)
1361 {
1362 	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1363 
1364 	INIT_WORK(&p->release_work, kfd_process_wq_release);
1365 	queue_work(kfd_process_wq, &p->release_work);
1366 }
1367 
1368 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1369 {
1370 	/* This increments p->ref counter if kfd process p exists */
1371 	struct kfd_process *p = kfd_lookup_process_by_mm(mm);
1372 
1373 	return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1374 }
1375 
1376 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1377 {
1378 	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1379 }
1380 
1381 static void kfd_process_table_remove(struct kfd_process *p)
1382 {
1383 	mutex_lock(&kfd_processes_mutex);
1384 	/*
1385 	 * Do early return if table is empty.
1386 	 *
1387 	 * This could potentially happen if this function is called concurrently
1388 	 * by mmu_notifier and by kfd_cleanup_pocesses.
1389 	 *
1390 	 */
1391 	if (hash_empty(kfd_processes_table)) {
1392 		mutex_unlock(&kfd_processes_mutex);
1393 		return;
1394 	}
1395 	hash_del_rcu(&p->kfd_processes);
1396 	mutex_unlock(&kfd_processes_mutex);
1397 	synchronize_srcu(&kfd_processes_srcu);
1398 }
1399 
1400 void kfd_process_notifier_release_internal(struct kfd_process *p)
1401 {
1402 	int i;
1403 
1404 	kfd_process_table_remove(p);
1405 	cancel_delayed_work_sync(&p->eviction_work);
1406 	cancel_delayed_work_sync(&p->restore_work);
1407 	/*
1408 	 * If work pending, cancel it and drop the extra ref
1409 	 */
1410 	if (cancel_delayed_work_sync(&p->signal_work))
1411 		kfd_unref_process(p);
1412 
1413 	/*
1414 	 * Disable debug BEFORE dequeuing/flushing the MES process context.
1415 	 *
1416 	 * kfd_dbg_trap_disable() -> kfd_dbg_trap_deactivate() issues
1417 	 * SET_SHADER_DEBUGGER (process_ctx_flush=0) packets that re-add the
1418 	 * process to the MES scheduler list. kfd_process_dequeue_from_all_devices()
1419 	 * sends the process_ctx_flush=1 that retires the context. If debug
1420 	 * teardown ran after the dequeue, those packets would put the process
1421 	 * back on the MES list and, once proc_ctx_bo is freed, MES would fault
1422 	 * on the freed process context. Retiring the context last avoids this.
1423 	 */
1424 	kfd_dbg_trap_disable(p);
1425 
1426 	if (atomic_read(&p->debugged_process_count) > 0) {
1427 		struct kfd_process *target;
1428 		unsigned int temp;
1429 		int idx = srcu_read_lock(&kfd_processes_srcu);
1430 
1431 		hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1432 			if (target->debugger_process && target->debugger_process == p) {
1433 				mutex_lock_nested(&target->mutex, 1);
1434 				kfd_dbg_trap_disable(target);
1435 				mutex_unlock(&target->mutex);
1436 				if (atomic_read(&p->debugged_process_count) == 0)
1437 					break;
1438 			}
1439 		}
1440 
1441 		srcu_read_unlock(&kfd_processes_srcu, idx);
1442 	}
1443 
1444 	/*
1445 	 * Dequeue and destroy user queues, it is not safe for GPU to access
1446 	 * system memory after mmu release notifier callback returns because
1447 	 * exit_mmap free process memory afterwards.
1448 	 */
1449 	kfd_process_dequeue_from_all_devices(p);
1450 	pqm_uninit(&p->pqm);
1451 
1452 	for (i = 0; i < p->n_pdds; i++) {
1453 		struct kfd_process_device *pdd = p->pdds[i];
1454 
1455 		/* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1456 		if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1457 			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1458 	}
1459 
1460 	/* Indicate to other users that MM is no longer valid */
1461 	p->mm = NULL;
1462 
1463 	if (p->context_id == KFD_CONTEXT_ID_PRIMARY)
1464 		mmu_notifier_put(&p->mmu_notifier);
1465 }
1466 
1467 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1468 					struct mm_struct *mm)
1469 {
1470 	struct kfd_process *p;
1471 
1472 	/*
1473 	 * The kfd_process structure can not be free because the
1474 	 * mmu_notifier srcu is read locked
1475 	 */
1476 	p = container_of(mn, struct kfd_process, mmu_notifier);
1477 	if (WARN_ON(p->mm != mm))
1478 		return;
1479 
1480 	kfd_process_notifier_release_internal(p);
1481 }
1482 
1483 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1484 	.release = kfd_process_notifier_release,
1485 	.alloc_notifier = kfd_process_alloc_notifier,
1486 	.free_notifier = kfd_process_free_notifier,
1487 };
1488 
1489 /*
1490  * This code handles the case when driver is being unloaded before all
1491  * mm_struct are released.  We need to safely free the kfd_process and
1492  * avoid race conditions with mmu_notifier that might try to free them.
1493  *
1494  */
1495 void kfd_cleanup_processes(void)
1496 {
1497 	struct kfd_process *p;
1498 	struct hlist_node *p_temp;
1499 	unsigned int temp;
1500 	HLIST_HEAD(cleanup_list);
1501 
1502 	/*
1503 	 * Move all remaining kfd_process from the process table to a
1504 	 * temp list for processing.   Once done, callback from mmu_notifier
1505 	 * release will not see the kfd_process in the table and do early return,
1506 	 * avoiding double free issues.
1507 	 */
1508 	mutex_lock(&kfd_processes_mutex);
1509 	hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1510 		hash_del_rcu(&p->kfd_processes);
1511 		synchronize_srcu(&kfd_processes_srcu);
1512 		hlist_add_head(&p->kfd_processes, &cleanup_list);
1513 	}
1514 	mutex_unlock(&kfd_processes_mutex);
1515 
1516 	hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1517 		kfd_process_notifier_release_internal(p);
1518 
1519 	/*
1520 	 * Ensures that all outstanding free_notifier get called, triggering
1521 	 * the release of the kfd_process struct.
1522 	 */
1523 	mmu_notifier_synchronize();
1524 }
1525 
1526 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1527 {
1528 	struct kfd_node *dev = pdd->dev;
1529 	struct qcm_process_device *qpd = &pdd->qpd;
1530 	u32 flags = KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1531 			| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1532 	struct kgd_mem *mem;
1533 	void *kaddr;
1534 	int ret;
1535 
1536 	if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
1537 		return 0;
1538 
1539 	if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
1540 		flags |= KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
1541 	else
1542 		flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT;
1543 
1544 	/* Allocate CWSR TBA/TMA buffers */
1545 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1546 				      KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1547 	if (ret)
1548 		return ret;
1549 
1550 	qpd->cwsr_mem = mem;
1551 
1552 	/* Set up iosys_map based on whether memory is MMIO or system memory */
1553 	if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
1554 		iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
1555 	else
1556 		iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
1557 
1558 	qpd->tba_addr = qpd->cwsr_base;
1559 
1560 	/* Copy CWSR ISA to buffer using appropriate accessor */
1561 	iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
1562 			    dev->kfd->cwsr_isa_size);
1563 
1564 	kfd_process_set_trap_debug_flag(&pdd->qpd,
1565 					pdd->process->debug_trap_enabled);
1566 
1567 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1568 	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
1569 		 qpd->tba_addr, qpd->tma_addr,
1570 		 qpd->cwsr_map.is_iomem ? "iomem" : "system",
1571 		 qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
1572 					  qpd->cwsr_map.vaddr);
1573 
1574 	return 0;
1575 }
1576 
1577 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1578 {
1579 	struct kfd_node *dev = pdd->dev;
1580 	struct qcm_process_device *qpd = &pdd->qpd;
1581 
1582 	if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
1583 		return;
1584 
1585 	kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
1586 }
1587 
1588 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1589 				  uint64_t tba_addr,
1590 				  uint64_t tma_addr)
1591 {
1592 	if (!iosys_map_is_null(&qpd->cwsr_map)) {
1593 		/* KFD trap handler is bound, record as second-level TBA/TMA
1594 		 * in first-level TMA. First-level trap will jump to second.
1595 		 */
1596 		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
1597 			     uint64_t, tba_addr);
1598 		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
1599 			     uint64_t, tma_addr);
1600 	} else {
1601 		/* No trap handler bound, bind as first-level TBA/TMA. */
1602 		qpd->tba_addr = tba_addr;
1603 		qpd->tma_addr = tma_addr;
1604 	}
1605 }
1606 
1607 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1608 {
1609 	int i;
1610 
1611 	/* On most GFXv9 GPUs, the retry mode in the SQ must match the
1612 	 * boot time retry setting. Mixing processes with different
1613 	 * XNACK/retry settings can hang the GPU.
1614 	 *
1615 	 * Different GPUs can have different noretry settings depending
1616 	 * on HW bugs or limitations. We need to find at least one
1617 	 * XNACK mode for this process that's compatible with all GPUs.
1618 	 * Fortunately GPUs with retry enabled (noretry=0) can run code
1619 	 * built for XNACK-off. On GFXv9 it may perform slower.
1620 	 *
1621 	 * Therefore applications built for XNACK-off can always be
1622 	 * supported and will be our fallback if any GPU does not
1623 	 * support retry.
1624 	 */
1625 	for (i = 0; i < p->n_pdds; i++) {
1626 		struct kfd_node *dev = p->pdds[i]->dev;
1627 
1628 		/* Only consider GFXv9 and higher GPUs. Older GPUs don't
1629 		 * support the SVM APIs and don't need to be considered
1630 		 * for the XNACK mode selection.
1631 		 */
1632 		if (!KFD_IS_SOC15(dev))
1633 			continue;
1634 		/* Aldebaran can always support XNACK because it can support
1635 		 * per-process XNACK mode selection. But let the dev->noretry
1636 		 * setting still influence the default XNACK mode.
1637 		 */
1638 		if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1639 			if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {
1640 				pr_debug("SRIOV platform xnack not supported\n");
1641 				return false;
1642 			}
1643 			continue;
1644 		}
1645 
1646 		/* GFXv10 and later GPUs do not support shader preemption
1647 		 * during page faults. This can lead to poor QoS for queue
1648 		 * management and memory-manager-related preemptions or
1649 		 * even deadlocks.
1650 		 */
1651 		if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1) &&
1652 		    KFD_GC_VERSION(dev) < IP_VERSION(12, 1, 0))
1653 			return false;
1654 
1655 		if (dev->kfd->noretry)
1656 			return false;
1657 	}
1658 
1659 	return true;
1660 }
1661 
1662 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1663 				     bool enabled)
1664 {
1665 	if (!iosys_map_is_null(&qpd->cwsr_map)) {
1666 		iosys_map_wr(&qpd->cwsr_map,
1667 			     KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
1668 			     uint64_t, enabled);
1669 	}
1670 }
1671 
1672 /*
1673  * On return the kfd_process is fully operational and will be freed when the
1674  * mm is released
1675  */
1676 struct kfd_process *create_process(const struct task_struct *thread, bool primary)
1677 {
1678 	struct kfd_process *process;
1679 	struct mmu_notifier *mn;
1680 	int err = -ENOMEM;
1681 
1682 	process = kzalloc_obj(*process);
1683 	if (!process)
1684 		goto err_alloc_process;
1685 
1686 	kref_init(&process->ref);
1687 	mutex_init(&process->mutex);
1688 	process->mm = thread->mm;
1689 	process->lead_thread = thread->group_leader;
1690 	process->n_pdds = 0;
1691 	process->queues_paused = false;
1692 
1693 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1694 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1695 	INIT_DELAYED_WORK(&process->signal_work, kfd_signal_sigbus_delayed_fn);
1696 	process->last_restore_timestamp = get_jiffies_64();
1697 	err = kfd_event_init_process(process);
1698 	if (err)
1699 		goto err_event_init;
1700 	process->is_32bit_user_mode = in_compat_syscall();
1701 	process->debug_trap_enabled = false;
1702 	process->debugger_process = NULL;
1703 	process->exception_enable_mask = 0;
1704 	atomic_set(&process->debugged_process_count, 0);
1705 	sema_init(&process->runtime_enable_sema, 0);
1706 
1707 	err = pqm_init(&process->pqm, process);
1708 	if (err != 0)
1709 		goto err_process_pqm_init;
1710 
1711 	/* init process apertures*/
1712 	err = kfd_init_apertures(process);
1713 	if (err != 0)
1714 		goto err_init_apertures;
1715 
1716 	/* Check XNACK support after PDDs are created in kfd_init_apertures */
1717 	process->xnack_enabled = kfd_process_xnack_mode(process, false);
1718 
1719 	err = svm_range_list_init(process);
1720 	if (err)
1721 		goto err_init_svm_range_list;
1722 
1723 	/* alloc_notifier needs to find the process in the hash table */
1724 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1725 			(uintptr_t)process->mm);
1726 
1727 	/* Avoid free_notifier to start kfd_process_wq_release if
1728 	 * mmu_notifier_get failed because of pending signal.
1729 	 */
1730 	kref_get(&process->ref);
1731 
1732 	/* MMU notifier registration must be the last call that can fail
1733 	 * because after this point we cannot unwind the process creation.
1734 	 * After this point, mmu_notifier_put will trigger the cleanup by
1735 	 * dropping the last process reference in the free_notifier.
1736 	 */
1737 	if (primary) {
1738 		process->context_id = KFD_CONTEXT_ID_PRIMARY;
1739 		mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1740 		if (IS_ERR(mn)) {
1741 			err = PTR_ERR(mn);
1742 			goto err_register_notifier;
1743 		}
1744 		BUG_ON(mn != &process->mmu_notifier);
1745 		ida_init(&process->id_table);
1746 	}
1747 
1748 	err = kfd_process_alloc_id(process);
1749 	if (err) {
1750 		pr_err("Creating kfd process: failed to alloc an id\n");
1751 		goto err_alloc_id;
1752 	}
1753 
1754 	kfd_unref_process(process);
1755 	get_task_struct(process->lead_thread);
1756 
1757 	INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1758 
1759 	return process;
1760 
1761 err_alloc_id:
1762 	kfd_process_free_id(process);
1763 err_register_notifier:
1764 	hash_del_rcu(&process->kfd_processes);
1765 	svm_range_list_fini(process);
1766 err_init_svm_range_list:
1767 	kfd_process_free_outstanding_kfd_bos(process);
1768 	kfd_process_destroy_pdds(process);
1769 err_init_apertures:
1770 	pqm_uninit(&process->pqm);
1771 err_process_pqm_init:
1772 	kfd_event_free_process(process);
1773 err_event_init:
1774 	mutex_destroy(&process->mutex);
1775 	kfree(process);
1776 err_alloc_process:
1777 	return ERR_PTR(err);
1778 }
1779 
1780 struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1781 							struct kfd_process *p)
1782 {
1783 	int i;
1784 
1785 	for (i = 0; i < p->n_pdds; i++)
1786 		if (p->pdds[i]->dev == dev)
1787 			return p->pdds[i];
1788 
1789 	return NULL;
1790 }
1791 
1792 struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1793 							struct kfd_process *p)
1794 {
1795 	struct kfd_process_device *pdd = NULL;
1796 
1797 	if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1798 		return NULL;
1799 	pdd = kzalloc_obj(*pdd);
1800 	if (!pdd)
1801 		return NULL;
1802 
1803 	pdd->dev = dev;
1804 	INIT_LIST_HEAD(&pdd->qpd.queues_list);
1805 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1806 	mutex_init(&pdd->doorbell_mutex);
1807 	pdd->qpd.dqm = dev->dqm;
1808 	pdd->qpd.pqm = &p->pqm;
1809 	pdd->qpd.evicted = 0;
1810 	pdd->qpd.mapped_gws_queue = false;
1811 	pdd->process = p;
1812 	pdd->bound = PDD_UNBOUND;
1813 	pdd->already_dequeued = false;
1814 	pdd->runtime_inuse = false;
1815 	atomic64_set(&pdd->vram_usage, 0);
1816 	pdd->sdma_past_activity_counter = 0;
1817 	pdd->user_gpu_id = dev->id;
1818 	atomic64_set(&pdd->evict_duration_counter, 0);
1819 
1820 	p->pdds[p->n_pdds++] = pdd;
1821 	if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1822 		pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1823 							pdd->dev->adev,
1824 							false,
1825 							0);
1826 
1827 	/* Init idr used for memory handle translation */
1828 	idr_init(&pdd->alloc_idr);
1829 
1830 	atomic_inc(&dev->kfd->kfd_processes_count);
1831 
1832 	return pdd;
1833 }
1834 
1835 /**
1836  * kfd_process_device_init_vm - Initialize a VM for a process-device
1837  *
1838  * @pdd: The process-device
1839  * @drm_file: Optional pointer to a DRM file descriptor
1840  *
1841  * If @drm_file is specified, it will be used to acquire the VM from
1842  * that file descriptor. If successful, the @pdd takes ownership of
1843  * the file descriptor.
1844  *
1845  * If @drm_file is NULL, a new VM is created.
1846  *
1847  * Returns 0 on success, -errno on failure.
1848  */
1849 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1850 			       struct file *drm_file)
1851 {
1852 	struct amdgpu_fpriv *drv_priv;
1853 	struct amdgpu_vm *avm;
1854 	struct kfd_process *p;
1855 	struct dma_fence *ef;
1856 	struct kfd_node *dev;
1857 	int ret;
1858 
1859 	if (pdd->drm_priv)
1860 		return -EBUSY;
1861 
1862 	ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1863 	if (ret)
1864 		return ret;
1865 	avm = &drv_priv->vm;
1866 
1867 	p = pdd->process;
1868 	dev = pdd->dev;
1869 
1870 	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1871 						     &p->kgd_process_info,
1872 						     p->ef ? NULL : &ef);
1873 	if (ret) {
1874 		dev_err(dev->adev->dev, "Failed to create process VM object\n");
1875 		return ret;
1876 	}
1877 
1878 	if (!p->ef)
1879 		RCU_INIT_POINTER(p->ef, ef);
1880 
1881 	pdd->drm_priv = drm_file->private_data;
1882 
1883 	ret = kfd_process_device_reserve_ib_mem(pdd);
1884 	if (ret)
1885 		goto err_reserve_ib_mem;
1886 	ret = kfd_process_device_init_cwsr_dgpu(pdd);
1887 	if (ret)
1888 		goto err_init_cwsr;
1889 
1890 	if (unlikely(!avm->pasid)) {
1891 		dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated",
1892 				 avm);
1893 		ret = -EINVAL;
1894 		goto err_get_pasid;
1895 	}
1896 
1897 	pdd->pasid = avm->pasid;
1898 	pdd->drm_file = drm_file;
1899 
1900 	kfd_smi_event_process(pdd, true);
1901 
1902 	return 0;
1903 
1904 err_get_pasid:
1905 	kfd_process_device_destroy_cwsr_dgpu(pdd);
1906 err_init_cwsr:
1907 	kfd_process_device_destroy_ib_mem(pdd);
1908 err_reserve_ib_mem:
1909 	pdd->drm_priv = NULL;
1910 	amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1911 
1912 	return ret;
1913 }
1914 
1915 /*
1916  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1917  * to the device.
1918  * Unbinding occurs when the process dies or the device is removed.
1919  *
1920  * Assumes that the process lock is held.
1921  */
1922 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1923 							struct kfd_process *p)
1924 {
1925 	struct kfd_process_device *pdd;
1926 	int err;
1927 
1928 	pdd = kfd_get_process_device_data(dev, p);
1929 	if (!pdd) {
1930 		dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1931 		return ERR_PTR(-ENOMEM);
1932 	}
1933 
1934 	if (!pdd->drm_priv)
1935 		return ERR_PTR(-ENODEV);
1936 
1937 	/*
1938 	 * signal runtime-pm system to auto resume and prevent
1939 	 * further runtime suspend once device pdd is created until
1940 	 * pdd is destroyed.
1941 	 */
1942 	if (!pdd->runtime_inuse) {
1943 		err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1944 		if (err < 0) {
1945 			pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1946 			return ERR_PTR(err);
1947 		}
1948 	}
1949 
1950 	/*
1951 	 * make sure that runtime_usage counter is incremented just once
1952 	 * per pdd
1953 	 */
1954 	pdd->runtime_inuse = true;
1955 
1956 	return pdd;
1957 }
1958 
1959 /* Create specific handle mapped to mem from process local memory idr
1960  * Assumes that the process lock is held.
1961  */
1962 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1963 					void *mem)
1964 {
1965 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1966 }
1967 
1968 /* Translate specific handle from process local memory idr
1969  * Assumes that the process lock is held.
1970  */
1971 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1972 					int handle)
1973 {
1974 	if (handle < 0)
1975 		return NULL;
1976 
1977 	return idr_find(&pdd->alloc_idr, handle);
1978 }
1979 
1980 /* Remove specific handle from process local memory idr
1981  * Assumes that the process lock is held.
1982  */
1983 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1984 					int handle)
1985 {
1986 	if (handle >= 0)
1987 		idr_remove(&pdd->alloc_idr, handle);
1988 }
1989 
1990 static struct kfd_process_device *kfd_lookup_process_device_by_pasid(u32 pasid)
1991 {
1992 	struct kfd_process_device *ret_p = NULL;
1993 	struct kfd_process *p;
1994 	unsigned int temp;
1995 	int i;
1996 
1997 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1998 		for (i = 0; i < p->n_pdds; i++) {
1999 			if (p->pdds[i]->pasid == pasid) {
2000 				ret_p = p->pdds[i];
2001 				break;
2002 			}
2003 		}
2004 		if (ret_p)
2005 			break;
2006 	}
2007 	return ret_p;
2008 }
2009 
2010 /* This increments the process->ref counter. */
2011 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
2012 						struct kfd_process_device **pdd)
2013 {
2014 	struct kfd_process_device *ret_p;
2015 
2016 	int idx = srcu_read_lock(&kfd_processes_srcu);
2017 
2018 	ret_p = kfd_lookup_process_device_by_pasid(pasid);
2019 	if (ret_p) {
2020 		if (pdd)
2021 			*pdd = ret_p;
2022 		kref_get(&ret_p->process->ref);
2023 
2024 		srcu_read_unlock(&kfd_processes_srcu, idx);
2025 		return ret_p->process;
2026 	}
2027 
2028 	srcu_read_unlock(&kfd_processes_srcu, idx);
2029 
2030 	if (pdd)
2031 		*pdd = NULL;
2032 
2033 	return NULL;
2034 }
2035 
2036 /* This increments the process->ref counter. */
2037 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
2038 {
2039 	struct kfd_process *p;
2040 
2041 	int idx = srcu_read_lock(&kfd_processes_srcu);
2042 
2043 	p = find_process_by_mm(mm);
2044 	if (p)
2045 		kref_get(&p->ref);
2046 
2047 	srcu_read_unlock(&kfd_processes_srcu, idx);
2048 
2049 	return p;
2050 }
2051 
2052 /* This increments the process->ref counter. */
2053 struct kfd_process *kfd_lookup_process_by_id(const struct mm_struct *mm, u16 id)
2054 {
2055 	struct kfd_process *p, *ret_p = NULL;
2056 	unsigned int temp;
2057 
2058 	int idx = srcu_read_lock(&kfd_processes_srcu);
2059 
2060 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2061 		if (p->mm == mm && p->context_id == id) {
2062 			kref_get(&p->ref);
2063 			ret_p = p;
2064 			break;
2065 		}
2066 	}
2067 
2068 	srcu_read_unlock(&kfd_processes_srcu, idx);
2069 
2070 	return ret_p;
2071 }
2072 
2073 /* kfd_process_evict_queues - Evict all user queues of a process
2074  *
2075  * Eviction is reference-counted per process-device. This means multiple
2076  * evictions from different sources can be nested safely.
2077  */
2078 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
2079 {
2080 	int r = 0;
2081 	int i;
2082 	unsigned int n_evicted = 0;
2083 
2084 	for (i = 0; i < p->n_pdds; i++) {
2085 		struct kfd_process_device *pdd = p->pdds[i];
2086 		struct device *dev = pdd->dev->adev->dev;
2087 
2088 		kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread,
2089 					     trigger);
2090 
2091 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
2092 							    &pdd->qpd);
2093 		/* evict return -EIO if HWS is hang or asic is resetting, in this case
2094 		 * we would like to set all the queues to be in evicted state to prevent
2095 		 * them been add back since they actually not be saved right now.
2096 		 */
2097 		if (r && r != -EIO) {
2098 			dev_err(dev, "Failed to evict process queues\n");
2099 			goto fail;
2100 		}
2101 		n_evicted++;
2102 
2103 		pdd->dev->dqm->is_hws_hang = false;
2104 	}
2105 
2106 	return r;
2107 
2108 fail:
2109 	/* To keep state consistent, roll back partial eviction by
2110 	 * restoring queues
2111 	 */
2112 	for (i = 0; i < p->n_pdds; i++) {
2113 		struct kfd_process_device *pdd = p->pdds[i];
2114 
2115 		if (n_evicted == 0)
2116 			break;
2117 
2118 		kfd_smi_event_queue_restore(pdd->dev, p->lead_thread);
2119 
2120 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
2121 							      &pdd->qpd))
2122 			dev_err(pdd->dev->adev->dev,
2123 				"Failed to restore queues\n");
2124 
2125 		n_evicted--;
2126 	}
2127 
2128 	return r;
2129 }
2130 
2131 /* kfd_process_restore_queues - Restore all user queues of a process */
2132 int kfd_process_restore_queues(struct kfd_process *p)
2133 {
2134 	int r, ret = 0;
2135 	int i;
2136 
2137 	for (i = 0; i < p->n_pdds; i++) {
2138 		struct kfd_process_device *pdd = p->pdds[i];
2139 		struct device *dev = pdd->dev->adev->dev;
2140 
2141 		kfd_smi_event_queue_restore(pdd->dev, p->lead_thread);
2142 
2143 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
2144 							      &pdd->qpd);
2145 		if (r) {
2146 			dev_err(dev, "Failed to restore process queues\n");
2147 			if (!ret)
2148 				ret = r;
2149 		}
2150 	}
2151 
2152 	return ret;
2153 }
2154 
2155 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
2156 {
2157 	int i;
2158 
2159 	for (i = 0; i < p->n_pdds; i++)
2160 		if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
2161 			return i;
2162 	return -EINVAL;
2163 }
2164 
2165 int
2166 kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
2167 			    uint32_t *gpuid, uint32_t *gpuidx)
2168 {
2169 	int i;
2170 
2171 	for (i = 0; i < p->n_pdds; i++)
2172 		if (p->pdds[i] && p->pdds[i]->dev == node) {
2173 			*gpuid = p->pdds[i]->user_gpu_id;
2174 			*gpuidx = i;
2175 			return 0;
2176 		}
2177 	return -EINVAL;
2178 }
2179 
2180 static bool signal_eviction_fence(struct kfd_process *p)
2181 {
2182 	struct dma_fence *ef;
2183 	bool ret;
2184 
2185 	rcu_read_lock();
2186 	ef = dma_fence_get_rcu_safe(&p->ef);
2187 	rcu_read_unlock();
2188 	if (!ef)
2189 		return true;
2190 
2191 	ret = dma_fence_check_and_signal(ef);
2192 	dma_fence_put(ef);
2193 
2194 	return ret;
2195 }
2196 
2197 static void evict_process_worker(struct work_struct *work)
2198 {
2199 	int ret;
2200 	struct kfd_process *p;
2201 	struct delayed_work *dwork;
2202 
2203 	dwork = to_delayed_work(work);
2204 
2205 	/* Process termination destroys this worker thread. So during the
2206 	 * lifetime of this thread, kfd_process p will be valid
2207 	 */
2208 	p = container_of(dwork, struct kfd_process, eviction_work);
2209 
2210 	pr_debug("Started evicting process pid %d\n", p->lead_thread->pid);
2211 	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
2212 	if (!ret) {
2213 		/* If another thread already signaled the eviction fence,
2214 		 * they are responsible stopping the queues and scheduling
2215 		 * the restore work.
2216 		 */
2217 		if (signal_eviction_fence(p) ||
2218 		    mod_delayed_work(kfd_restore_wq, &p->restore_work,
2219 				     msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2220 			kfd_process_restore_queues(p);
2221 
2222 		pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid);
2223 	} else
2224 		pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid);
2225 }
2226 
2227 static int restore_process_helper(struct kfd_process *p)
2228 {
2229 	int ret = 0;
2230 
2231 	/* VMs may not have been acquired yet during debugging. */
2232 	if (p->kgd_process_info) {
2233 		ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
2234 			p->kgd_process_info, &p->ef);
2235 		if (ret)
2236 			return ret;
2237 	}
2238 
2239 	ret = kfd_process_restore_queues(p);
2240 	if (!ret)
2241 		pr_debug("Finished restoring process pid %d\n",
2242 			p->lead_thread->pid);
2243 	else
2244 		pr_err("Failed to restore queues of process pid %d\n",
2245 		      p->lead_thread->pid);
2246 
2247 	return ret;
2248 }
2249 
2250 static void restore_process_worker(struct work_struct *work)
2251 {
2252 	struct delayed_work *dwork;
2253 	struct kfd_process *p;
2254 	int ret = 0;
2255 
2256 	dwork = to_delayed_work(work);
2257 
2258 	/* Process termination destroys this worker thread. So during the
2259 	 * lifetime of this thread, kfd_process p will be valid
2260 	 */
2261 	p = container_of(dwork, struct kfd_process, restore_work);
2262 	pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid);
2263 
2264 	/* Setting last_restore_timestamp before successful restoration.
2265 	 * Otherwise this would have to be set by KGD (restore_process_bos)
2266 	 * before KFD BOs are unreserved. If not, the process can be evicted
2267 	 * again before the timestamp is set.
2268 	 * If restore fails, the timestamp will be set again in the next
2269 	 * attempt. This would mean that the minimum GPU quanta would be
2270 	 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2271 	 * functions)
2272 	 */
2273 
2274 	p->last_restore_timestamp = get_jiffies_64();
2275 
2276 	ret = restore_process_helper(p);
2277 	if (ret) {
2278 		pr_debug("Failed to restore BOs of process pid %d, retry after %d ms\n",
2279 			 p->lead_thread->pid, PROCESS_BACK_OFF_TIME_MS);
2280 		if (mod_delayed_work(kfd_restore_wq, &p->restore_work,
2281 				     msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2282 			kfd_process_restore_queues(p);
2283 	}
2284 }
2285 
2286 void kfd_suspend_all_processes(void)
2287 {
2288 	struct kfd_process *p;
2289 	unsigned int temp;
2290 	int idx = srcu_read_lock(&kfd_processes_srcu);
2291 
2292 	WARN(debug_evictions, "Evicting all processes");
2293 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2294 		if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2295 			pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid);
2296 		signal_eviction_fence(p);
2297 	}
2298 	srcu_read_unlock(&kfd_processes_srcu, idx);
2299 }
2300 
2301 int kfd_resume_all_processes(void)
2302 {
2303 	struct kfd_process *p;
2304 	unsigned int temp;
2305 	int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2306 
2307 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2308 		if (restore_process_helper(p)) {
2309 			pr_err("Restore process pid %d failed during resume\n",
2310 			      p->lead_thread->pid);
2311 			ret = -EFAULT;
2312 		}
2313 	}
2314 	srcu_read_unlock(&kfd_processes_srcu, idx);
2315 	return ret;
2316 }
2317 
2318 /* assumes caller holds process lock. */
2319 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2320 {
2321 	uint32_t irq_drain_fence[8];
2322 	uint8_t node_id = 0;
2323 	int r = 0;
2324 
2325 	if (!KFD_IS_SOC15(pdd->dev))
2326 		return 0;
2327 
2328 	pdd->process->irq_drain_is_open = true;
2329 
2330 	memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2331 	irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2332 							KFD_IRQ_FENCE_CLIENTID;
2333 	irq_drain_fence[3] = pdd->pasid;
2334 
2335 	/*
2336 	 * For GFX 9.4.3/9.5.0, send the NodeId also in IH cookie DW[3]
2337 	 */
2338 	if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
2339 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||
2340 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0) ||
2341 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(12, 1, 0)) {
2342 		node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2343 		irq_drain_fence[3] |= node_id << 16;
2344 	}
2345 
2346 	/* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2347 	if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2348 						     irq_drain_fence)) {
2349 		pdd->process->irq_drain_is_open = false;
2350 		return 0;
2351 	}
2352 
2353 	r = wait_event_interruptible(pdd->process->wait_irq_drain,
2354 				     !READ_ONCE(pdd->process->irq_drain_is_open));
2355 	if (r)
2356 		pdd->process->irq_drain_is_open = false;
2357 
2358 	return r;
2359 }
2360 
2361 void kfd_process_close_interrupt_drain(unsigned int pasid)
2362 {
2363 	struct kfd_process *p;
2364 
2365 	p = kfd_lookup_process_by_pasid(pasid, NULL);
2366 
2367 	if (!p)
2368 		return;
2369 
2370 	WRITE_ONCE(p->irq_drain_is_open, false);
2371 	wake_up_all(&p->wait_irq_drain);
2372 	kfd_unref_process(p);
2373 }
2374 
2375 struct send_exception_work_handler_workarea {
2376 	struct work_struct work;
2377 	struct kfd_process *p;
2378 	unsigned int queue_id;
2379 	uint64_t error_reason;
2380 };
2381 
2382 static void send_exception_work_handler(struct work_struct *work)
2383 {
2384 	struct send_exception_work_handler_workarea *workarea;
2385 	struct kfd_process *p;
2386 	struct queue *q;
2387 	struct mm_struct *mm;
2388 	struct kfd_context_save_area_header __user *csa_header;
2389 	uint64_t __user *err_payload_ptr;
2390 	uint64_t cur_err;
2391 	uint32_t ev_id;
2392 
2393 	workarea = container_of(work,
2394 				struct send_exception_work_handler_workarea,
2395 				work);
2396 	p = workarea->p;
2397 
2398 	mm = get_task_mm(p->lead_thread);
2399 
2400 	if (!mm)
2401 		return;
2402 
2403 	kthread_use_mm(mm);
2404 
2405 	q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2406 
2407 	if (!q)
2408 		goto out;
2409 
2410 	csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2411 
2412 	get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2413 	get_user(cur_err, err_payload_ptr);
2414 	cur_err |= workarea->error_reason;
2415 	put_user(cur_err, err_payload_ptr);
2416 	get_user(ev_id, &csa_header->err_event_id);
2417 
2418 	kfd_set_event(p, ev_id);
2419 
2420 out:
2421 	kthread_unuse_mm(mm);
2422 	mmput(mm);
2423 }
2424 
2425 int kfd_send_exception_to_runtime(struct kfd_process *p,
2426 			unsigned int queue_id,
2427 			uint64_t error_reason)
2428 {
2429 	struct send_exception_work_handler_workarea worker;
2430 
2431 	INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2432 
2433 	worker.p = p;
2434 	worker.queue_id = queue_id;
2435 	worker.error_reason = error_reason;
2436 
2437 	schedule_work(&worker.work);
2438 	flush_work(&worker.work);
2439 	destroy_work_on_stack(&worker.work);
2440 
2441 	return 0;
2442 }
2443 
2444 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2445 {
2446 	int i;
2447 
2448 	if (gpu_id) {
2449 		for (i = 0; i < p->n_pdds; i++) {
2450 			struct kfd_process_device *pdd = p->pdds[i];
2451 
2452 			if (pdd->user_gpu_id == gpu_id)
2453 				return pdd;
2454 		}
2455 	}
2456 	return NULL;
2457 }
2458 
2459 int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2460 {
2461 	int i;
2462 
2463 	if (!actual_gpu_id)
2464 		return 0;
2465 
2466 	for (i = 0; i < p->n_pdds; i++) {
2467 		struct kfd_process_device *pdd = p->pdds[i];
2468 
2469 		if (pdd->dev->id == actual_gpu_id)
2470 			return pdd->user_gpu_id;
2471 	}
2472 	return -EINVAL;
2473 }
2474 
2475 #if defined(CONFIG_DEBUG_FS)
2476 
2477 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2478 {
2479 	struct kfd_process *p;
2480 	unsigned int temp;
2481 	int r = 0;
2482 
2483 	int idx = srcu_read_lock(&kfd_processes_srcu);
2484 
2485 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2486 		seq_printf(m, "Process %d PASID %d:\n",
2487 			   p->lead_thread->tgid, p->lead_thread->pid);
2488 
2489 		mutex_lock(&p->mutex);
2490 		r = pqm_debugfs_mqds(m, &p->pqm);
2491 		mutex_unlock(&p->mutex);
2492 
2493 		if (r)
2494 			break;
2495 	}
2496 
2497 	srcu_read_unlock(&kfd_processes_srcu, idx);
2498 
2499 	return r;
2500 }
2501 
2502 #endif
2503