xref: /linux/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 /*
2  * Copyright 2023 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 #include "priv.h"
23 
24 #include <core/pci.h>
25 #include <subdev/timer.h>
26 #include <subdev/vfn.h>
27 #include <engine/fifo/chan.h>
28 #include <engine/sec2.h>
29 
30 #include <nvfw/fw.h>
31 
32 #include <nvrm/nvtypes.h>
33 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h>
34 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h>
35 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h>
36 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h>
37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h>
38 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
39 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
40 #include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h>
41 #include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h>
42 #include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h>
43 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h>
44 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h>
45 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h>
46 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h>
47 #include <nvrm/535.113.01/nvidia/generated/g_allclasses.h>
48 #include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h>
49 #include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h>
50 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h>
51 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h>
52 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>
53 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h>
54 #include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h>
55 
56 #include <linux/acpi.h>
57 #include <linux/ctype.h>
58 #include <linux/parser.h>
59 
60 #define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE
61 #define GSP_MSG_MAX_SIZE GSP_PAGE_MIN_SIZE * 16
62 
63 struct r535_gsp_msg {
64 	u8 auth_tag_buffer[16];
65 	u8 aad_buffer[16];
66 	u32 checksum;
67 	u32 sequence;
68 	u32 elem_count;
69 	u32 pad;
70 	u8  data[];
71 };
72 
73 #define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data)
74 
75 static int
r535_rpc_status_to_errno(uint32_t rpc_status)76 r535_rpc_status_to_errno(uint32_t rpc_status)
77 {
78 	switch (rpc_status) {
79 	case 0x55: /* NV_ERR_NOT_READY */
80 	case 0x66: /* NV_ERR_TIMEOUT_RETRY */
81 		return -EAGAIN;
82 	case 0x51: /* NV_ERR_NO_MEMORY */
83 		return -ENOMEM;
84 	default:
85 		return -EINVAL;
86 	}
87 }
88 
89 static void *
r535_gsp_msgq_wait(struct nvkm_gsp * gsp,u32 repc,u32 * prepc,int * ptime)90 r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime)
91 {
92 	struct r535_gsp_msg *mqe;
93 	u32 size, rptr = *gsp->msgq.rptr;
94 	int used;
95 	u8 *msg;
96 	u32 len;
97 
98 	size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + repc, GSP_PAGE_SIZE);
99 	if (WARN_ON(!size || size >= gsp->msgq.cnt))
100 		return ERR_PTR(-EINVAL);
101 
102 	do {
103 		u32 wptr = *gsp->msgq.wptr;
104 
105 		used = wptr + gsp->msgq.cnt - rptr;
106 		if (used >= gsp->msgq.cnt)
107 			used -= gsp->msgq.cnt;
108 		if (used >= size)
109 			break;
110 
111 		usleep_range(1, 2);
112 	} while (--(*ptime));
113 
114 	if (WARN_ON(!*ptime))
115 		return ERR_PTR(-ETIMEDOUT);
116 
117 	mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + rptr * 0x1000);
118 
119 	if (prepc) {
120 		*prepc = (used * GSP_PAGE_SIZE) - sizeof(*mqe);
121 		return mqe->data;
122 	}
123 
124 	msg = kvmalloc(repc, GFP_KERNEL);
125 	if (!msg)
126 		return ERR_PTR(-ENOMEM);
127 
128 	len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe);
129 	len = min_t(u32, repc, len);
130 	memcpy(msg, mqe->data, len);
131 
132 	rptr += DIV_ROUND_UP(len, GSP_PAGE_SIZE);
133 	if (rptr == gsp->msgq.cnt)
134 		rptr = 0;
135 
136 	repc -= len;
137 
138 	if (repc) {
139 		mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000);
140 		memcpy(msg + len, mqe, repc);
141 
142 		rptr += DIV_ROUND_UP(repc, GSP_PAGE_SIZE);
143 	}
144 
145 	mb();
146 	(*gsp->msgq.rptr) = rptr;
147 	return msg;
148 }
149 
150 static void *
r535_gsp_msgq_recv(struct nvkm_gsp * gsp,u32 repc,int * ptime)151 r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime)
152 {
153 	return r535_gsp_msgq_wait(gsp, repc, NULL, ptime);
154 }
155 
156 static int
r535_gsp_cmdq_push(struct nvkm_gsp * gsp,void * argv)157 r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv)
158 {
159 	struct r535_gsp_msg *cmd = container_of(argv, typeof(*cmd), data);
160 	struct r535_gsp_msg *cqe;
161 	u32 argc = cmd->checksum;
162 	u64 *ptr = (void *)cmd;
163 	u64 *end;
164 	u64 csum = 0;
165 	int free, time = 1000000;
166 	u32 wptr, size;
167 	u32 off = 0;
168 
169 	argc = ALIGN(GSP_MSG_HDR_SIZE + argc, GSP_PAGE_SIZE);
170 
171 	end = (u64 *)((char *)ptr + argc);
172 	cmd->pad = 0;
173 	cmd->checksum = 0;
174 	cmd->sequence = gsp->cmdq.seq++;
175 	cmd->elem_count = DIV_ROUND_UP(argc, 0x1000);
176 
177 	while (ptr < end)
178 		csum ^= *ptr++;
179 
180 	cmd->checksum = upper_32_bits(csum) ^ lower_32_bits(csum);
181 
182 	wptr = *gsp->cmdq.wptr;
183 	do {
184 		do {
185 			free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1;
186 			if (free >= gsp->cmdq.cnt)
187 				free -= gsp->cmdq.cnt;
188 			if (free >= 1)
189 				break;
190 
191 			usleep_range(1, 2);
192 		} while(--time);
193 
194 		if (WARN_ON(!time)) {
195 			kvfree(cmd);
196 			return -ETIMEDOUT;
197 		}
198 
199 		cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000);
200 		size = min_t(u32, argc, (gsp->cmdq.cnt - wptr) * GSP_PAGE_SIZE);
201 		memcpy(cqe, (u8 *)cmd + off, size);
202 
203 		wptr += DIV_ROUND_UP(size, 0x1000);
204 		if (wptr == gsp->cmdq.cnt)
205 			wptr = 0;
206 
207 		off  += size;
208 		argc -= size;
209 	} while(argc);
210 
211 	nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr);
212 	wmb();
213 	(*gsp->cmdq.wptr) = wptr;
214 	mb();
215 
216 	nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000);
217 
218 	kvfree(cmd);
219 	return 0;
220 }
221 
222 static void *
r535_gsp_cmdq_get(struct nvkm_gsp * gsp,u32 argc)223 r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc)
224 {
225 	struct r535_gsp_msg *cmd;
226 	u32 size = GSP_MSG_HDR_SIZE + argc;
227 
228 	size = ALIGN(size, GSP_MSG_MIN_SIZE);
229 	cmd = kvzalloc(size, GFP_KERNEL);
230 	if (!cmd)
231 		return ERR_PTR(-ENOMEM);
232 
233 	cmd->checksum = argc;
234 	return cmd->data;
235 }
236 
237 struct nvfw_gsp_rpc {
238 	u32 header_version;
239 	u32 signature;
240 	u32 length;
241 	u32 function;
242 	u32 rpc_result;
243 	u32 rpc_result_private;
244 	u32 sequence;
245 	union {
246 		u32 spare;
247 		u32 cpuRmGfid;
248 	};
249 	u8  data[];
250 };
251 
252 static void
r535_gsp_msg_done(struct nvkm_gsp * gsp,struct nvfw_gsp_rpc * msg)253 r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg)
254 {
255 	kvfree(msg);
256 }
257 
258 static void
r535_gsp_msg_dump(struct nvkm_gsp * gsp,struct nvfw_gsp_rpc * msg,int lvl)259 r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl)
260 {
261 	if (gsp->subdev.debug >= lvl) {
262 		nvkm_printk__(&gsp->subdev, lvl, info,
263 			      "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n",
264 			      msg->function, msg->length, msg->length - sizeof(*msg),
265 			      msg->rpc_result, msg->rpc_result_private);
266 		print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1,
267 			       msg->data, msg->length - sizeof(*msg), true);
268 	}
269 }
270 
271 static struct nvfw_gsp_rpc *
r535_gsp_msg_recv(struct nvkm_gsp * gsp,int fn,u32 repc)272 r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc)
273 {
274 	struct nvkm_subdev *subdev = &gsp->subdev;
275 	struct nvfw_gsp_rpc *msg;
276 	int time = 4000000, i;
277 	u32 size;
278 
279 retry:
280 	msg = r535_gsp_msgq_wait(gsp, sizeof(*msg), &size, &time);
281 	if (IS_ERR_OR_NULL(msg))
282 		return msg;
283 
284 	msg = r535_gsp_msgq_recv(gsp, msg->length, &time);
285 	if (IS_ERR_OR_NULL(msg))
286 		return msg;
287 
288 	if (msg->rpc_result) {
289 		r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
290 		r535_gsp_msg_done(gsp, msg);
291 		return ERR_PTR(-EINVAL);
292 	}
293 
294 	r535_gsp_msg_dump(gsp, msg, NV_DBG_TRACE);
295 
296 	if (fn && msg->function == fn) {
297 		if (repc) {
298 			if (msg->length < sizeof(*msg) + repc) {
299 				nvkm_error(subdev, "msg len %d < %zd\n",
300 					   msg->length, sizeof(*msg) + repc);
301 				r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
302 				r535_gsp_msg_done(gsp, msg);
303 				return ERR_PTR(-EIO);
304 			}
305 
306 			return msg;
307 		}
308 
309 		r535_gsp_msg_done(gsp, msg);
310 		return NULL;
311 	}
312 
313 	for (i = 0; i < gsp->msgq.ntfy_nr; i++) {
314 		struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i];
315 
316 		if (ntfy->fn == msg->function) {
317 			if (ntfy->func)
318 				ntfy->func(ntfy->priv, ntfy->fn, msg->data, msg->length - sizeof(*msg));
319 			break;
320 		}
321 	}
322 
323 	if (i == gsp->msgq.ntfy_nr)
324 		r535_gsp_msg_dump(gsp, msg, NV_DBG_WARN);
325 
326 	r535_gsp_msg_done(gsp, msg);
327 	if (fn)
328 		goto retry;
329 
330 	if (*gsp->msgq.rptr != *gsp->msgq.wptr)
331 		goto retry;
332 
333 	return NULL;
334 }
335 
336 static int
r535_gsp_msg_ntfy_add(struct nvkm_gsp * gsp,u32 fn,nvkm_gsp_msg_ntfy_func func,void * priv)337 r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv)
338 {
339 	int ret = 0;
340 
341 	mutex_lock(&gsp->msgq.mutex);
342 	if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) {
343 		ret = -ENOSPC;
344 	} else {
345 		gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn;
346 		gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func;
347 		gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv;
348 		gsp->msgq.ntfy_nr++;
349 	}
350 	mutex_unlock(&gsp->msgq.mutex);
351 	return ret;
352 }
353 
354 static int
r535_gsp_rpc_poll(struct nvkm_gsp * gsp,u32 fn)355 r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn)
356 {
357 	void *repv;
358 
359 	mutex_lock(&gsp->cmdq.mutex);
360 	repv = r535_gsp_msg_recv(gsp, fn, 0);
361 	mutex_unlock(&gsp->cmdq.mutex);
362 	if (IS_ERR(repv))
363 		return PTR_ERR(repv);
364 
365 	return 0;
366 }
367 
368 static void *
r535_gsp_rpc_send(struct nvkm_gsp * gsp,void * argv,bool wait,u32 repc)369 r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
370 {
371 	struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
372 	struct nvfw_gsp_rpc *msg;
373 	u32 fn = rpc->function;
374 	void *repv = NULL;
375 	int ret;
376 
377 	if (gsp->subdev.debug >= NV_DBG_TRACE) {
378 		nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function,
379 			   rpc->length, rpc->length - sizeof(*rpc));
380 		print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1,
381 			       rpc->data, rpc->length - sizeof(*rpc), true);
382 	}
383 
384 	ret = r535_gsp_cmdq_push(gsp, rpc);
385 	if (ret)
386 		return ERR_PTR(ret);
387 
388 	if (wait) {
389 		msg = r535_gsp_msg_recv(gsp, fn, repc);
390 		if (!IS_ERR_OR_NULL(msg))
391 			repv = msg->data;
392 		else
393 			repv = msg;
394 	}
395 
396 	return repv;
397 }
398 
399 static void
r535_gsp_event_dtor(struct nvkm_gsp_event * event)400 r535_gsp_event_dtor(struct nvkm_gsp_event *event)
401 {
402 	struct nvkm_gsp_device *device = event->device;
403 	struct nvkm_gsp_client *client = device->object.client;
404 	struct nvkm_gsp *gsp = client->gsp;
405 
406 	mutex_lock(&gsp->client_id.mutex);
407 	if (event->func) {
408 		list_del(&event->head);
409 		event->func = NULL;
410 	}
411 	mutex_unlock(&gsp->client_id.mutex);
412 
413 	nvkm_gsp_rm_free(&event->object);
414 	event->device = NULL;
415 }
416 
417 static int
r535_gsp_device_event_get(struct nvkm_gsp_event * event)418 r535_gsp_device_event_get(struct nvkm_gsp_event *event)
419 {
420 	struct nvkm_gsp_device *device = event->device;
421 	NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS *ctrl;
422 
423 	ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice,
424 				    NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION, sizeof(*ctrl));
425 	if (IS_ERR(ctrl))
426 		return PTR_ERR(ctrl);
427 
428 	ctrl->event = event->id;
429 	ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT;
430 	return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl);
431 }
432 
433 static int
r535_gsp_device_event_ctor(struct nvkm_gsp_device * device,u32 handle,u32 id,nvkm_gsp_event_func func,struct nvkm_gsp_event * event)434 r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id,
435 			   nvkm_gsp_event_func func, struct nvkm_gsp_event *event)
436 {
437 	struct nvkm_gsp_client *client = device->object.client;
438 	struct nvkm_gsp *gsp = client->gsp;
439 	NV0005_ALLOC_PARAMETERS *args;
440 	int ret;
441 
442 	args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle,
443 				     NV01_EVENT_KERNEL_CALLBACK_EX, sizeof(*args),
444 				     &event->object);
445 	if (IS_ERR(args))
446 		return PTR_ERR(args);
447 
448 	args->hParentClient = client->object.handle;
449 	args->hSrcResource = 0;
450 	args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX;
451 	args->notifyIndex = NV01_EVENT_CLIENT_RM | id;
452 	args->data = NULL;
453 
454 	ret = nvkm_gsp_rm_alloc_wr(&event->object, args);
455 	if (ret)
456 		return ret;
457 
458 	event->device = device;
459 	event->id = id;
460 
461 	ret = r535_gsp_device_event_get(event);
462 	if (ret) {
463 		nvkm_gsp_event_dtor(event);
464 		return ret;
465 	}
466 
467 	mutex_lock(&gsp->client_id.mutex);
468 	event->func = func;
469 	list_add(&event->head, &client->events);
470 	mutex_unlock(&gsp->client_id.mutex);
471 	return 0;
472 }
473 
474 static void
r535_gsp_device_dtor(struct nvkm_gsp_device * device)475 r535_gsp_device_dtor(struct nvkm_gsp_device *device)
476 {
477 	nvkm_gsp_rm_free(&device->subdevice);
478 	nvkm_gsp_rm_free(&device->object);
479 }
480 
481 static int
r535_gsp_subdevice_ctor(struct nvkm_gsp_device * device)482 r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device)
483 {
484 	NV2080_ALLOC_PARAMETERS *args;
485 
486 	return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args),
487 				 &device->subdevice);
488 }
489 
490 static int
r535_gsp_device_ctor(struct nvkm_gsp_client * client,struct nvkm_gsp_device * device)491 r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device)
492 {
493 	NV0080_ALLOC_PARAMETERS *args;
494 	int ret;
495 
496 	args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args),
497 				     &device->object);
498 	if (IS_ERR(args))
499 		return PTR_ERR(args);
500 
501 	args->hClientShare = client->object.handle;
502 
503 	ret = nvkm_gsp_rm_alloc_wr(&device->object, args);
504 	if (ret)
505 		return ret;
506 
507 	ret = r535_gsp_subdevice_ctor(device);
508 	if (ret)
509 		nvkm_gsp_rm_free(&device->object);
510 
511 	return ret;
512 }
513 
514 static void
r535_gsp_client_dtor(struct nvkm_gsp_client * client)515 r535_gsp_client_dtor(struct nvkm_gsp_client *client)
516 {
517 	struct nvkm_gsp *gsp = client->gsp;
518 
519 	nvkm_gsp_rm_free(&client->object);
520 
521 	mutex_lock(&gsp->client_id.mutex);
522 	idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff);
523 	mutex_unlock(&gsp->client_id.mutex);
524 
525 	client->gsp = NULL;
526 }
527 
528 static int
r535_gsp_client_ctor(struct nvkm_gsp * gsp,struct nvkm_gsp_client * client)529 r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client)
530 {
531 	NV0000_ALLOC_PARAMETERS *args;
532 	int ret;
533 
534 	mutex_lock(&gsp->client_id.mutex);
535 	ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL);
536 	mutex_unlock(&gsp->client_id.mutex);
537 	if (ret < 0)
538 		return ret;
539 
540 	client->gsp = gsp;
541 	client->object.client = client;
542 	INIT_LIST_HEAD(&client->events);
543 
544 	args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args),
545 				     &client->object);
546 	if (IS_ERR(args)) {
547 		r535_gsp_client_dtor(client);
548 		return ret;
549 	}
550 
551 	args->hClient = client->object.handle;
552 	args->processID = ~0;
553 
554 	ret = nvkm_gsp_rm_alloc_wr(&client->object, args);
555 	if (ret) {
556 		r535_gsp_client_dtor(client);
557 		return ret;
558 	}
559 
560 	return 0;
561 }
562 
563 static int
r535_gsp_rpc_rm_free(struct nvkm_gsp_object * object)564 r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object)
565 {
566 	struct nvkm_gsp_client *client = object->client;
567 	struct nvkm_gsp *gsp = client->gsp;
568 	rpc_free_v03_00 *rpc;
569 
570 	nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n",
571 		   client->object.handle, object->handle);
572 
573 	rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc));
574 	if (WARN_ON(IS_ERR_OR_NULL(rpc)))
575 		return -EIO;
576 
577 	rpc->params.hRoot = client->object.handle;
578 	rpc->params.hObjectParent = 0;
579 	rpc->params.hObjectOld = object->handle;
580 	return nvkm_gsp_rpc_wr(gsp, rpc, true);
581 }
582 
583 static void
r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object * object,void * repv)584 r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv)
585 {
586 	rpc_gsp_rm_alloc_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
587 
588 	nvkm_gsp_rpc_done(object->client->gsp, rpc);
589 }
590 
591 static void *
r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object * object,void * argv,u32 repc)592 r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc)
593 {
594 	rpc_gsp_rm_alloc_v03_00 *rpc = container_of(argv, typeof(*rpc), params);
595 	struct nvkm_gsp *gsp = object->client->gsp;
596 	void *ret;
597 
598 	rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc) + repc);
599 	if (IS_ERR_OR_NULL(rpc))
600 		return rpc;
601 
602 	if (rpc->status) {
603 		ret = ERR_PTR(r535_rpc_status_to_errno(rpc->status));
604 		if (PTR_ERR(ret) != -EAGAIN)
605 			nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status);
606 	} else {
607 		ret = repc ? rpc->params : NULL;
608 	}
609 
610 	nvkm_gsp_rpc_done(gsp, rpc);
611 
612 	return ret;
613 }
614 
615 static void *
r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object * object,u32 oclass,u32 argc)616 r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc)
617 {
618 	struct nvkm_gsp_client *client = object->client;
619 	struct nvkm_gsp *gsp = client->gsp;
620 	rpc_gsp_rm_alloc_v03_00 *rpc;
621 
622 	nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x cls:0x%08x argc:%d\n",
623 		   client->object.handle, object->parent->handle, object->handle, oclass, argc);
624 
625 	rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, sizeof(*rpc) + argc);
626 	if (IS_ERR(rpc))
627 		return rpc;
628 
629 	rpc->hClient = client->object.handle;
630 	rpc->hParent = object->parent->handle;
631 	rpc->hObject = object->handle;
632 	rpc->hClass = oclass;
633 	rpc->status = 0;
634 	rpc->paramsSize = argc;
635 	return rpc->params;
636 }
637 
638 static void
r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object * object,void * repv)639 r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv)
640 {
641 	rpc_gsp_rm_control_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
642 
643 	if (!repv)
644 		return;
645 	nvkm_gsp_rpc_done(object->client->gsp, rpc);
646 }
647 
648 static int
r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object * object,void ** argv,u32 repc)649 r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void **argv, u32 repc)
650 {
651 	rpc_gsp_rm_control_v03_00 *rpc = container_of((*argv), typeof(*rpc), params);
652 	struct nvkm_gsp *gsp = object->client->gsp;
653 	int ret = 0;
654 
655 	rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc);
656 	if (IS_ERR_OR_NULL(rpc)) {
657 		*argv = NULL;
658 		return PTR_ERR(rpc);
659 	}
660 
661 	if (rpc->status) {
662 		ret = r535_rpc_status_to_errno(rpc->status);
663 		if (ret != -EAGAIN)
664 			nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n",
665 				   object->client->object.handle, object->handle, rpc->cmd, rpc->status);
666 	}
667 
668 	if (repc)
669 		*argv = rpc->params;
670 	else
671 		nvkm_gsp_rpc_done(gsp, rpc);
672 
673 	return ret;
674 }
675 
676 static void *
r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object * object,u32 cmd,u32 argc)677 r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc)
678 {
679 	struct nvkm_gsp_client *client = object->client;
680 	struct nvkm_gsp *gsp = client->gsp;
681 	rpc_gsp_rm_control_v03_00 *rpc;
682 
683 	nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x argc:%d\n",
684 		   client->object.handle, object->handle, cmd, argc);
685 
686 	rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, sizeof(*rpc) + argc);
687 	if (IS_ERR(rpc))
688 		return rpc;
689 
690 	rpc->hClient    = client->object.handle;
691 	rpc->hObject    = object->handle;
692 	rpc->cmd	= cmd;
693 	rpc->status     = 0;
694 	rpc->paramsSize = argc;
695 	return rpc->params;
696 }
697 
698 static void
r535_gsp_rpc_done(struct nvkm_gsp * gsp,void * repv)699 r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv)
700 {
701 	struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data);
702 
703 	r535_gsp_msg_done(gsp, rpc);
704 }
705 
706 static void *
r535_gsp_rpc_get(struct nvkm_gsp * gsp,u32 fn,u32 argc)707 r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc)
708 {
709 	struct nvfw_gsp_rpc *rpc;
710 
711 	rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + argc, sizeof(u64)));
712 	if (IS_ERR(rpc))
713 		return ERR_CAST(rpc);
714 
715 	rpc->header_version = 0x03000000;
716 	rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V';
717 	rpc->function = fn;
718 	rpc->rpc_result = 0xffffffff;
719 	rpc->rpc_result_private = 0xffffffff;
720 	rpc->length = sizeof(*rpc) + argc;
721 	return rpc->data;
722 }
723 
724 static void *
r535_gsp_rpc_push(struct nvkm_gsp * gsp,void * argv,bool wait,u32 repc)725 r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
726 {
727 	struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
728 	struct r535_gsp_msg *cmd = container_of((void *)rpc, typeof(*cmd), data);
729 	const u32 max_msg_size = (16 * 0x1000) - sizeof(struct r535_gsp_msg);
730 	const u32 max_rpc_size = max_msg_size - sizeof(*rpc);
731 	u32 rpc_size = rpc->length - sizeof(*rpc);
732 	void *repv;
733 
734 	mutex_lock(&gsp->cmdq.mutex);
735 	if (rpc_size > max_rpc_size) {
736 		const u32 fn = rpc->function;
737 
738 		/* Adjust length, and send initial RPC. */
739 		rpc->length = sizeof(*rpc) + max_rpc_size;
740 		cmd->checksum = rpc->length;
741 
742 		repv = r535_gsp_rpc_send(gsp, argv, false, 0);
743 		if (IS_ERR(repv))
744 			goto done;
745 
746 		argv += max_rpc_size;
747 		rpc_size -= max_rpc_size;
748 
749 		/* Remaining chunks sent as CONTINUATION_RECORD RPCs. */
750 		while (rpc_size) {
751 			u32 size = min(rpc_size, max_rpc_size);
752 			void *next;
753 
754 			next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size);
755 			if (IS_ERR(next)) {
756 				repv = next;
757 				goto done;
758 			}
759 
760 			memcpy(next, argv, size);
761 
762 			repv = r535_gsp_rpc_send(gsp, next, false, 0);
763 			if (IS_ERR(repv))
764 				goto done;
765 
766 			argv += size;
767 			rpc_size -= size;
768 		}
769 
770 		/* Wait for reply. */
771 		if (wait) {
772 			rpc = r535_gsp_msg_recv(gsp, fn, repc);
773 			if (!IS_ERR_OR_NULL(rpc))
774 				repv = rpc->data;
775 			else
776 				repv = rpc;
777 		} else {
778 			repv = NULL;
779 		}
780 	} else {
781 		repv = r535_gsp_rpc_send(gsp, argv, wait, repc);
782 	}
783 
784 done:
785 	mutex_unlock(&gsp->cmdq.mutex);
786 	return repv;
787 }
788 
789 const struct nvkm_gsp_rm
790 r535_gsp_rm = {
791 	.rpc_get = r535_gsp_rpc_get,
792 	.rpc_push = r535_gsp_rpc_push,
793 	.rpc_done = r535_gsp_rpc_done,
794 
795 	.rm_ctrl_get = r535_gsp_rpc_rm_ctrl_get,
796 	.rm_ctrl_push = r535_gsp_rpc_rm_ctrl_push,
797 	.rm_ctrl_done = r535_gsp_rpc_rm_ctrl_done,
798 
799 	.rm_alloc_get = r535_gsp_rpc_rm_alloc_get,
800 	.rm_alloc_push = r535_gsp_rpc_rm_alloc_push,
801 	.rm_alloc_done = r535_gsp_rpc_rm_alloc_done,
802 
803 	.rm_free = r535_gsp_rpc_rm_free,
804 
805 	.client_ctor = r535_gsp_client_ctor,
806 	.client_dtor = r535_gsp_client_dtor,
807 
808 	.device_ctor = r535_gsp_device_ctor,
809 	.device_dtor = r535_gsp_device_dtor,
810 
811 	.event_ctor = r535_gsp_device_event_ctor,
812 	.event_dtor = r535_gsp_event_dtor,
813 };
814 
815 static void
r535_gsp_msgq_work(struct work_struct * work)816 r535_gsp_msgq_work(struct work_struct *work)
817 {
818 	struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work);
819 
820 	mutex_lock(&gsp->cmdq.mutex);
821 	if (*gsp->msgq.rptr != *gsp->msgq.wptr)
822 		r535_gsp_msg_recv(gsp, 0, 0);
823 	mutex_unlock(&gsp->cmdq.mutex);
824 }
825 
826 static irqreturn_t
r535_gsp_intr(struct nvkm_inth * inth)827 r535_gsp_intr(struct nvkm_inth *inth)
828 {
829 	struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth);
830 	struct nvkm_subdev *subdev = &gsp->subdev;
831 	u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008);
832 	u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 +
833 						  gsp->falcon.func->riscv_irqmask);
834 	u32 stat = intr & inte;
835 
836 	if (!stat) {
837 		nvkm_debug(subdev, "inte %08x %08x\n", intr, inte);
838 		return IRQ_NONE;
839 	}
840 
841 	if (stat & 0x00000040) {
842 		nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040);
843 		schedule_work(&gsp->msgq.work);
844 		stat &= ~0x00000040;
845 	}
846 
847 	if (stat) {
848 		nvkm_error(subdev, "intr %08x\n", stat);
849 		nvkm_falcon_wr32(&gsp->falcon, 0x014, stat);
850 		nvkm_falcon_wr32(&gsp->falcon, 0x004, stat);
851 	}
852 
853 	nvkm_falcon_intr_retrigger(&gsp->falcon);
854 	return IRQ_HANDLED;
855 }
856 
857 static int
r535_gsp_intr_get_table(struct nvkm_gsp * gsp)858 r535_gsp_intr_get_table(struct nvkm_gsp *gsp)
859 {
860 	NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl;
861 	int ret = 0;
862 
863 	ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,
864 				    NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl));
865 	if (IS_ERR(ctrl))
866 		return PTR_ERR(ctrl);
867 
868 	ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl));
869 	if (WARN_ON(ret)) {
870 		nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
871 		return ret;
872 	}
873 
874 	for (unsigned i = 0; i < ctrl->tableLen; i++) {
875 		enum nvkm_subdev_type type;
876 		int inst;
877 
878 		nvkm_debug(&gsp->subdev,
879 			   "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i,
880 			   ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask,
881 			   ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall);
882 
883 		switch (ctrl->table[i].engineIdx) {
884 		case MC_ENGINE_IDX_GSP:
885 			type = NVKM_SUBDEV_GSP;
886 			inst = 0;
887 			break;
888 		case MC_ENGINE_IDX_DISP:
889 			type = NVKM_ENGINE_DISP;
890 			inst = 0;
891 			break;
892 		case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9:
893 			type = NVKM_ENGINE_CE;
894 			inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0;
895 			break;
896 		case MC_ENGINE_IDX_GR0:
897 			type = NVKM_ENGINE_GR;
898 			inst = 0;
899 			break;
900 		case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7:
901 			type = NVKM_ENGINE_NVDEC;
902 			inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0;
903 			break;
904 		case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2:
905 			type = NVKM_ENGINE_NVENC;
906 			inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC;
907 			break;
908 		case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7:
909 			type = NVKM_ENGINE_NVJPG;
910 			inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0;
911 			break;
912 		case MC_ENGINE_IDX_OFA0:
913 			type = NVKM_ENGINE_OFA;
914 			inst = 0;
915 			break;
916 		default:
917 			continue;
918 		}
919 
920 		if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) {
921 			ret = -ENOSPC;
922 			break;
923 		}
924 
925 		gsp->intr[gsp->intr_nr].type = type;
926 		gsp->intr[gsp->intr_nr].inst = inst;
927 		gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall;
928 		gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall;
929 		gsp->intr_nr++;
930 	}
931 
932 	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
933 	return ret;
934 }
935 
936 static int
r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp * gsp)937 r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp)
938 {
939 	GspStaticConfigInfo *rpc;
940 	int last_usable = -1;
941 
942 	rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc));
943 	if (IS_ERR(rpc))
944 		return PTR_ERR(rpc);
945 
946 	gsp->internal.client.object.client = &gsp->internal.client;
947 	gsp->internal.client.object.parent = NULL;
948 	gsp->internal.client.object.handle = rpc->hInternalClient;
949 	gsp->internal.client.gsp = gsp;
950 
951 	gsp->internal.device.object.client = &gsp->internal.client;
952 	gsp->internal.device.object.parent = &gsp->internal.client.object;
953 	gsp->internal.device.object.handle = rpc->hInternalDevice;
954 
955 	gsp->internal.device.subdevice.client = &gsp->internal.client;
956 	gsp->internal.device.subdevice.parent = &gsp->internal.device.object;
957 	gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice;
958 
959 	gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase;
960 	gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase;
961 
962 	for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) {
963 		NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg =
964 			&rpc->fbRegionInfoParams.fbRegion[i];
965 
966 		nvkm_debug(&gsp->subdev, "fb region %d: "
967 			   "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i,
968 			   reg->base, reg->limit, reg->reserved, reg->performance,
969 			   reg->supportCompressed, reg->supportISO, reg->bProtected);
970 
971 		if (!reg->reserved && !reg->bProtected) {
972 			if (reg->supportCompressed && reg->supportISO &&
973 			    !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) {
974 					const u64 size = (reg->limit + 1) - reg->base;
975 
976 					gsp->fb.region[gsp->fb.region_nr].addr = reg->base;
977 					gsp->fb.region[gsp->fb.region_nr].size = size;
978 					gsp->fb.region_nr++;
979 			}
980 
981 			last_usable = i;
982 		}
983 	}
984 
985 	if (last_usable >= 0) {
986 		u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1;
987 
988 		gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base;
989 	}
990 
991 	for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) {
992 		if (rpc->gpcInfo.gpcMask & BIT(gpc)) {
993 			gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask);
994 			gsp->gr.gpcs++;
995 		}
996 	}
997 
998 	nvkm_gsp_rpc_done(gsp, rpc);
999 	return 0;
1000 }
1001 
1002 static void
nvkm_gsp_mem_dtor(struct nvkm_gsp * gsp,struct nvkm_gsp_mem * mem)1003 nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem)
1004 {
1005 	if (mem->data) {
1006 		/*
1007 		 * Poison the buffer to catch any unexpected access from
1008 		 * GSP-RM if the buffer was prematurely freed.
1009 		 */
1010 		memset(mem->data, 0xFF, mem->size);
1011 
1012 		dma_free_coherent(gsp->subdev.device->dev, mem->size, mem->data, mem->addr);
1013 		memset(mem, 0, sizeof(*mem));
1014 	}
1015 }
1016 
1017 static int
nvkm_gsp_mem_ctor(struct nvkm_gsp * gsp,size_t size,struct nvkm_gsp_mem * mem)1018 nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem)
1019 {
1020 	mem->size = size;
1021 	mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL);
1022 	if (WARN_ON(!mem->data))
1023 		return -ENOMEM;
1024 
1025 	return 0;
1026 }
1027 
1028 static int
r535_gsp_postinit(struct nvkm_gsp * gsp)1029 r535_gsp_postinit(struct nvkm_gsp *gsp)
1030 {
1031 	struct nvkm_device *device = gsp->subdev.device;
1032 	int ret;
1033 
1034 	ret = r535_gsp_rpc_get_gsp_static_info(gsp);
1035 	if (WARN_ON(ret))
1036 		return ret;
1037 
1038 	INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work);
1039 
1040 	ret = r535_gsp_intr_get_table(gsp);
1041 	if (WARN_ON(ret))
1042 		return ret;
1043 
1044 	ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst);
1045 	if (WARN_ON(ret < 0))
1046 		return ret;
1047 
1048 	ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev,
1049 			    r535_gsp_intr, &gsp->subdev.inth);
1050 	if (WARN_ON(ret))
1051 		return ret;
1052 
1053 	nvkm_inth_allow(&gsp->subdev.inth);
1054 	nvkm_wr32(device, 0x110004, 0x00000040);
1055 
1056 	/* Release the DMA buffers that were needed only for boot and init */
1057 	nvkm_gsp_mem_dtor(gsp, &gsp->boot.fw);
1058 	nvkm_gsp_mem_dtor(gsp, &gsp->libos);
1059 
1060 	return ret;
1061 }
1062 
1063 static int
r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp * gsp,bool suspend)1064 r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend)
1065 {
1066 	rpc_unloading_guest_driver_v1F_07 *rpc;
1067 
1068 	rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc));
1069 	if (IS_ERR(rpc))
1070 		return PTR_ERR(rpc);
1071 
1072 	if (suspend) {
1073 		rpc->bInPMTransition = 1;
1074 		rpc->bGc6Entering = 0;
1075 		rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1076 	} else {
1077 		rpc->bInPMTransition = 0;
1078 		rpc->bGc6Entering = 0;
1079 		rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0;
1080 	}
1081 
1082 	return nvkm_gsp_rpc_wr(gsp, rpc, true);
1083 }
1084 
1085 enum registry_type {
1086 	REGISTRY_TABLE_ENTRY_TYPE_DWORD  = 1, /* 32-bit unsigned integer */
1087 	REGISTRY_TABLE_ENTRY_TYPE_BINARY = 2, /* Binary blob */
1088 	REGISTRY_TABLE_ENTRY_TYPE_STRING = 3, /* Null-terminated string */
1089 };
1090 
1091 /* An arbitrary limit to the length of a registry key */
1092 #define REGISTRY_MAX_KEY_LENGTH		64
1093 
1094 /**
1095  * registry_list_entry - linked list member for a registry key/value
1096  * @head: list_head struct
1097  * @type: dword, binary, or string
1098  * @klen: the length of name of the key
1099  * @vlen: the length of the value
1100  * @key: the key name
1101  * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD
1102  * @binary: the data, if TYPE_BINARY or TYPE_STRING
1103  *
1104  * Every registry key/value is represented internally by this struct.
1105  *
1106  * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in
1107  * @dword.
1108  *
1109  * Types BINARY and STRING are variable-length binary blobs.  The only real
1110  * difference between BINARY and STRING is that STRING is null-terminated and
1111  * is expected to contain only printable characters.
1112  *
1113  * Note: it is technically possible to have multiple keys with the same name
1114  * but different types, but this is not useful since GSP-RM expects keys to
1115  * have only one specific type.
1116  */
1117 struct registry_list_entry {
1118 	struct list_head head;
1119 	enum registry_type type;
1120 	size_t klen;
1121 	char key[REGISTRY_MAX_KEY_LENGTH];
1122 	size_t vlen;
1123 	u32 dword;			/* TYPE_DWORD */
1124 	u8 binary[] __counted_by(vlen);	/* TYPE_BINARY or TYPE_STRING */
1125 };
1126 
1127 /**
1128  * add_registry -- adds a registry entry
1129  * @gsp: gsp pointer
1130  * @key: name of the registry key
1131  * @type: type of data
1132  * @data: pointer to value
1133  * @length: size of data, in bytes
1134  *
1135  * Adds a registry key/value pair to the registry database.
1136  *
1137  * This function collects the registry information in a linked list.  After
1138  * all registry keys have been added, build_registry() is used to create the
1139  * RPC data structure.
1140  *
1141  * registry_rpc_size is a running total of the size of all registry keys.
1142  * It's used to avoid an O(n) calculation of the size when the RPC is built.
1143  *
1144  * Returns 0 on success, or negative error code on error.
1145  */
add_registry(struct nvkm_gsp * gsp,const char * key,enum registry_type type,const void * data,size_t length)1146 static int add_registry(struct nvkm_gsp *gsp, const char *key,
1147 			enum registry_type type, const void *data, size_t length)
1148 {
1149 	struct registry_list_entry *reg;
1150 	const size_t nlen = strnlen(key, REGISTRY_MAX_KEY_LENGTH) + 1;
1151 	size_t alloc_size; /* extra bytes to alloc for binary or string value */
1152 
1153 	if (nlen > REGISTRY_MAX_KEY_LENGTH)
1154 		return -EINVAL;
1155 
1156 	alloc_size = (type == REGISTRY_TABLE_ENTRY_TYPE_DWORD) ? 0 : length;
1157 
1158 	reg = kmalloc(sizeof(*reg) + alloc_size, GFP_KERNEL);
1159 	if (!reg)
1160 		return -ENOMEM;
1161 
1162 	switch (type) {
1163 	case REGISTRY_TABLE_ENTRY_TYPE_DWORD:
1164 		reg->dword = *(const u32 *)(data);
1165 		break;
1166 	case REGISTRY_TABLE_ENTRY_TYPE_BINARY:
1167 	case REGISTRY_TABLE_ENTRY_TYPE_STRING:
1168 		memcpy(reg->binary, data, alloc_size);
1169 		break;
1170 	default:
1171 		nvkm_error(&gsp->subdev, "unrecognized registry type %u for '%s'\n",
1172 			   type, key);
1173 		kfree(reg);
1174 		return -EINVAL;
1175 	}
1176 
1177 	memcpy(reg->key, key, nlen);
1178 	reg->klen = nlen;
1179 	reg->vlen = length;
1180 	reg->type = type;
1181 
1182 	list_add_tail(&reg->head, &gsp->registry_list);
1183 	gsp->registry_rpc_size += sizeof(PACKED_REGISTRY_ENTRY) + nlen + alloc_size;
1184 
1185 	return 0;
1186 }
1187 
add_registry_num(struct nvkm_gsp * gsp,const char * key,u32 value)1188 static int add_registry_num(struct nvkm_gsp *gsp, const char *key, u32 value)
1189 {
1190 	return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_DWORD,
1191 			    &value, sizeof(u32));
1192 }
1193 
add_registry_string(struct nvkm_gsp * gsp,const char * key,const char * value)1194 static int add_registry_string(struct nvkm_gsp *gsp, const char *key, const char *value)
1195 {
1196 	return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_STRING,
1197 			    value, strlen(value) + 1);
1198 }
1199 
1200 /**
1201  * build_registry -- create the registry RPC data
1202  * @gsp: gsp pointer
1203  * @registry: pointer to the RPC payload to fill
1204  *
1205  * After all registry key/value pairs have been added, call this function to
1206  * build the RPC.
1207  *
1208  * The registry RPC looks like this:
1209  *
1210  * +-----------------+
1211  * |NvU32 size;      |
1212  * |NvU32 numEntries;|
1213  * +-----------------+
1214  * +----------------------------------------+
1215  * |PACKED_REGISTRY_ENTRY                   |
1216  * +----------------------------------------+
1217  * |Null-terminated key (string) for entry 0|
1218  * +----------------------------------------+
1219  * |Binary/string data value for entry 0    | (only if necessary)
1220  * +----------------------------------------+
1221  *
1222  * +----------------------------------------+
1223  * |PACKED_REGISTRY_ENTRY                   |
1224  * +----------------------------------------+
1225  * |Null-terminated key (string) for entry 1|
1226  * +----------------------------------------+
1227  * |Binary/string data value for entry 1    | (only if necessary)
1228  * +----------------------------------------+
1229  * ... (and so on, one copy for each entry)
1230  *
1231  *
1232  * The 'data' field of an entry is either a 32-bit integer (for type DWORD)
1233  * or an offset into the PACKED_REGISTRY_TABLE (for types BINARY and STRING).
1234  *
1235  * All memory allocated by add_registry() is released.
1236  */
build_registry(struct nvkm_gsp * gsp,PACKED_REGISTRY_TABLE * registry)1237 static void build_registry(struct nvkm_gsp *gsp, PACKED_REGISTRY_TABLE *registry)
1238 {
1239 	struct registry_list_entry *reg, *n;
1240 	size_t str_offset;
1241 	unsigned int i = 0;
1242 
1243 	registry->numEntries = list_count_nodes(&gsp->registry_list);
1244 	str_offset = struct_size(registry, entries, registry->numEntries);
1245 
1246 	list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {
1247 		registry->entries[i].type = reg->type;
1248 		registry->entries[i].length = reg->vlen;
1249 
1250 		/* Append the key name to the table */
1251 		registry->entries[i].nameOffset = str_offset;
1252 		memcpy((void *)registry + str_offset, reg->key, reg->klen);
1253 		str_offset += reg->klen;
1254 
1255 		switch (reg->type) {
1256 		case REGISTRY_TABLE_ENTRY_TYPE_DWORD:
1257 			registry->entries[i].data = reg->dword;
1258 			break;
1259 		case REGISTRY_TABLE_ENTRY_TYPE_BINARY:
1260 		case REGISTRY_TABLE_ENTRY_TYPE_STRING:
1261 			/* If the type is binary or string, also append the value */
1262 			memcpy((void *)registry + str_offset, reg->binary, reg->vlen);
1263 			registry->entries[i].data = str_offset;
1264 			str_offset += reg->vlen;
1265 			break;
1266 		default:
1267 			break;
1268 		}
1269 
1270 		i++;
1271 		list_del(&reg->head);
1272 		kfree(reg);
1273 	}
1274 
1275 	/* Double-check that we calculated the sizes correctly */
1276 	WARN_ON(gsp->registry_rpc_size != str_offset);
1277 
1278 	registry->size = gsp->registry_rpc_size;
1279 }
1280 
1281 /**
1282  * clean_registry -- clean up registry memory in case of error
1283  * @gsp: gsp pointer
1284  *
1285  * Call this function to clean up all memory allocated by add_registry()
1286  * in case of error and build_registry() is not called.
1287  */
clean_registry(struct nvkm_gsp * gsp)1288 static void clean_registry(struct nvkm_gsp *gsp)
1289 {
1290 	struct registry_list_entry *reg, *n;
1291 
1292 	list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {
1293 		list_del(&reg->head);
1294 		kfree(reg);
1295 	}
1296 
1297 	gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);
1298 }
1299 
1300 MODULE_PARM_DESC(NVreg_RegistryDwords,
1301 		 "A semicolon-separated list of key=integer pairs of GSP-RM registry keys");
1302 static char *NVreg_RegistryDwords;
1303 module_param(NVreg_RegistryDwords, charp, 0400);
1304 
1305 /* dword only */
1306 struct nv_gsp_registry_entries {
1307 	const char *name;
1308 	u32 value;
1309 };
1310 
1311 /**
1312  * r535_registry_entries - required registry entries for GSP-RM
1313  *
1314  * This array lists registry entries that are required for GSP-RM to
1315  * function correctly.
1316  *
1317  * RMSecBusResetEnable - enables PCI secondary bus reset
1318  * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration
1319  *   registers on any PCI reset.
1320  */
1321 static const struct nv_gsp_registry_entries r535_registry_entries[] = {
1322 	{ "RMSecBusResetEnable", 1 },
1323 	{ "RMForcePcieConfigSave", 1 },
1324 };
1325 #define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries)
1326 
1327 /**
1328  * strip - strips all characters in 'reject' from 's'
1329  * @s: string to strip
1330  * @reject: string of characters to remove
1331  *
1332  * 's' is modified.
1333  *
1334  * Returns the length of the new string.
1335  */
strip(char * s,const char * reject)1336 static size_t strip(char *s, const char *reject)
1337 {
1338 	char *p = s, *p2 = s;
1339 	size_t length = 0;
1340 	char c;
1341 
1342 	do {
1343 		while ((c = *p2) && strchr(reject, c))
1344 			p2++;
1345 
1346 		*p++ = c = *p2++;
1347 		length++;
1348 	} while (c);
1349 
1350 	return length;
1351 }
1352 
1353 /**
1354  * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM
1355  * @gsp: gsp pointer
1356  *
1357  * The GSP-RM registry is a set of key/value pairs that configure some aspects
1358  * of GSP-RM. The keys are strings, and the values are 32-bit integers.
1359  *
1360  * The registry is built from a combination of a static hard-coded list (see
1361  * above) and entries passed on the driver's command line.
1362  */
1363 static int
r535_gsp_rpc_set_registry(struct nvkm_gsp * gsp)1364 r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp)
1365 {
1366 	PACKED_REGISTRY_TABLE *rpc;
1367 	unsigned int i;
1368 	int ret;
1369 
1370 	INIT_LIST_HEAD(&gsp->registry_list);
1371 	gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);
1372 
1373 	for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) {
1374 		ret = add_registry_num(gsp, r535_registry_entries[i].name,
1375 				       r535_registry_entries[i].value);
1376 		if (ret)
1377 			goto fail;
1378 	}
1379 
1380 	/*
1381 	 * The NVreg_RegistryDwords parameter is a string of key=value
1382 	 * pairs separated by semicolons. We need to extract and trim each
1383 	 * substring, and then parse the substring to extract the key and
1384 	 * value.
1385 	 */
1386 	if (NVreg_RegistryDwords) {
1387 		char *p = kstrdup(NVreg_RegistryDwords, GFP_KERNEL);
1388 		char *start, *next = p, *equal;
1389 
1390 		if (!p) {
1391 			ret = -ENOMEM;
1392 			goto fail;
1393 		}
1394 
1395 		/* Remove any whitespace from the parameter string */
1396 		strip(p, " \t\n");
1397 
1398 		while ((start = strsep(&next, ";"))) {
1399 			long value;
1400 
1401 			equal = strchr(start, '=');
1402 			if (!equal || equal == start || equal[1] == 0) {
1403 				nvkm_error(&gsp->subdev,
1404 					   "ignoring invalid registry string '%s'\n",
1405 					   start);
1406 				continue;
1407 			}
1408 
1409 			/* Truncate the key=value string to just key */
1410 			*equal = 0;
1411 
1412 			ret = kstrtol(equal + 1, 0, &value);
1413 			if (!ret) {
1414 				ret = add_registry_num(gsp, start, value);
1415 			} else {
1416 				/* Not a number, so treat it as a string */
1417 				ret = add_registry_string(gsp, start, equal + 1);
1418 			}
1419 
1420 			if (ret) {
1421 				nvkm_error(&gsp->subdev,
1422 					   "ignoring invalid registry key/value '%s=%s'\n",
1423 					   start, equal + 1);
1424 				continue;
1425 			}
1426 		}
1427 
1428 		kfree(p);
1429 	}
1430 
1431 	rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, gsp->registry_rpc_size);
1432 	if (IS_ERR(rpc)) {
1433 		ret = PTR_ERR(rpc);
1434 		goto fail;
1435 	}
1436 
1437 	build_registry(gsp, rpc);
1438 
1439 	return nvkm_gsp_rpc_wr(gsp, rpc, false);
1440 
1441 fail:
1442 	clean_registry(gsp);
1443 	return ret;
1444 }
1445 
1446 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1447 static void
r535_gsp_acpi_caps(acpi_handle handle,CAPS_METHOD_DATA * caps)1448 r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps)
1449 {
1450 	const guid_t NVOP_DSM_GUID =
1451 		GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B,
1452 			  0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0);
1453 	u64 NVOP_DSM_REV = 0x00000100;
1454 	union acpi_object argv4 = {
1455 		.buffer.type    = ACPI_TYPE_BUFFER,
1456 		.buffer.length  = 4,
1457 		.buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),
1458 	}, *obj;
1459 
1460 	caps->status = 0xffff;
1461 
1462 	if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a)))
1463 		return;
1464 
1465 	obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4);
1466 	if (!obj)
1467 		return;
1468 
1469 	if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1470 	    WARN_ON(obj->buffer.length != 4))
1471 		return;
1472 
1473 	caps->status = 0;
1474 	caps->optimusCaps = *(u32 *)obj->buffer.pointer;
1475 
1476 	ACPI_FREE(obj);
1477 
1478 	kfree(argv4.buffer.pointer);
1479 }
1480 
1481 static void
r535_gsp_acpi_jt(acpi_handle handle,JT_METHOD_DATA * jt)1482 r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt)
1483 {
1484 	const guid_t JT_DSM_GUID =
1485 		GUID_INIT(0xCBECA351L, 0x067B, 0x4924,
1486 			  0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34);
1487 	u64 JT_DSM_REV = 0x00000103;
1488 	u32 caps;
1489 	union acpi_object argv4 = {
1490 		.buffer.type    = ACPI_TYPE_BUFFER,
1491 		.buffer.length  = sizeof(caps),
1492 		.buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),
1493 	}, *obj;
1494 
1495 	jt->status = 0xffff;
1496 
1497 	obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4);
1498 	if (!obj)
1499 		return;
1500 
1501 	if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1502 	    WARN_ON(obj->buffer.length != 4))
1503 		return;
1504 
1505 	jt->status = 0;
1506 	jt->jtCaps = *(u32 *)obj->buffer.pointer;
1507 	jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20;
1508 	jt->bSBIOSCaps = 0;
1509 
1510 	ACPI_FREE(obj);
1511 
1512 	kfree(argv4.buffer.pointer);
1513 }
1514 
1515 static void
r535_gsp_acpi_mux_id(acpi_handle handle,u32 id,MUX_METHOD_DATA_ELEMENT * mode,MUX_METHOD_DATA_ELEMENT * part)1516 r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
1517 						 MUX_METHOD_DATA_ELEMENT *part)
1518 {
1519 	union acpi_object mux_arg = { ACPI_TYPE_INTEGER };
1520 	struct acpi_object_list input = { 1, &mux_arg };
1521 	acpi_handle iter = NULL, handle_mux = NULL;
1522 	acpi_status status;
1523 	unsigned long long value;
1524 
1525 	mode->status = 0xffff;
1526 	part->status = 0xffff;
1527 
1528 	do {
1529 		status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter);
1530 		if (ACPI_FAILURE(status) || !iter)
1531 			return;
1532 
1533 		status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);
1534 		if (ACPI_FAILURE(status) || value != id)
1535 			continue;
1536 
1537 		handle_mux = iter;
1538 	} while (!handle_mux);
1539 
1540 	if (!handle_mux)
1541 		return;
1542 
1543 	/* I -think- 0 means "acquire" according to nvidia's driver source */
1544 	input.pointer->integer.type = ACPI_TYPE_INTEGER;
1545 	input.pointer->integer.value = 0;
1546 
1547 	status = acpi_evaluate_integer(handle_mux, "MXDM", &input, &value);
1548 	if (ACPI_SUCCESS(status)) {
1549 		mode->acpiId = id;
1550 		mode->mode   = value;
1551 		mode->status = 0;
1552 	}
1553 
1554 	status = acpi_evaluate_integer(handle_mux, "MXDS", &input, &value);
1555 	if (ACPI_SUCCESS(status)) {
1556 		part->acpiId = id;
1557 		part->mode   = value;
1558 		part->status = 0;
1559 	}
1560 }
1561 
1562 static void
r535_gsp_acpi_mux(acpi_handle handle,DOD_METHOD_DATA * dod,MUX_METHOD_DATA * mux)1563 r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux)
1564 {
1565 	mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]);
1566 
1567 	for (int i = 0; i < mux->tableLen; i++) {
1568 		r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i],
1569 								 &mux->acpiIdMuxPartTable[i]);
1570 	}
1571 }
1572 
1573 static void
r535_gsp_acpi_dod(acpi_handle handle,DOD_METHOD_DATA * dod)1574 r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod)
1575 {
1576 	acpi_status status;
1577 	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
1578 	union acpi_object *_DOD;
1579 
1580 	dod->status = 0xffff;
1581 
1582 	status = acpi_evaluate_object(handle, "_DOD", NULL, &output);
1583 	if (ACPI_FAILURE(status))
1584 		return;
1585 
1586 	_DOD = output.pointer;
1587 
1588 	if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) ||
1589 	    WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList)))
1590 		return;
1591 
1592 	for (int i = 0; i < _DOD->package.count; i++) {
1593 		if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER))
1594 			return;
1595 
1596 		dod->acpiIdList[i] = _DOD->package.elements[i].integer.value;
1597 		dod->acpiIdListLen += sizeof(dod->acpiIdList[0]);
1598 	}
1599 
1600 	dod->status = 0;
1601 	kfree(output.pointer);
1602 }
1603 #endif
1604 
1605 static void
r535_gsp_acpi_info(struct nvkm_gsp * gsp,ACPI_METHOD_DATA * acpi)1606 r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi)
1607 {
1608 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1609 	acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev);
1610 
1611 	if (!handle)
1612 		return;
1613 
1614 	acpi->bValid = 1;
1615 
1616 	r535_gsp_acpi_dod(handle, &acpi->dodMethodData);
1617 	if (acpi->dodMethodData.status == 0)
1618 		r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData);
1619 
1620 	r535_gsp_acpi_jt(handle, &acpi->jtMethodData);
1621 	r535_gsp_acpi_caps(handle, &acpi->capsMethodData);
1622 #endif
1623 }
1624 
1625 static int
r535_gsp_rpc_set_system_info(struct nvkm_gsp * gsp)1626 r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)
1627 {
1628 	struct nvkm_device *device = gsp->subdev.device;
1629 	struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device);
1630 	GspSystemInfo *info;
1631 
1632 	if (WARN_ON(device->type == NVKM_DEVICE_TEGRA))
1633 		return -ENOSYS;
1634 
1635 	info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info));
1636 	if (IS_ERR(info))
1637 		return PTR_ERR(info);
1638 
1639 	info->gpuPhysAddr = device->func->resource_addr(device, 0);
1640 	info->gpuPhysFbAddr = device->func->resource_addr(device, 1);
1641 	info->gpuPhysInstAddr = device->func->resource_addr(device, 3);
1642 	info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev);
1643 	info->maxUserVa = TASK_SIZE;
1644 	info->pciConfigMirrorBase = 0x088000;
1645 	info->pciConfigMirrorSize = 0x001000;
1646 	r535_gsp_acpi_info(gsp, &info->acpiMethodData);
1647 
1648 	return nvkm_gsp_rpc_wr(gsp, info, false);
1649 }
1650 
1651 static int
r535_gsp_msg_os_error_log(void * priv,u32 fn,void * repv,u32 repc)1652 r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc)
1653 {
1654 	struct nvkm_gsp *gsp = priv;
1655 	struct nvkm_subdev *subdev = &gsp->subdev;
1656 	rpc_os_error_log_v17_00 *msg = repv;
1657 
1658 	if (WARN_ON(repc < sizeof(*msg)))
1659 		return -EINVAL;
1660 
1661 	nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString);
1662 	return 0;
1663 }
1664 
1665 static int
r535_gsp_msg_rc_triggered(void * priv,u32 fn,void * repv,u32 repc)1666 r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc)
1667 {
1668 	rpc_rc_triggered_v17_02 *msg = repv;
1669 	struct nvkm_gsp *gsp = priv;
1670 	struct nvkm_subdev *subdev = &gsp->subdev;
1671 	struct nvkm_chan *chan;
1672 	unsigned long flags;
1673 
1674 	if (WARN_ON(repc < sizeof(*msg)))
1675 		return -EINVAL;
1676 
1677 	nvkm_error(subdev, "rc engn:%08x chid:%d type:%d scope:%d part:%d\n",
1678 		   msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope,
1679 		   msg->partitionAttributionId);
1680 
1681 	chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags);
1682 	if (!chan) {
1683 		nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid);
1684 		return 0;
1685 	}
1686 
1687 	nvkm_chan_error(chan, false);
1688 	nvkm_chan_put(&chan, flags);
1689 	return 0;
1690 }
1691 
1692 static int
r535_gsp_msg_mmu_fault_queued(void * priv,u32 fn,void * repv,u32 repc)1693 r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc)
1694 {
1695 	struct nvkm_gsp *gsp = priv;
1696 	struct nvkm_subdev *subdev = &gsp->subdev;
1697 
1698 	WARN_ON(repc != 0);
1699 
1700 	nvkm_error(subdev, "mmu fault queued\n");
1701 	return 0;
1702 }
1703 
1704 static int
r535_gsp_msg_post_event(void * priv,u32 fn,void * repv,u32 repc)1705 r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc)
1706 {
1707 	struct nvkm_gsp *gsp = priv;
1708 	struct nvkm_gsp_client *client;
1709 	struct nvkm_subdev *subdev = &gsp->subdev;
1710 	rpc_post_event_v17_00 *msg = repv;
1711 
1712 	if (WARN_ON(repc < sizeof(*msg)))
1713 		return -EINVAL;
1714 	if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize))
1715 		return -EINVAL;
1716 
1717 	nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n",
1718 		   msg->hClient, msg->hEvent, msg->notifyIndex, msg->data,
1719 		   msg->status, msg->eventDataSize, msg->bNotifyList);
1720 
1721 	mutex_lock(&gsp->client_id.mutex);
1722 	client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff);
1723 	if (client) {
1724 		struct nvkm_gsp_event *event;
1725 		bool handled = false;
1726 
1727 		list_for_each_entry(event, &client->events, head) {
1728 			if (event->object.handle == msg->hEvent) {
1729 				event->func(event, msg->eventData, msg->eventDataSize);
1730 				handled = true;
1731 			}
1732 		}
1733 
1734 		if (!handled) {
1735 			nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n",
1736 				   msg->hClient, msg->hEvent);
1737 		}
1738 	} else {
1739 		nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient);
1740 	}
1741 	mutex_unlock(&gsp->client_id.mutex);
1742 	return 0;
1743 }
1744 
1745 /**
1746  * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP
1747  * @priv: gsp pointer
1748  * @fn: function number (ignored)
1749  * @repv: pointer to libos print RPC
1750  * @repc: message size
1751  *
1752  * The GSP sequencer is a list of I/O commands that the GSP can send to
1753  * the driver to perform for various purposes.  The most common usage is to
1754  * perform a special mid-initialization reset.
1755  */
1756 static int
r535_gsp_msg_run_cpu_sequencer(void * priv,u32 fn,void * repv,u32 repc)1757 r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc)
1758 {
1759 	struct nvkm_gsp *gsp = priv;
1760 	struct nvkm_subdev *subdev = &gsp->subdev;
1761 	struct nvkm_device *device = subdev->device;
1762 	rpc_run_cpu_sequencer_v17_00 *seq = repv;
1763 	int ptr = 0, ret;
1764 
1765 	nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex);
1766 
1767 	while (ptr < seq->cmdIndex) {
1768 		GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr];
1769 
1770 		ptr += 1;
1771 		ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode);
1772 
1773 		switch (cmd->opCode) {
1774 		case GSP_SEQ_BUF_OPCODE_REG_WRITE: {
1775 			u32 addr = cmd->payload.regWrite.addr;
1776 			u32 data = cmd->payload.regWrite.val;
1777 
1778 			nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data);
1779 			nvkm_wr32(device, addr, data);
1780 		}
1781 			break;
1782 		case GSP_SEQ_BUF_OPCODE_REG_MODIFY: {
1783 			u32 addr = cmd->payload.regModify.addr;
1784 			u32 mask = cmd->payload.regModify.mask;
1785 			u32 data = cmd->payload.regModify.val;
1786 
1787 			nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data);
1788 			nvkm_mask(device, addr, mask, data);
1789 		}
1790 			break;
1791 		case GSP_SEQ_BUF_OPCODE_REG_POLL: {
1792 			u32 addr = cmd->payload.regPoll.addr;
1793 			u32 mask = cmd->payload.regPoll.mask;
1794 			u32 data = cmd->payload.regPoll.val;
1795 			u32 usec = cmd->payload.regPoll.timeout ?: 4000000;
1796 			//u32 error = cmd->payload.regPoll.error;
1797 
1798 			nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec);
1799 			nvkm_rd32(device, addr);
1800 			nvkm_usec(device, usec,
1801 				if ((nvkm_rd32(device, addr) & mask) == data)
1802 					break;
1803 			);
1804 		}
1805 			break;
1806 		case GSP_SEQ_BUF_OPCODE_DELAY_US: {
1807 			u32 usec = cmd->payload.delayUs.val;
1808 
1809 			nvkm_trace(subdev, "seq usec %d\n", usec);
1810 			udelay(usec);
1811 		}
1812 			break;
1813 		case GSP_SEQ_BUF_OPCODE_REG_STORE: {
1814 			u32 addr = cmd->payload.regStore.addr;
1815 			u32 slot = cmd->payload.regStore.index;
1816 
1817 			seq->regSaveArea[slot] = nvkm_rd32(device, addr);
1818 			nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot,
1819 				   seq->regSaveArea[slot]);
1820 		}
1821 			break;
1822 		case GSP_SEQ_BUF_OPCODE_CORE_RESET:
1823 			nvkm_trace(subdev, "seq core reset\n");
1824 			nvkm_falcon_reset(&gsp->falcon);
1825 			nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080);
1826 			nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000);
1827 			break;
1828 		case GSP_SEQ_BUF_OPCODE_CORE_START:
1829 			nvkm_trace(subdev, "seq core start\n");
1830 			if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040)
1831 				nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002);
1832 			else
1833 				nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002);
1834 			break;
1835 		case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT:
1836 			nvkm_trace(subdev, "seq core wait halt\n");
1837 			nvkm_msec(device, 2000,
1838 				if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010)
1839 					break;
1840 			);
1841 			break;
1842 		case GSP_SEQ_BUF_OPCODE_CORE_RESUME: {
1843 			struct nvkm_sec2 *sec2 = device->sec2;
1844 			u32 mbox0;
1845 
1846 			nvkm_trace(subdev, "seq core resume\n");
1847 
1848 			ret = gsp->func->reset(gsp);
1849 			if (WARN_ON(ret))
1850 				return ret;
1851 
1852 			nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
1853 			nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
1854 
1855 			nvkm_falcon_start(&sec2->falcon);
1856 
1857 			if (nvkm_msec(device, 2000,
1858 				if (nvkm_rd32(device, 0x1180f8) & 0x04000000)
1859 					break;
1860 			) < 0)
1861 				return -ETIMEDOUT;
1862 
1863 			mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040);
1864 			if (WARN_ON(mbox0)) {
1865 				nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0);
1866 				return -EIO;
1867 			}
1868 
1869 			nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1870 
1871 			if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1872 				return -EIO;
1873 		}
1874 			break;
1875 		default:
1876 			nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode);
1877 			return -EINVAL;
1878 		}
1879 	}
1880 
1881 	return 0;
1882 }
1883 
1884 static int
r535_gsp_booter_unload(struct nvkm_gsp * gsp,u32 mbox0,u32 mbox1)1885 r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1886 {
1887 	struct nvkm_subdev *subdev = &gsp->subdev;
1888 	struct nvkm_device *device = subdev->device;
1889 	u32 wpr2_hi;
1890 	int ret;
1891 
1892 	wpr2_hi = nvkm_rd32(device, 0x1fa828);
1893 	if (!wpr2_hi) {
1894 		nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n");
1895 		return 0;
1896 	}
1897 
1898 	ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1899 	if (WARN_ON(ret))
1900 		return ret;
1901 
1902 	wpr2_hi = nvkm_rd32(device, 0x1fa828);
1903 	if (WARN_ON(wpr2_hi))
1904 		return -EIO;
1905 
1906 	return 0;
1907 }
1908 
1909 static int
r535_gsp_booter_load(struct nvkm_gsp * gsp,u32 mbox0,u32 mbox1)1910 r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1911 {
1912 	int ret;
1913 
1914 	ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1915 	if (ret)
1916 		return ret;
1917 
1918 	nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1919 
1920 	if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1921 		return -EIO;
1922 
1923 	return 0;
1924 }
1925 
1926 static int
r535_gsp_wpr_meta_init(struct nvkm_gsp * gsp)1927 r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp)
1928 {
1929 	GspFwWprMeta *meta;
1930 	int ret;
1931 
1932 	ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta);
1933 	if (ret)
1934 		return ret;
1935 
1936 	meta = gsp->wpr_meta.data;
1937 
1938 	meta->magic = GSP_FW_WPR_META_MAGIC;
1939 	meta->revision = GSP_FW_WPR_META_REVISION;
1940 
1941 	meta->sysmemAddrOfRadix3Elf = gsp->radix3.lvl0.addr;
1942 	meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size;
1943 
1944 	meta->sysmemAddrOfBootloader = gsp->boot.fw.addr;
1945 	meta->sizeOfBootloader = gsp->boot.fw.size;
1946 	meta->bootloaderCodeOffset = gsp->boot.code_offset;
1947 	meta->bootloaderDataOffset = gsp->boot.data_offset;
1948 	meta->bootloaderManifestOffset = gsp->boot.manifest_offset;
1949 
1950 	meta->sysmemAddrOfSignature = gsp->sig.addr;
1951 	meta->sizeOfSignature = gsp->sig.size;
1952 
1953 	meta->gspFwRsvdStart = gsp->fb.heap.addr;
1954 	meta->nonWprHeapOffset = gsp->fb.heap.addr;
1955 	meta->nonWprHeapSize = gsp->fb.heap.size;
1956 	meta->gspFwWprStart = gsp->fb.wpr2.addr;
1957 	meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr;
1958 	meta->gspFwHeapSize = gsp->fb.wpr2.heap.size;
1959 	meta->gspFwOffset = gsp->fb.wpr2.elf.addr;
1960 	meta->bootBinOffset = gsp->fb.wpr2.boot.addr;
1961 	meta->frtsOffset = gsp->fb.wpr2.frts.addr;
1962 	meta->frtsSize = gsp->fb.wpr2.frts.size;
1963 	meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000);
1964 	meta->fbSize = gsp->fb.size;
1965 	meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr;
1966 	meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size;
1967 	meta->bootCount = 0;
1968 	meta->partitionRpcAddr = 0;
1969 	meta->partitionRpcRequestOffset = 0;
1970 	meta->partitionRpcReplyOffset = 0;
1971 	meta->verified = 0;
1972 	return 0;
1973 }
1974 
1975 static int
r535_gsp_shared_init(struct nvkm_gsp * gsp)1976 r535_gsp_shared_init(struct nvkm_gsp *gsp)
1977 {
1978 	struct {
1979 		msgqTxHeader tx;
1980 		msgqRxHeader rx;
1981 	} *cmdq, *msgq;
1982 	int ret, i;
1983 
1984 	gsp->shm.cmdq.size = 0x40000;
1985 	gsp->shm.msgq.size = 0x40000;
1986 
1987 	gsp->shm.ptes.nr  = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT;
1988 	gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1989 	gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1990 
1991 	ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size +
1992 				     gsp->shm.cmdq.size +
1993 				     gsp->shm.msgq.size,
1994 				&gsp->shm.mem);
1995 	if (ret)
1996 		return ret;
1997 
1998 	gsp->shm.ptes.ptr = gsp->shm.mem.data;
1999 	gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size;
2000 	gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size;
2001 
2002 	for (i = 0; i < gsp->shm.ptes.nr; i++)
2003 		gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT);
2004 
2005 	cmdq = gsp->shm.cmdq.ptr;
2006 	cmdq->tx.version = 0;
2007 	cmdq->tx.size = gsp->shm.cmdq.size;
2008 	cmdq->tx.entryOff = GSP_PAGE_SIZE;
2009 	cmdq->tx.msgSize = GSP_PAGE_SIZE;
2010 	cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize;
2011 	cmdq->tx.writePtr = 0;
2012 	cmdq->tx.flags = 1;
2013 	cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr);
2014 
2015 	msgq = gsp->shm.msgq.ptr;
2016 
2017 	gsp->cmdq.cnt = cmdq->tx.msgCount;
2018 	gsp->cmdq.wptr = &cmdq->tx.writePtr;
2019 	gsp->cmdq.rptr = &msgq->rx.readPtr;
2020 	gsp->msgq.cnt = cmdq->tx.msgCount;
2021 	gsp->msgq.wptr = &msgq->tx.writePtr;
2022 	gsp->msgq.rptr = &cmdq->rx.readPtr;
2023 	return 0;
2024 }
2025 
2026 static int
r535_gsp_rmargs_init(struct nvkm_gsp * gsp,bool resume)2027 r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume)
2028 {
2029 	GSP_ARGUMENTS_CACHED *args;
2030 	int ret;
2031 
2032 	if (!resume) {
2033 		ret = r535_gsp_shared_init(gsp);
2034 		if (ret)
2035 			return ret;
2036 
2037 		ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs);
2038 		if (ret)
2039 			return ret;
2040 	}
2041 
2042 	args = gsp->rmargs.data;
2043 	args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr;
2044 	args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr;
2045 	args->messageQueueInitArguments.cmdQueueOffset =
2046 		(u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data;
2047 	args->messageQueueInitArguments.statQueueOffset =
2048 		(u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data;
2049 
2050 	if (!resume) {
2051 		args->srInitArguments.oldLevel = 0;
2052 		args->srInitArguments.flags = 0;
2053 		args->srInitArguments.bInPMTransition = 0;
2054 	} else {
2055 		args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
2056 		args->srInitArguments.flags = 0;
2057 		args->srInitArguments.bInPMTransition = 1;
2058 	}
2059 
2060 	return 0;
2061 }
2062 
2063 static inline u64
r535_gsp_libos_id8(const char * name)2064 r535_gsp_libos_id8(const char *name)
2065 {
2066 	u64 id = 0;
2067 
2068 	for (int i = 0; i < sizeof(id) && *name; i++, name++)
2069 		id = (id << 8) | *name;
2070 
2071 	return id;
2072 }
2073 
2074 /**
2075  * create_pte_array() - creates a PTE array of a physically contiguous buffer
2076  * @ptes: pointer to the array
2077  * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned)
2078  * @size: size of the buffer
2079  *
2080  * GSP-RM sometimes expects physically-contiguous buffers to have an array of
2081  * "PTEs" for each page in that buffer.  Although in theory that allows for
2082  * the buffer to be physically discontiguous, GSP-RM does not currently
2083  * support that.
2084  *
2085  * In this case, the PTEs are DMA addresses of each page of the buffer.  Since
2086  * the buffer is physically contiguous, calculating all the PTEs is simple
2087  * math.
2088  *
2089  * See memdescGetPhysAddrsForGpu()
2090  */
create_pte_array(u64 * ptes,dma_addr_t addr,size_t size)2091 static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size)
2092 {
2093 	unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE);
2094 	unsigned int i;
2095 
2096 	for (i = 0; i < num_pages; i++)
2097 		ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT);
2098 }
2099 
2100 /**
2101  * r535_gsp_libos_init() -- create the libos arguments structure
2102  * @gsp: gsp pointer
2103  *
2104  * The logging buffers are byte queues that contain encoded printf-like
2105  * messages from GSP-RM.  They need to be decoded by a special application
2106  * that can parse the buffers.
2107  *
2108  * The 'loginit' buffer contains logs from early GSP-RM init and
2109  * exception dumps.  The 'logrm' buffer contains the subsequent logs. Both are
2110  * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.
2111  *
2112  * The physical address map for the log buffer is stored in the buffer
2113  * itself, starting with offset 1. Offset 0 contains the "put" pointer.
2114  *
2115  * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is
2116  * configured for a larger page size (e.g. 64K pages), we need to give
2117  * the GSP an array of 4K pages. Fortunately, since the buffer is
2118  * physically contiguous, it's simple math to calculate the addresses.
2119  *
2120  * The buffers must be a multiple of GSP_PAGE_SIZE.  GSP-RM also currently
2121  * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the
2122  * buffers to be physically contiguous anyway.
2123  *
2124  * The memory allocated for the arguments must remain until the GSP sends the
2125  * init_done RPC.
2126  *
2127  * See _kgspInitLibosLoggingStructures (allocates memory for buffers)
2128  * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
2129  */
2130 static int
r535_gsp_libos_init(struct nvkm_gsp * gsp)2131 r535_gsp_libos_init(struct nvkm_gsp *gsp)
2132 {
2133 	LibosMemoryRegionInitArgument *args;
2134 	int ret;
2135 
2136 	ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos);
2137 	if (ret)
2138 		return ret;
2139 
2140 	args = gsp->libos.data;
2141 
2142 	ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit);
2143 	if (ret)
2144 		return ret;
2145 
2146 	args[0].id8  = r535_gsp_libos_id8("LOGINIT");
2147 	args[0].pa   = gsp->loginit.addr;
2148 	args[0].size = gsp->loginit.size;
2149 	args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2150 	args[0].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2151 	create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size);
2152 
2153 	ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr);
2154 	if (ret)
2155 		return ret;
2156 
2157 	args[1].id8  = r535_gsp_libos_id8("LOGINTR");
2158 	args[1].pa   = gsp->logintr.addr;
2159 	args[1].size = gsp->logintr.size;
2160 	args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2161 	args[1].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2162 	create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size);
2163 
2164 	ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm);
2165 	if (ret)
2166 		return ret;
2167 
2168 	args[2].id8  = r535_gsp_libos_id8("LOGRM");
2169 	args[2].pa   = gsp->logrm.addr;
2170 	args[2].size = gsp->logrm.size;
2171 	args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2172 	args[2].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2173 	create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size);
2174 
2175 	ret = r535_gsp_rmargs_init(gsp, false);
2176 	if (ret)
2177 		return ret;
2178 
2179 	args[3].id8  = r535_gsp_libos_id8("RMARGS");
2180 	args[3].pa   = gsp->rmargs.addr;
2181 	args[3].size = gsp->rmargs.size;
2182 	args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2183 	args[3].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2184 	return 0;
2185 }
2186 
2187 void
nvkm_gsp_sg_free(struct nvkm_device * device,struct sg_table * sgt)2188 nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt)
2189 {
2190 	struct scatterlist *sgl;
2191 	int i;
2192 
2193 	dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
2194 
2195 	for_each_sgtable_sg(sgt, sgl, i) {
2196 		struct page *page = sg_page(sgl);
2197 
2198 		__free_page(page);
2199 	}
2200 
2201 	sg_free_table(sgt);
2202 }
2203 
2204 int
nvkm_gsp_sg(struct nvkm_device * device,u64 size,struct sg_table * sgt)2205 nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)
2206 {
2207 	const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE);
2208 	struct scatterlist *sgl;
2209 	int ret, i;
2210 
2211 	ret = sg_alloc_table(sgt, pages, GFP_KERNEL);
2212 	if (ret)
2213 		return ret;
2214 
2215 	for_each_sgtable_sg(sgt, sgl, i) {
2216 		struct page *page = alloc_page(GFP_KERNEL);
2217 
2218 		if (!page) {
2219 			nvkm_gsp_sg_free(device, sgt);
2220 			return -ENOMEM;
2221 		}
2222 
2223 		sg_set_page(sgl, page, PAGE_SIZE, 0);
2224 	}
2225 
2226 	ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
2227 	if (ret)
2228 		nvkm_gsp_sg_free(device, sgt);
2229 
2230 	return ret;
2231 }
2232 
2233 static void
nvkm_gsp_radix3_dtor(struct nvkm_gsp * gsp,struct nvkm_gsp_radix3 * rx3)2234 nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)
2235 {
2236 	nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2);
2237 	nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);
2238 	nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);
2239 }
2240 
2241 /**
2242  * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list
2243  * @gsp: gsp pointer
2244  * @sgt: S/G list to traverse
2245  * @size: size of the image, in bytes
2246  * @rx3: radix3 array to update
2247  *
2248  * The GSP uses a three-level page table, called radix3, to map the firmware.
2249  * Each 64-bit "pointer" in the table is either the bus address of an entry in
2250  * the next table (for levels 0 and 1) or the bus address of the next page in
2251  * the GSP firmware image itself.
2252  *
2253  * Level 0 contains a single entry in one page that points to the first page
2254  * of level 1.
2255  *
2256  * Level 1, since it's also only one page in size, contains up to 512 entries,
2257  * one for each page in Level 2.
2258  *
2259  * Level 2 can be up to 512 pages in size, and each of those entries points to
2260  * the next page of the firmware image.  Since there can be up to 512*512
2261  * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB.
2262  *
2263  * Internally, the GSP has its window into system memory, but the base
2264  * physical address of the aperture is not 0.  In fact, it varies depending on
2265  * the GPU architecture.  Since the GPU is a PCI device, this window is
2266  * accessed via DMA and is therefore bound by IOMMU translation.  The end
2267  * result is that GSP-RM must translate the bus addresses in the table to GSP
2268  * physical addresses.  All this should happen transparently.
2269  *
2270  * Returns 0 on success, or negative error code
2271  *
2272  * See kgspCreateRadix3_IMPL
2273  */
2274 static int
nvkm_gsp_radix3_sg(struct nvkm_gsp * gsp,struct sg_table * sgt,u64 size,struct nvkm_gsp_radix3 * rx3)2275 nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size,
2276 		   struct nvkm_gsp_radix3 *rx3)
2277 {
2278 	struct sg_dma_page_iter sg_dma_iter;
2279 	struct scatterlist *sg;
2280 	size_t bufsize;
2281 	u64 *pte;
2282 	int ret, i, page_idx = 0;
2283 
2284 	ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0);
2285 	if (ret)
2286 		return ret;
2287 
2288 	ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1);
2289 	if (ret)
2290 		goto lvl1_fail;
2291 
2292 	// Allocate level 2
2293 	bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
2294 	ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2);
2295 	if (ret)
2296 		goto lvl2_fail;
2297 
2298 	// Write the bus address of level 1 to level 0
2299 	pte = rx3->lvl0.data;
2300 	*pte = rx3->lvl1.addr;
2301 
2302 	// Write the bus address of each page in level 2 to level 1
2303 	pte = rx3->lvl1.data;
2304 	for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0)
2305 		*pte++ = sg_page_iter_dma_address(&sg_dma_iter);
2306 
2307 	// Finally, write the bus address of each page in sgt to level 2
2308 	for_each_sgtable_sg(&rx3->lvl2, sg, i) {
2309 		void *sgl_end;
2310 
2311 		pte = sg_virt(sg);
2312 		sgl_end = (void *)pte + sg->length;
2313 
2314 		for_each_sgtable_dma_page(sgt, &sg_dma_iter, page_idx) {
2315 			*pte++ = sg_page_iter_dma_address(&sg_dma_iter);
2316 			page_idx++;
2317 
2318 			// Go to the next scatterlist for level 2 if we've reached the end
2319 			if ((void *)pte >= sgl_end)
2320 				break;
2321 		}
2322 	}
2323 
2324 	if (ret) {
2325 lvl2_fail:
2326 		nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);
2327 lvl1_fail:
2328 		nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);
2329 	}
2330 
2331 	return ret;
2332 }
2333 
2334 int
r535_gsp_fini(struct nvkm_gsp * gsp,bool suspend)2335 r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)
2336 {
2337 	u32 mbox0 = 0xff, mbox1 = 0xff;
2338 	int ret;
2339 
2340 	if (!gsp->running)
2341 		return 0;
2342 
2343 	if (suspend) {
2344 		GspFwWprMeta *meta = gsp->wpr_meta.data;
2345 		u64 len = meta->gspFwWprEnd - meta->gspFwWprStart;
2346 		GspFwSRMeta *sr;
2347 
2348 		ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt);
2349 		if (ret)
2350 			return ret;
2351 
2352 		ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3);
2353 		if (ret)
2354 			return ret;
2355 
2356 		ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta);
2357 		if (ret)
2358 			return ret;
2359 
2360 		sr = gsp->sr.meta.data;
2361 		sr->magic = GSP_FW_SR_META_MAGIC;
2362 		sr->revision = GSP_FW_SR_META_REVISION;
2363 		sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr;
2364 		sr->sizeOfSuspendResumeData = len;
2365 
2366 		mbox0 = lower_32_bits(gsp->sr.meta.addr);
2367 		mbox1 = upper_32_bits(gsp->sr.meta.addr);
2368 	}
2369 
2370 	ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend);
2371 	if (WARN_ON(ret))
2372 		return ret;
2373 
2374 	nvkm_msec(gsp->subdev.device, 2000,
2375 		if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000)
2376 			break;
2377 	);
2378 
2379 	nvkm_falcon_reset(&gsp->falcon);
2380 
2381 	ret = nvkm_gsp_fwsec_sb(gsp);
2382 	WARN_ON(ret);
2383 
2384 	ret = r535_gsp_booter_unload(gsp, mbox0, mbox1);
2385 	WARN_ON(ret);
2386 
2387 	gsp->running = false;
2388 	return 0;
2389 }
2390 
2391 int
r535_gsp_init(struct nvkm_gsp * gsp)2392 r535_gsp_init(struct nvkm_gsp *gsp)
2393 {
2394 	u32 mbox0, mbox1;
2395 	int ret;
2396 
2397 	if (!gsp->sr.meta.data) {
2398 		mbox0 = lower_32_bits(gsp->wpr_meta.addr);
2399 		mbox1 = upper_32_bits(gsp->wpr_meta.addr);
2400 	} else {
2401 		r535_gsp_rmargs_init(gsp, true);
2402 
2403 		mbox0 = lower_32_bits(gsp->sr.meta.addr);
2404 		mbox1 = upper_32_bits(gsp->sr.meta.addr);
2405 	}
2406 
2407 	/* Execute booter to handle (eventually...) booting GSP-RM. */
2408 	ret = r535_gsp_booter_load(gsp, mbox0, mbox1);
2409 	if (WARN_ON(ret))
2410 		goto done;
2411 
2412 	ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE);
2413 	if (ret)
2414 		goto done;
2415 
2416 	gsp->running = true;
2417 
2418 done:
2419 	if (gsp->sr.meta.data) {
2420 		nvkm_gsp_mem_dtor(gsp, &gsp->sr.meta);
2421 		nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);
2422 		nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);
2423 		return ret;
2424 	}
2425 
2426 	if (ret == 0)
2427 		ret = r535_gsp_postinit(gsp);
2428 
2429 	return ret;
2430 }
2431 
2432 static int
r535_gsp_rm_boot_ctor(struct nvkm_gsp * gsp)2433 r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp)
2434 {
2435 	const struct firmware *fw = gsp->fws.bl;
2436 	const struct nvfw_bin_hdr *hdr;
2437 	RM_RISCV_UCODE_DESC *desc;
2438 	int ret;
2439 
2440 	hdr = nvfw_bin_hdr(&gsp->subdev, fw->data);
2441 	desc = (void *)fw->data + hdr->header_offset;
2442 
2443 	ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw);
2444 	if (ret)
2445 		return ret;
2446 
2447 	memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size);
2448 
2449 	gsp->boot.code_offset = desc->monitorCodeOffset;
2450 	gsp->boot.data_offset = desc->monitorDataOffset;
2451 	gsp->boot.manifest_offset = desc->manifestOffset;
2452 	gsp->boot.app_version = desc->appVersion;
2453 	return 0;
2454 }
2455 
2456 static const struct nvkm_firmware_func
2457 r535_gsp_fw = {
2458 	.type = NVKM_FIRMWARE_IMG_SGT,
2459 };
2460 
2461 static int
r535_gsp_elf_section(struct nvkm_gsp * gsp,const char * name,const u8 ** pdata,u64 * psize)2462 r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize)
2463 {
2464 	const u8 *img = gsp->fws.rm->data;
2465 	const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img;
2466 	const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff];
2467 	const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset];
2468 
2469 	for (int i = 0; i < ehdr->e_shnum; i++, shdr++) {
2470 		if (!strcmp(&names[shdr->sh_name], name)) {
2471 			*pdata = &img[shdr->sh_offset];
2472 			*psize = shdr->sh_size;
2473 			return 0;
2474 		}
2475 	}
2476 
2477 	nvkm_error(&gsp->subdev, "section '%s' not found\n", name);
2478 	return -ENOENT;
2479 }
2480 
2481 static void
r535_gsp_dtor_fws(struct nvkm_gsp * gsp)2482 r535_gsp_dtor_fws(struct nvkm_gsp *gsp)
2483 {
2484 	nvkm_firmware_put(gsp->fws.bl);
2485 	gsp->fws.bl = NULL;
2486 	nvkm_firmware_put(gsp->fws.booter.unload);
2487 	gsp->fws.booter.unload = NULL;
2488 	nvkm_firmware_put(gsp->fws.booter.load);
2489 	gsp->fws.booter.load = NULL;
2490 	nvkm_firmware_put(gsp->fws.rm);
2491 	gsp->fws.rm = NULL;
2492 }
2493 
2494 void
r535_gsp_dtor(struct nvkm_gsp * gsp)2495 r535_gsp_dtor(struct nvkm_gsp *gsp)
2496 {
2497 	idr_destroy(&gsp->client_id.idr);
2498 	mutex_destroy(&gsp->client_id.mutex);
2499 
2500 	nvkm_gsp_radix3_dtor(gsp, &gsp->radix3);
2501 	nvkm_gsp_mem_dtor(gsp, &gsp->sig);
2502 	nvkm_firmware_dtor(&gsp->fw);
2503 
2504 	nvkm_falcon_fw_dtor(&gsp->booter.unload);
2505 	nvkm_falcon_fw_dtor(&gsp->booter.load);
2506 
2507 	mutex_destroy(&gsp->msgq.mutex);
2508 	mutex_destroy(&gsp->cmdq.mutex);
2509 
2510 	r535_gsp_dtor_fws(gsp);
2511 
2512 	nvkm_gsp_mem_dtor(gsp, &gsp->rmargs);
2513 	nvkm_gsp_mem_dtor(gsp, &gsp->wpr_meta);
2514 	nvkm_gsp_mem_dtor(gsp, &gsp->shm.mem);
2515 	nvkm_gsp_mem_dtor(gsp, &gsp->loginit);
2516 	nvkm_gsp_mem_dtor(gsp, &gsp->logintr);
2517 	nvkm_gsp_mem_dtor(gsp, &gsp->logrm);
2518 }
2519 
2520 int
r535_gsp_oneinit(struct nvkm_gsp * gsp)2521 r535_gsp_oneinit(struct nvkm_gsp *gsp)
2522 {
2523 	struct nvkm_device *device = gsp->subdev.device;
2524 	const u8 *data;
2525 	u64 size;
2526 	int ret;
2527 
2528 	mutex_init(&gsp->cmdq.mutex);
2529 	mutex_init(&gsp->msgq.mutex);
2530 
2531 	ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load,
2532 				     &device->sec2->falcon, &gsp->booter.load);
2533 	if (ret)
2534 		return ret;
2535 
2536 	ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload,
2537 				     &device->sec2->falcon, &gsp->booter.unload);
2538 	if (ret)
2539 		return ret;
2540 
2541 	/* Load GSP firmware from ELF image into DMA-accessible memory. */
2542 	ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size);
2543 	if (ret)
2544 		return ret;
2545 
2546 	ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw);
2547 	if (ret)
2548 		return ret;
2549 
2550 	/* Load relevant signature from ELF image. */
2551 	ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size);
2552 	if (ret)
2553 		return ret;
2554 
2555 	ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig);
2556 	if (ret)
2557 		return ret;
2558 
2559 	memcpy(gsp->sig.data, data, size);
2560 
2561 	/* Build radix3 page table for ELF image. */
2562 	ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3);
2563 	if (ret)
2564 		return ret;
2565 
2566 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,
2567 			      r535_gsp_msg_run_cpu_sequencer, gsp);
2568 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp);
2569 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED,
2570 			      r535_gsp_msg_rc_triggered, gsp);
2571 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,
2572 			      r535_gsp_msg_mmu_fault_queued, gsp);
2573 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp);
2574 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL);
2575 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL);
2576 	r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL);
2577 	ret = r535_gsp_rm_boot_ctor(gsp);
2578 	if (ret)
2579 		return ret;
2580 
2581 	/* Release FW images - we've copied them to DMA buffers now. */
2582 	r535_gsp_dtor_fws(gsp);
2583 
2584 	/* Calculate FB layout. */
2585 	gsp->fb.wpr2.frts.size = 0x100000;
2586 	gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;
2587 
2588 	gsp->fb.wpr2.boot.size = gsp->boot.fw.size;
2589 	gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000);
2590 
2591 	gsp->fb.wpr2.elf.size = gsp->fw.len;
2592 	gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000);
2593 
2594 	{
2595 		u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30);
2596 
2597 		gsp->fb.wpr2.heap.size =
2598 			gsp->func->wpr_heap.os_carveout_size +
2599 			gsp->func->wpr_heap.base_size +
2600 			ALIGN(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fb_size_gb, 1 << 20) +
2601 			ALIGN(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20);
2602 
2603 		gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size);
2604 	}
2605 
2606 	gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000);
2607 	gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000);
2608 
2609 	gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000);
2610 	gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr;
2611 
2612 	gsp->fb.heap.size = 0x100000;
2613 	gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size;
2614 
2615 	ret = nvkm_gsp_fwsec_frts(gsp);
2616 	if (WARN_ON(ret))
2617 		return ret;
2618 
2619 	ret = r535_gsp_libos_init(gsp);
2620 	if (WARN_ON(ret))
2621 		return ret;
2622 
2623 	ret = r535_gsp_wpr_meta_init(gsp);
2624 	if (WARN_ON(ret))
2625 		return ret;
2626 
2627 	ret = r535_gsp_rpc_set_system_info(gsp);
2628 	if (WARN_ON(ret))
2629 		return ret;
2630 
2631 	ret = r535_gsp_rpc_set_registry(gsp);
2632 	if (WARN_ON(ret))
2633 		return ret;
2634 
2635 	/* Reset GSP into RISC-V mode. */
2636 	ret = gsp->func->reset(gsp);
2637 	if (WARN_ON(ret))
2638 		return ret;
2639 
2640 	nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
2641 	nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
2642 
2643 	mutex_init(&gsp->client_id.mutex);
2644 	idr_init(&gsp->client_id.idr);
2645 	return 0;
2646 }
2647 
2648 static int
r535_gsp_load_fw(struct nvkm_gsp * gsp,const char * name,const char * ver,const struct firmware ** pfw)2649 r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver,
2650 		 const struct firmware **pfw)
2651 {
2652 	char fwname[64];
2653 
2654 	snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver);
2655 	return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw);
2656 }
2657 
2658 int
r535_gsp_load(struct nvkm_gsp * gsp,int ver,const struct nvkm_gsp_fwif * fwif)2659 r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif)
2660 {
2661 	struct nvkm_subdev *subdev = &gsp->subdev;
2662 	int ret;
2663 	bool enable_gsp = fwif->enable;
2664 
2665 #if IS_ENABLED(CONFIG_DRM_NOUVEAU_GSP_DEFAULT)
2666 	enable_gsp = true;
2667 #endif
2668 	if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", enable_gsp))
2669 		return -EINVAL;
2670 
2671 	if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) ||
2672 	    (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) ||
2673 	    (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) ||
2674 	    (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) {
2675 		r535_gsp_dtor_fws(gsp);
2676 		return ret;
2677 	}
2678 
2679 	return 0;
2680 }
2681 
2682 #define NVKM_GSP_FIRMWARE(chip)                                  \
2683 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin");   \
2684 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \
2685 MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin");    \
2686 MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin")
2687 
2688 NVKM_GSP_FIRMWARE(tu102);
2689 NVKM_GSP_FIRMWARE(tu104);
2690 NVKM_GSP_FIRMWARE(tu106);
2691 
2692 NVKM_GSP_FIRMWARE(tu116);
2693 NVKM_GSP_FIRMWARE(tu117);
2694 
2695 NVKM_GSP_FIRMWARE(ga100);
2696 
2697 NVKM_GSP_FIRMWARE(ga102);
2698 NVKM_GSP_FIRMWARE(ga103);
2699 NVKM_GSP_FIRMWARE(ga104);
2700 NVKM_GSP_FIRMWARE(ga106);
2701 NVKM_GSP_FIRMWARE(ga107);
2702 
2703 NVKM_GSP_FIRMWARE(ad102);
2704 NVKM_GSP_FIRMWARE(ad103);
2705 NVKM_GSP_FIRMWARE(ad104);
2706 NVKM_GSP_FIRMWARE(ad106);
2707 NVKM_GSP_FIRMWARE(ad107);
2708