1 /*
2 * Copyright 2023 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include <rm/rpc.h>
23
24 #include "priv.h"
25
26 #include <core/pci.h>
27 #include <subdev/pci/priv.h>
28 #include <subdev/timer.h>
29 #include <subdev/vfn.h>
30 #include <engine/fifo/chan.h>
31 #include <engine/sec2.h>
32 #include <nvif/log.h>
33
34 #include <nvfw/fw.h>
35
36 #include "nvrm/gsp.h"
37 #include "nvrm/rpcfn.h"
38 #include "nvrm/msgfn.h"
39 #include "nvrm/event.h"
40 #include "nvrm/fifo.h"
41
42 #include <linux/acpi.h>
43 #include <linux/ctype.h>
44 #include <linux/parser.h>
45
46 extern struct dentry *nouveau_debugfs_root;
47
48 static void
r535_gsp_msgq_work(struct work_struct * work)49 r535_gsp_msgq_work(struct work_struct *work)
50 {
51 struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work);
52
53 mutex_lock(&gsp->cmdq.mutex);
54 if (*gsp->msgq.rptr != *gsp->msgq.wptr)
55 r535_gsp_msg_recv(gsp, 0, 0);
56 mutex_unlock(&gsp->cmdq.mutex);
57 }
58
59 static irqreturn_t
r535_gsp_intr(struct nvkm_inth * inth)60 r535_gsp_intr(struct nvkm_inth *inth)
61 {
62 struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth);
63 struct nvkm_subdev *subdev = &gsp->subdev;
64 u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008);
65 u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 +
66 gsp->falcon.func->riscv_irqmask);
67 u32 stat = intr & inte;
68
69 if (!stat) {
70 nvkm_debug(subdev, "inte %08x %08x\n", intr, inte);
71 return IRQ_NONE;
72 }
73
74 if (stat & 0x00000040) {
75 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040);
76 schedule_work(&gsp->msgq.work);
77 stat &= ~0x00000040;
78 }
79
80 if (stat) {
81 nvkm_error(subdev, "intr %08x\n", stat);
82 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat);
83 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat);
84 }
85
86 nvkm_falcon_intr_retrigger(&gsp->falcon);
87 return IRQ_HANDLED;
88 }
89
90 static bool
r535_gsp_xlat_mc_engine_idx(u32 mc_engine_idx,enum nvkm_subdev_type * ptype,int * pinst)91 r535_gsp_xlat_mc_engine_idx(u32 mc_engine_idx, enum nvkm_subdev_type *ptype, int *pinst)
92 {
93 switch (mc_engine_idx) {
94 case MC_ENGINE_IDX_GSP:
95 *ptype = NVKM_SUBDEV_GSP;
96 *pinst = 0;
97 return true;
98 case MC_ENGINE_IDX_DISP:
99 *ptype = NVKM_ENGINE_DISP;
100 *pinst = 0;
101 return true;
102 case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9:
103 *ptype = NVKM_ENGINE_CE;
104 *pinst = mc_engine_idx - MC_ENGINE_IDX_CE0;
105 return true;
106 case MC_ENGINE_IDX_GR0:
107 *ptype = NVKM_ENGINE_GR;
108 *pinst = 0;
109 return true;
110 case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7:
111 *ptype = NVKM_ENGINE_NVDEC;
112 *pinst = mc_engine_idx - MC_ENGINE_IDX_NVDEC0;
113 return true;
114 case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2:
115 *ptype = NVKM_ENGINE_NVENC;
116 *pinst = mc_engine_idx - MC_ENGINE_IDX_MSENC;
117 return true;
118 case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7:
119 *ptype = NVKM_ENGINE_NVJPG;
120 *pinst = mc_engine_idx - MC_ENGINE_IDX_NVJPEG0;
121 return true;
122 case MC_ENGINE_IDX_OFA0:
123 *ptype = NVKM_ENGINE_OFA;
124 *pinst = 0;
125 return true;
126 default:
127 return false;
128 }
129 }
130
131 static int
r535_gsp_intr_get_table(struct nvkm_gsp * gsp)132 r535_gsp_intr_get_table(struct nvkm_gsp *gsp)
133 {
134 NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl;
135 const struct nvkm_rm_api *rmapi = gsp->rm->api;
136 int ret = 0;
137
138 ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,
139 NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl));
140 if (IS_ERR(ctrl))
141 return PTR_ERR(ctrl);
142
143 ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl));
144 if (WARN_ON(ret)) {
145 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
146 return ret;
147 }
148
149 for (unsigned i = 0; i < ctrl->tableLen; i++) {
150 enum nvkm_subdev_type type;
151 int inst;
152
153 nvkm_debug(&gsp->subdev,
154 "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i,
155 ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask,
156 ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall);
157
158 if (!rmapi->gsp->xlat_mc_engine_idx(ctrl->table[i].engineIdx, &type, &inst))
159 continue;
160
161 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) {
162 ret = -ENOSPC;
163 break;
164 }
165
166 gsp->intr[gsp->intr_nr].type = type;
167 gsp->intr[gsp->intr_nr].inst = inst;
168 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall;
169 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall;
170 gsp->intr_nr++;
171 }
172
173 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
174 return ret;
175 }
176
177 void
r535_gsp_get_static_info_fb(struct nvkm_gsp * gsp,const struct NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS * info)178 r535_gsp_get_static_info_fb(struct nvkm_gsp *gsp,
179 const struct NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS *info)
180 {
181 int last_usable = -1;
182
183 for (int i = 0; i < info->numFBRegions; i++) {
184 const NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg = &info->fbRegion[i];
185
186 nvkm_debug(&gsp->subdev, "fb region %d: "
187 "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i,
188 reg->base, reg->limit, reg->reserved, reg->performance,
189 reg->supportCompressed, reg->supportISO, reg->bProtected);
190
191 if (!reg->reserved && !reg->bProtected) {
192 if (reg->supportCompressed && reg->supportISO &&
193 !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) {
194 const u64 size = (reg->limit + 1) - reg->base;
195
196 gsp->fb.region[gsp->fb.region_nr].addr = reg->base;
197 gsp->fb.region[gsp->fb.region_nr].size = size;
198 gsp->fb.region_nr++;
199 }
200
201 last_usable = i;
202 }
203 }
204
205 if (last_usable >= 0) {
206 u32 rsvd_base = info->fbRegion[last_usable].limit + 1;
207
208 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base;
209 }
210 }
211
212 static int
r535_gsp_get_static_info(struct nvkm_gsp * gsp)213 r535_gsp_get_static_info(struct nvkm_gsp *gsp)
214 {
215 GspStaticConfigInfo *rpc;
216
217 rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc));
218 if (IS_ERR(rpc))
219 return PTR_ERR(rpc);
220
221 gsp->internal.client.object.client = &gsp->internal.client;
222 gsp->internal.client.object.parent = NULL;
223 gsp->internal.client.object.handle = rpc->hInternalClient;
224 gsp->internal.client.gsp = gsp;
225
226 gsp->internal.device.object.client = &gsp->internal.client;
227 gsp->internal.device.object.parent = &gsp->internal.client.object;
228 gsp->internal.device.object.handle = rpc->hInternalDevice;
229
230 gsp->internal.device.subdevice.client = &gsp->internal.client;
231 gsp->internal.device.subdevice.parent = &gsp->internal.device.object;
232 gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice;
233
234 gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase;
235 gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase;
236
237 r535_gsp_get_static_info_fb(gsp, &rpc->fbRegionInfoParams);
238
239 for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) {
240 if (rpc->gpcInfo.gpcMask & BIT(gpc)) {
241 gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask);
242 gsp->gr.gpcs++;
243 }
244 }
245
246 nvkm_gsp_rpc_done(gsp, rpc);
247 return 0;
248 }
249
250 void
nvkm_gsp_mem_dtor(struct nvkm_gsp_mem * mem)251 nvkm_gsp_mem_dtor(struct nvkm_gsp_mem *mem)
252 {
253 if (mem->data) {
254 /*
255 * Poison the buffer to catch any unexpected access from
256 * GSP-RM if the buffer was prematurely freed.
257 */
258 memset(mem->data, 0xFF, mem->size);
259
260 dma_free_coherent(mem->dev, mem->size, mem->data, mem->addr);
261 put_device(mem->dev);
262
263 memset(mem, 0, sizeof(*mem));
264 }
265 }
266
267 /**
268 * nvkm_gsp_mem_ctor - constructor for nvkm_gsp_mem objects
269 * @gsp: gsp pointer
270 * @size: number of bytes to allocate
271 * @mem: nvkm_gsp_mem object to initialize
272 *
273 * Allocates a block of memory for use with GSP.
274 *
275 * This memory block can potentially out-live the driver's remove() callback,
276 * so we take a device reference to ensure its lifetime. The reference is
277 * dropped in the destructor.
278 */
279 int
nvkm_gsp_mem_ctor(struct nvkm_gsp * gsp,size_t size,struct nvkm_gsp_mem * mem)280 nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem)
281 {
282 mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL);
283 if (WARN_ON(!mem->data))
284 return -ENOMEM;
285
286 mem->size = size;
287 mem->dev = get_device(gsp->subdev.device->dev);
288
289 return 0;
290 }
291
292 static int
r535_gsp_postinit(struct nvkm_gsp * gsp)293 r535_gsp_postinit(struct nvkm_gsp *gsp)
294 {
295 struct nvkm_device *device = gsp->subdev.device;
296 const struct nvkm_rm_api *rmapi = gsp->rm->api;
297 int ret;
298
299 ret = rmapi->gsp->get_static_info(gsp);
300 if (WARN_ON(ret))
301 return ret;
302
303 INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work);
304
305 ret = r535_gsp_intr_get_table(gsp);
306 if (WARN_ON(ret))
307 return ret;
308
309 ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst);
310 if (WARN_ON(ret < 0))
311 return ret;
312
313 ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev,
314 r535_gsp_intr, &gsp->subdev.inth);
315 if (WARN_ON(ret))
316 return ret;
317
318 nvkm_inth_allow(&gsp->subdev.inth);
319 nvkm_wr32(device, 0x110004, 0x00000040);
320
321 /* Release the DMA buffers that were needed only for boot and init */
322 nvkm_gsp_mem_dtor(&gsp->boot.fw);
323 nvkm_gsp_mem_dtor(&gsp->libos);
324
325 return ret;
326 }
327
328 static int
r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp * gsp,bool suspend)329 r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend)
330 {
331 rpc_unloading_guest_driver_v1F_07 *rpc;
332
333 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc));
334 if (IS_ERR(rpc))
335 return PTR_ERR(rpc);
336
337 if (suspend) {
338 rpc->bInPMTransition = 1;
339 rpc->bGc6Entering = 0;
340 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
341 } else {
342 rpc->bInPMTransition = 0;
343 rpc->bGc6Entering = 0;
344 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0;
345 }
346
347 return nvkm_gsp_rpc_wr(gsp, rpc, NVKM_GSP_RPC_REPLY_RECV);
348 }
349
350 enum registry_type {
351 REGISTRY_TABLE_ENTRY_TYPE_DWORD = 1, /* 32-bit unsigned integer */
352 REGISTRY_TABLE_ENTRY_TYPE_BINARY = 2, /* Binary blob */
353 REGISTRY_TABLE_ENTRY_TYPE_STRING = 3, /* Null-terminated string */
354 };
355
356 /* An arbitrary limit to the length of a registry key */
357 #define REGISTRY_MAX_KEY_LENGTH 64
358
359 /**
360 * struct registry_list_entry - linked list member for a registry key/value
361 * @head: list_head struct
362 * @type: dword, binary, or string
363 * @klen: the length of name of the key
364 * @vlen: the length of the value
365 * @key: the key name
366 * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD
367 * @binary: the data, if TYPE_BINARY or TYPE_STRING
368 *
369 * Every registry key/value is represented internally by this struct.
370 *
371 * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in
372 * @dword.
373 *
374 * Types BINARY and STRING are variable-length binary blobs. The only real
375 * difference between BINARY and STRING is that STRING is null-terminated and
376 * is expected to contain only printable characters.
377 *
378 * Note: it is technically possible to have multiple keys with the same name
379 * but different types, but this is not useful since GSP-RM expects keys to
380 * have only one specific type.
381 */
382 struct registry_list_entry {
383 struct list_head head;
384 enum registry_type type;
385 size_t klen;
386 char key[REGISTRY_MAX_KEY_LENGTH];
387 size_t vlen;
388 u32 dword; /* TYPE_DWORD */
389 u8 binary[] __counted_by(vlen); /* TYPE_BINARY or TYPE_STRING */
390 };
391
392 /**
393 * add_registry -- adds a registry entry
394 * @gsp: gsp pointer
395 * @key: name of the registry key
396 * @type: type of data
397 * @data: pointer to value
398 * @length: size of data, in bytes
399 *
400 * Adds a registry key/value pair to the registry database.
401 *
402 * This function collects the registry information in a linked list. After
403 * all registry keys have been added, build_registry() is used to create the
404 * RPC data structure.
405 *
406 * registry_rpc_size is a running total of the size of all registry keys.
407 * It's used to avoid an O(n) calculation of the size when the RPC is built.
408 *
409 * Returns 0 on success, or negative error code on error.
410 */
add_registry(struct nvkm_gsp * gsp,const char * key,enum registry_type type,const void * data,size_t length)411 static int add_registry(struct nvkm_gsp *gsp, const char *key,
412 enum registry_type type, const void *data, size_t length)
413 {
414 struct registry_list_entry *reg;
415 const size_t nlen = strnlen(key, REGISTRY_MAX_KEY_LENGTH) + 1;
416 size_t alloc_size; /* extra bytes to alloc for binary or string value */
417
418 if (nlen > REGISTRY_MAX_KEY_LENGTH)
419 return -EINVAL;
420
421 alloc_size = (type == REGISTRY_TABLE_ENTRY_TYPE_DWORD) ? 0 : length;
422
423 reg = kmalloc(sizeof(*reg) + alloc_size, GFP_KERNEL);
424 if (!reg)
425 return -ENOMEM;
426
427 switch (type) {
428 case REGISTRY_TABLE_ENTRY_TYPE_DWORD:
429 reg->dword = *(const u32 *)(data);
430 break;
431 case REGISTRY_TABLE_ENTRY_TYPE_BINARY:
432 case REGISTRY_TABLE_ENTRY_TYPE_STRING:
433 memcpy(reg->binary, data, alloc_size);
434 break;
435 default:
436 nvkm_error(&gsp->subdev, "unrecognized registry type %u for '%s'\n",
437 type, key);
438 kfree(reg);
439 return -EINVAL;
440 }
441
442 memcpy(reg->key, key, nlen);
443 reg->klen = nlen;
444 reg->vlen = length;
445 reg->type = type;
446
447 list_add_tail(®->head, &gsp->registry_list);
448 gsp->registry_rpc_size += sizeof(PACKED_REGISTRY_ENTRY) + nlen + alloc_size;
449
450 return 0;
451 }
452
add_registry_num(struct nvkm_gsp * gsp,const char * key,u32 value)453 static int add_registry_num(struct nvkm_gsp *gsp, const char *key, u32 value)
454 {
455 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_DWORD,
456 &value, sizeof(u32));
457 }
458
add_registry_string(struct nvkm_gsp * gsp,const char * key,const char * value)459 static int add_registry_string(struct nvkm_gsp *gsp, const char *key, const char *value)
460 {
461 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_STRING,
462 value, strlen(value) + 1);
463 }
464
465 /**
466 * build_registry -- create the registry RPC data
467 * @gsp: gsp pointer
468 * @registry: pointer to the RPC payload to fill
469 *
470 * After all registry key/value pairs have been added, call this function to
471 * build the RPC.
472 *
473 * The registry RPC looks like this:
474 *
475 * +-----------------+
476 * |NvU32 size; |
477 * |NvU32 numEntries;|
478 * +-----------------+
479 * +----------------------------------------+
480 * |PACKED_REGISTRY_ENTRY |
481 * +----------------------------------------+
482 * |Null-terminated key (string) for entry 0|
483 * +----------------------------------------+
484 * |Binary/string data value for entry 0 | (only if necessary)
485 * +----------------------------------------+
486 *
487 * +----------------------------------------+
488 * |PACKED_REGISTRY_ENTRY |
489 * +----------------------------------------+
490 * |Null-terminated key (string) for entry 1|
491 * +----------------------------------------+
492 * |Binary/string data value for entry 1 | (only if necessary)
493 * +----------------------------------------+
494 * ... (and so on, one copy for each entry)
495 *
496 *
497 * The 'data' field of an entry is either a 32-bit integer (for type DWORD)
498 * or an offset into the PACKED_REGISTRY_TABLE (for types BINARY and STRING).
499 *
500 * All memory allocated by add_registry() is released.
501 */
build_registry(struct nvkm_gsp * gsp,PACKED_REGISTRY_TABLE * registry)502 static void build_registry(struct nvkm_gsp *gsp, PACKED_REGISTRY_TABLE *registry)
503 {
504 struct registry_list_entry *reg, *n;
505 size_t str_offset;
506 unsigned int i = 0;
507
508 registry->numEntries = list_count_nodes(&gsp->registry_list);
509 str_offset = struct_size(registry, entries, registry->numEntries);
510
511 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {
512 registry->entries[i].type = reg->type;
513 registry->entries[i].length = reg->vlen;
514
515 /* Append the key name to the table */
516 registry->entries[i].nameOffset = str_offset;
517 memcpy((void *)registry + str_offset, reg->key, reg->klen);
518 str_offset += reg->klen;
519
520 switch (reg->type) {
521 case REGISTRY_TABLE_ENTRY_TYPE_DWORD:
522 registry->entries[i].data = reg->dword;
523 break;
524 case REGISTRY_TABLE_ENTRY_TYPE_BINARY:
525 case REGISTRY_TABLE_ENTRY_TYPE_STRING:
526 /* If the type is binary or string, also append the value */
527 memcpy((void *)registry + str_offset, reg->binary, reg->vlen);
528 registry->entries[i].data = str_offset;
529 str_offset += reg->vlen;
530 break;
531 default:
532 break;
533 }
534
535 i++;
536 list_del(®->head);
537 kfree(reg);
538 }
539
540 /* Double-check that we calculated the sizes correctly */
541 WARN_ON(gsp->registry_rpc_size != str_offset);
542
543 registry->size = gsp->registry_rpc_size;
544 }
545
546 /**
547 * clean_registry -- clean up registry memory in case of error
548 * @gsp: gsp pointer
549 *
550 * Call this function to clean up all memory allocated by add_registry()
551 * in case of error and build_registry() is not called.
552 */
clean_registry(struct nvkm_gsp * gsp)553 static void clean_registry(struct nvkm_gsp *gsp)
554 {
555 struct registry_list_entry *reg, *n;
556
557 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {
558 list_del(®->head);
559 kfree(reg);
560 }
561
562 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);
563 }
564
565 MODULE_PARM_DESC(NVreg_RegistryDwords,
566 "A semicolon-separated list of key=integer pairs of GSP-RM registry keys");
567 static char *NVreg_RegistryDwords;
568 module_param(NVreg_RegistryDwords, charp, 0400);
569
570 /* dword only */
571 struct nv_gsp_registry_entries {
572 const char *name;
573 u32 value;
574 };
575
576 /*
577 * r535_registry_entries - required registry entries for GSP-RM
578 *
579 * This array lists registry entries that are required for GSP-RM to
580 * function correctly.
581 *
582 * RMSecBusResetEnable - enables PCI secondary bus reset
583 * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration
584 * registers on any PCI reset.
585 * RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID
586 * is not found in the internal product name database.
587 */
588 static const struct nv_gsp_registry_entries r535_registry_entries[] = {
589 { "RMSecBusResetEnable", 1 },
590 { "RMForcePcieConfigSave", 1 },
591 { "RMDevidCheckIgnore", 1 },
592 };
593 #define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries)
594
595 /**
596 * strip - strips all characters in 'reject' from 's'
597 * @s: string to strip
598 * @reject: string of characters to remove
599 *
600 * 's' is modified.
601 *
602 * Returns the length of the new string.
603 */
strip(char * s,const char * reject)604 static size_t strip(char *s, const char *reject)
605 {
606 char *p = s, *p2 = s;
607 size_t length = 0;
608 char c;
609
610 do {
611 while ((c = *p2) && strchr(reject, c))
612 p2++;
613
614 *p++ = c = *p2++;
615 length++;
616 } while (c);
617
618 return length;
619 }
620
621 /**
622 * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM
623 * @gsp: gsp pointer
624 *
625 * The GSP-RM registry is a set of key/value pairs that configure some aspects
626 * of GSP-RM. The keys are strings, and the values are 32-bit integers.
627 *
628 * The registry is built from a combination of a static hard-coded list (see
629 * above) and entries passed on the driver's command line.
630 */
631 static int
r535_gsp_rpc_set_registry(struct nvkm_gsp * gsp)632 r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp)
633 {
634 PACKED_REGISTRY_TABLE *rpc;
635 unsigned int i;
636 int ret;
637
638 INIT_LIST_HEAD(&gsp->registry_list);
639 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);
640
641 for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) {
642 ret = add_registry_num(gsp, r535_registry_entries[i].name,
643 r535_registry_entries[i].value);
644 if (ret)
645 goto fail;
646 }
647
648 /*
649 * The NVreg_RegistryDwords parameter is a string of key=value
650 * pairs separated by semicolons. We need to extract and trim each
651 * substring, and then parse the substring to extract the key and
652 * value.
653 */
654 if (NVreg_RegistryDwords) {
655 char *p = kstrdup(NVreg_RegistryDwords, GFP_KERNEL);
656 char *start, *next = p, *equal;
657
658 if (!p) {
659 ret = -ENOMEM;
660 goto fail;
661 }
662
663 /* Remove any whitespace from the parameter string */
664 strip(p, " \t\n");
665
666 while ((start = strsep(&next, ";"))) {
667 long value;
668
669 equal = strchr(start, '=');
670 if (!equal || equal == start || equal[1] == 0) {
671 nvkm_error(&gsp->subdev,
672 "ignoring invalid registry string '%s'\n",
673 start);
674 continue;
675 }
676
677 /* Truncate the key=value string to just key */
678 *equal = 0;
679
680 ret = kstrtol(equal + 1, 0, &value);
681 if (!ret) {
682 ret = add_registry_num(gsp, start, value);
683 } else {
684 /* Not a number, so treat it as a string */
685 ret = add_registry_string(gsp, start, equal + 1);
686 }
687
688 if (ret) {
689 nvkm_error(&gsp->subdev,
690 "ignoring invalid registry key/value '%s=%s'\n",
691 start, equal + 1);
692 continue;
693 }
694 }
695
696 kfree(p);
697 }
698
699 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, gsp->registry_rpc_size);
700 if (IS_ERR(rpc)) {
701 ret = PTR_ERR(rpc);
702 goto fail;
703 }
704
705 build_registry(gsp, rpc);
706
707 return nvkm_gsp_rpc_wr(gsp, rpc, NVKM_GSP_RPC_REPLY_NOWAIT);
708
709 fail:
710 clean_registry(gsp);
711 return ret;
712 }
713
714 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
715 void
r535_gsp_acpi_caps(acpi_handle handle,CAPS_METHOD_DATA * caps)716 r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps)
717 {
718 const guid_t NVOP_DSM_GUID =
719 GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B,
720 0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0);
721 u64 NVOP_DSM_REV = 0x00000100;
722 union acpi_object argv4 = {
723 .buffer.type = ACPI_TYPE_BUFFER,
724 .buffer.length = 4,
725 }, *obj;
726
727 caps->status = 0xffff;
728
729 if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a)))
730 return;
731
732 argv4.buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL);
733 if (!argv4.buffer.pointer)
734 return;
735
736 obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4);
737 if (!obj)
738 goto done;
739
740 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
741 WARN_ON(obj->buffer.length != 4))
742 goto done;
743
744 caps->status = 0;
745 caps->optimusCaps = *(u32 *)obj->buffer.pointer;
746
747 done:
748 ACPI_FREE(obj);
749
750 kfree(argv4.buffer.pointer);
751 }
752
753 void
r535_gsp_acpi_jt(acpi_handle handle,JT_METHOD_DATA * jt)754 r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt)
755 {
756 const guid_t JT_DSM_GUID =
757 GUID_INIT(0xCBECA351L, 0x067B, 0x4924,
758 0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34);
759 u64 JT_DSM_REV = 0x00000103;
760 u32 caps;
761 union acpi_object argv4 = {
762 .buffer.type = ACPI_TYPE_BUFFER,
763 .buffer.length = sizeof(caps),
764 }, *obj;
765
766 jt->status = 0xffff;
767
768 argv4.buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL);
769 if (!argv4.buffer.pointer)
770 return;
771
772 obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4);
773 if (!obj)
774 goto done;
775
776 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
777 WARN_ON(obj->buffer.length != 4))
778 goto done;
779
780 jt->status = 0;
781 jt->jtCaps = *(u32 *)obj->buffer.pointer;
782 jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20;
783 jt->bSBIOSCaps = 0;
784
785 done:
786 ACPI_FREE(obj);
787
788 kfree(argv4.buffer.pointer);
789 }
790
791 static void
r535_gsp_acpi_mux_id(acpi_handle handle,u32 id,MUX_METHOD_DATA_ELEMENT * mode,MUX_METHOD_DATA_ELEMENT * part)792 r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
793 MUX_METHOD_DATA_ELEMENT *part)
794 {
795 union acpi_object mux_arg = { ACPI_TYPE_INTEGER };
796 struct acpi_object_list input = { 1, &mux_arg };
797 acpi_handle iter = NULL, handle_mux = NULL;
798 acpi_status status;
799 unsigned long long value;
800
801 mode->status = 0xffff;
802 part->status = 0xffff;
803
804 do {
805 status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter);
806 if (ACPI_FAILURE(status) || !iter)
807 return;
808
809 status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);
810 if (ACPI_FAILURE(status) || value != id)
811 continue;
812
813 handle_mux = iter;
814 } while (!handle_mux);
815
816 if (!handle_mux)
817 return;
818
819 /* I -think- 0 means "acquire" according to nvidia's driver source */
820 input.pointer->integer.type = ACPI_TYPE_INTEGER;
821 input.pointer->integer.value = 0;
822
823 status = acpi_evaluate_integer(handle_mux, "MXDM", &input, &value);
824 if (ACPI_SUCCESS(status)) {
825 mode->acpiId = id;
826 mode->mode = value;
827 mode->status = 0;
828 }
829
830 status = acpi_evaluate_integer(handle_mux, "MXDS", &input, &value);
831 if (ACPI_SUCCESS(status)) {
832 part->acpiId = id;
833 part->mode = value;
834 part->status = 0;
835 }
836 }
837
838 static void
r535_gsp_acpi_mux(acpi_handle handle,DOD_METHOD_DATA * dod,MUX_METHOD_DATA * mux)839 r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux)
840 {
841 mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]);
842
843 for (int i = 0; i < mux->tableLen; i++) {
844 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i],
845 &mux->acpiIdMuxPartTable[i]);
846 }
847 }
848
849 void
r535_gsp_acpi_dod(acpi_handle handle,DOD_METHOD_DATA * dod)850 r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod)
851 {
852 acpi_status status;
853 struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
854 union acpi_object *_DOD;
855
856 dod->status = 0xffff;
857
858 status = acpi_evaluate_object(handle, "_DOD", NULL, &output);
859 if (ACPI_FAILURE(status))
860 return;
861
862 _DOD = output.pointer;
863
864 if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) ||
865 WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList)))
866 return;
867
868 for (int i = 0; i < _DOD->package.count; i++) {
869 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER))
870 return;
871
872 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value;
873 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]);
874 }
875
876 dod->status = 0;
877 kfree(output.pointer);
878 }
879 #endif
880
881 static void
r535_gsp_acpi_info(struct nvkm_gsp * gsp,ACPI_METHOD_DATA * acpi)882 r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi)
883 {
884 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
885 acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev);
886
887 if (!handle)
888 return;
889
890 acpi->bValid = 1;
891
892 r535_gsp_acpi_dod(handle, &acpi->dodMethodData);
893 if (acpi->dodMethodData.status == 0)
894 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData);
895
896 r535_gsp_acpi_jt(handle, &acpi->jtMethodData);
897 r535_gsp_acpi_caps(handle, &acpi->capsMethodData);
898 #endif
899 }
900
901 static int
r535_gsp_set_system_info(struct nvkm_gsp * gsp)902 r535_gsp_set_system_info(struct nvkm_gsp *gsp)
903 {
904 struct nvkm_device *device = gsp->subdev.device;
905 struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device);
906 GspSystemInfo *info;
907
908 if (WARN_ON(device->type == NVKM_DEVICE_TEGRA))
909 return -ENOSYS;
910
911 info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info));
912 if (IS_ERR(info))
913 return PTR_ERR(info);
914
915 info->gpuPhysAddr = device->func->resource_addr(device, NVKM_BAR0_PRI);
916 info->gpuPhysFbAddr = device->func->resource_addr(device, NVKM_BAR1_FB);
917 info->gpuPhysInstAddr = device->func->resource_addr(device, NVKM_BAR2_INST);
918 info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev);
919 info->maxUserVa = TASK_SIZE;
920 info->pciConfigMirrorBase = device->pci->func->cfg.addr;
921 info->pciConfigMirrorSize = device->pci->func->cfg.size;
922 r535_gsp_acpi_info(gsp, &info->acpiMethodData);
923
924 return nvkm_gsp_rpc_wr(gsp, info, NVKM_GSP_RPC_REPLY_NOWAIT);
925 }
926
927 static int
r535_gsp_msg_os_error_log(void * priv,u32 fn,void * repv,u32 repc)928 r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc)
929 {
930 struct nvkm_gsp *gsp = priv;
931 struct nvkm_subdev *subdev = &gsp->subdev;
932 rpc_os_error_log_v17_00 *msg = repv;
933
934 if (WARN_ON(repc < sizeof(*msg)))
935 return -EINVAL;
936
937 nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString);
938 return 0;
939 }
940
941 static int
r535_gsp_msg_mmu_fault_queued(void * priv,u32 fn,void * repv,u32 repc)942 r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc)
943 {
944 struct nvkm_gsp *gsp = priv;
945 struct nvkm_subdev *subdev = &gsp->subdev;
946
947 WARN_ON(repc != 0);
948
949 nvkm_error(subdev, "mmu fault queued\n");
950 return 0;
951 }
952
953 static int
r535_gsp_msg_post_event(void * priv,u32 fn,void * repv,u32 repc)954 r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc)
955 {
956 struct nvkm_gsp *gsp = priv;
957 struct nvkm_gsp_client *client;
958 struct nvkm_subdev *subdev = &gsp->subdev;
959 rpc_post_event_v17_00 *msg = repv;
960
961 if (WARN_ON(repc < sizeof(*msg)))
962 return -EINVAL;
963 if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize))
964 return -EINVAL;
965
966 nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n",
967 msg->hClient, msg->hEvent, msg->notifyIndex, msg->data,
968 msg->status, msg->eventDataSize, msg->bNotifyList);
969
970 mutex_lock(&gsp->client_id.mutex);
971 client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff);
972 if (client) {
973 struct nvkm_gsp_event *event;
974 bool handled = false;
975
976 list_for_each_entry(event, &client->events, head) {
977 if (event->object.handle == msg->hEvent) {
978 event->func(event, msg->eventData, msg->eventDataSize);
979 handled = true;
980 }
981 }
982
983 if (!handled) {
984 nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n",
985 msg->hClient, msg->hEvent);
986 }
987 } else {
988 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient);
989 }
990 mutex_unlock(&gsp->client_id.mutex);
991 return 0;
992 }
993
994 /**
995 * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP
996 * @priv: gsp pointer
997 * @fn: function number (ignored)
998 * @repv: pointer to libos print RPC
999 * @repc: message size
1000 *
1001 * The GSP sequencer is a list of I/O commands that the GSP can send to
1002 * the driver to perform for various purposes. The most common usage is to
1003 * perform a special mid-initialization reset.
1004 */
1005 static int
r535_gsp_msg_run_cpu_sequencer(void * priv,u32 fn,void * repv,u32 repc)1006 r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc)
1007 {
1008 struct nvkm_gsp *gsp = priv;
1009 struct nvkm_subdev *subdev = &gsp->subdev;
1010 struct nvkm_device *device = subdev->device;
1011 rpc_run_cpu_sequencer_v17_00 *seq = repv;
1012 int ptr = 0, ret;
1013
1014 nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex);
1015
1016 while (ptr < seq->cmdIndex) {
1017 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr];
1018
1019 ptr += 1;
1020 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode);
1021
1022 switch (cmd->opCode) {
1023 case GSP_SEQ_BUF_OPCODE_REG_WRITE: {
1024 u32 addr = cmd->payload.regWrite.addr;
1025 u32 data = cmd->payload.regWrite.val;
1026
1027 nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data);
1028 nvkm_wr32(device, addr, data);
1029 }
1030 break;
1031 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: {
1032 u32 addr = cmd->payload.regModify.addr;
1033 u32 mask = cmd->payload.regModify.mask;
1034 u32 data = cmd->payload.regModify.val;
1035
1036 nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data);
1037 nvkm_mask(device, addr, mask, data);
1038 }
1039 break;
1040 case GSP_SEQ_BUF_OPCODE_REG_POLL: {
1041 u32 addr = cmd->payload.regPoll.addr;
1042 u32 mask = cmd->payload.regPoll.mask;
1043 u32 data = cmd->payload.regPoll.val;
1044 u32 usec = cmd->payload.regPoll.timeout ?: 4000000;
1045 //u32 error = cmd->payload.regPoll.error;
1046
1047 nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec);
1048 nvkm_rd32(device, addr);
1049 nvkm_usec(device, usec,
1050 if ((nvkm_rd32(device, addr) & mask) == data)
1051 break;
1052 );
1053 }
1054 break;
1055 case GSP_SEQ_BUF_OPCODE_DELAY_US: {
1056 u32 usec = cmd->payload.delayUs.val;
1057
1058 nvkm_trace(subdev, "seq usec %d\n", usec);
1059 udelay(usec);
1060 }
1061 break;
1062 case GSP_SEQ_BUF_OPCODE_REG_STORE: {
1063 u32 addr = cmd->payload.regStore.addr;
1064 u32 slot = cmd->payload.regStore.index;
1065
1066 seq->regSaveArea[slot] = nvkm_rd32(device, addr);
1067 nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot,
1068 seq->regSaveArea[slot]);
1069 }
1070 break;
1071 case GSP_SEQ_BUF_OPCODE_CORE_RESET:
1072 nvkm_trace(subdev, "seq core reset\n");
1073 nvkm_falcon_reset(&gsp->falcon);
1074 nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080);
1075 nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000);
1076 break;
1077 case GSP_SEQ_BUF_OPCODE_CORE_START:
1078 nvkm_trace(subdev, "seq core start\n");
1079 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040)
1080 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002);
1081 else
1082 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002);
1083 break;
1084 case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT:
1085 nvkm_trace(subdev, "seq core wait halt\n");
1086 nvkm_msec(device, 2000,
1087 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010)
1088 break;
1089 );
1090 break;
1091 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: {
1092 struct nvkm_sec2 *sec2 = device->sec2;
1093 u32 mbox0;
1094
1095 nvkm_trace(subdev, "seq core resume\n");
1096
1097 ret = gsp->func->reset(gsp);
1098 if (WARN_ON(ret))
1099 return ret;
1100
1101 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
1102 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
1103
1104 nvkm_falcon_start(&sec2->falcon);
1105
1106 if (nvkm_msec(device, 2000,
1107 if (nvkm_rd32(device, 0x1180f8) & 0x04000000)
1108 break;
1109 ) < 0)
1110 return -ETIMEDOUT;
1111
1112 mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040);
1113 if (WARN_ON(mbox0)) {
1114 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0);
1115 return -EIO;
1116 }
1117
1118 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1119
1120 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1121 return -EIO;
1122 }
1123 break;
1124 default:
1125 nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode);
1126 return -EINVAL;
1127 }
1128 }
1129
1130 return 0;
1131 }
1132
1133 static int
r535_gsp_shared_init(struct nvkm_gsp * gsp)1134 r535_gsp_shared_init(struct nvkm_gsp *gsp)
1135 {
1136 struct {
1137 msgqTxHeader tx;
1138 msgqRxHeader rx;
1139 } *cmdq, *msgq;
1140 int ret, i;
1141
1142 gsp->shm.cmdq.size = 0x40000;
1143 gsp->shm.msgq.size = 0x40000;
1144
1145 gsp->shm.ptes.nr = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT;
1146 gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1147 gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1148
1149 ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size +
1150 gsp->shm.cmdq.size +
1151 gsp->shm.msgq.size,
1152 &gsp->shm.mem);
1153 if (ret)
1154 return ret;
1155
1156 gsp->shm.ptes.ptr = gsp->shm.mem.data;
1157 gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size;
1158 gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size;
1159
1160 for (i = 0; i < gsp->shm.ptes.nr; i++)
1161 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT);
1162
1163 cmdq = gsp->shm.cmdq.ptr;
1164 cmdq->tx.version = 0;
1165 cmdq->tx.size = gsp->shm.cmdq.size;
1166 cmdq->tx.entryOff = GSP_PAGE_SIZE;
1167 cmdq->tx.msgSize = GSP_PAGE_SIZE;
1168 cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize;
1169 cmdq->tx.writePtr = 0;
1170 cmdq->tx.flags = 1;
1171 cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr);
1172
1173 msgq = gsp->shm.msgq.ptr;
1174
1175 gsp->cmdq.cnt = cmdq->tx.msgCount;
1176 gsp->cmdq.wptr = &cmdq->tx.writePtr;
1177 gsp->cmdq.rptr = &msgq->rx.readPtr;
1178 gsp->msgq.cnt = cmdq->tx.msgCount;
1179 gsp->msgq.wptr = &msgq->tx.writePtr;
1180 gsp->msgq.rptr = &cmdq->rx.readPtr;
1181 return 0;
1182 }
1183
1184 static void
r535_gsp_set_rmargs(struct nvkm_gsp * gsp,bool resume)1185 r535_gsp_set_rmargs(struct nvkm_gsp *gsp, bool resume)
1186 {
1187 GSP_ARGUMENTS_CACHED *args = gsp->rmargs.data;
1188
1189 args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr;
1190 args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr;
1191 args->messageQueueInitArguments.cmdQueueOffset =
1192 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data;
1193 args->messageQueueInitArguments.statQueueOffset =
1194 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data;
1195
1196 if (!resume) {
1197 args->srInitArguments.oldLevel = 0;
1198 args->srInitArguments.flags = 0;
1199 args->srInitArguments.bInPMTransition = 0;
1200 } else {
1201 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1202 args->srInitArguments.flags = 0;
1203 args->srInitArguments.bInPMTransition = 1;
1204 }
1205 }
1206
1207 static int
r535_gsp_rmargs_init(struct nvkm_gsp * gsp,bool resume)1208 r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume)
1209 {
1210 int ret;
1211
1212 if (!resume) {
1213 ret = r535_gsp_shared_init(gsp);
1214 if (ret)
1215 return ret;
1216
1217 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs);
1218 if (ret)
1219 return ret;
1220 }
1221
1222 gsp->rm->api->gsp->set_rmargs(gsp, resume);
1223 return 0;
1224 }
1225
1226 #ifdef CONFIG_DEBUG_FS
1227
1228 /*
1229 * If GSP-RM load fails, then the GSP nvkm object will be deleted, the logging
1230 * debugfs entries will be deleted, and it will not be possible to debug the
1231 * load failure. The keep_gsp_logging parameter tells Nouveau to copy the
1232 * logging buffers to new debugfs entries, and these entries are retained
1233 * until the driver unloads.
1234 */
1235 static bool keep_gsp_logging;
1236 module_param(keep_gsp_logging, bool, 0444);
1237 MODULE_PARM_DESC(keep_gsp_logging,
1238 "Migrate the GSP-RM logging debugfs entries upon exit");
1239
1240 /*
1241 * GSP-RM uses a pseudo-class mechanism to define of a variety of per-"engine"
1242 * data structures, and each engine has a "class ID" genererated by a
1243 * pre-processor. This is the class ID for the PMU.
1244 */
1245 #define NV_GSP_MSG_EVENT_UCODE_LIBOS_CLASS_PMU 0xf3d722
1246
1247 /**
1248 * struct rpc_ucode_libos_print_v1e_08 - RPC payload for libos print buffers
1249 * @ucode_eng_desc: the engine descriptor
1250 * @libos_print_buf_size: the size of the libos_print_buf[]
1251 * @libos_print_buf: the actual buffer
1252 *
1253 * The engine descriptor is divided into 31:8 "class ID" and 7:0 "instance
1254 * ID". We only care about messages from PMU.
1255 */
1256 struct rpc_ucode_libos_print_v1e_08 {
1257 u32 ucode_eng_desc;
1258 u32 libos_print_buf_size;
1259 u8 libos_print_buf[];
1260 };
1261
1262 /**
1263 * r535_gsp_msg_libos_print - capture log message from the PMU
1264 * @priv: gsp pointer
1265 * @fn: function number (ignored)
1266 * @repv: pointer to libos print RPC
1267 * @repc: message size
1268 *
1269 * Called when we receive a UCODE_LIBOS_PRINT event RPC from GSP-RM. This RPC
1270 * contains the contents of the libos print buffer from PMU. It is typically
1271 * only written to when PMU encounters an error.
1272 *
1273 * Technically this RPC can be used to pass print buffers from any number of
1274 * GSP-RM engines, but we only expect to receive them for the PMU.
1275 *
1276 * For the PMU, the buffer is 4K in size and the RPC always contains the full
1277 * contents.
1278 */
1279 static int
r535_gsp_msg_libos_print(void * priv,u32 fn,void * repv,u32 repc)1280 r535_gsp_msg_libos_print(void *priv, u32 fn, void *repv, u32 repc)
1281 {
1282 struct nvkm_gsp *gsp = priv;
1283 struct nvkm_subdev *subdev = &gsp->subdev;
1284 struct rpc_ucode_libos_print_v1e_08 *rpc = repv;
1285 unsigned int class = rpc->ucode_eng_desc >> 8;
1286
1287 nvkm_debug(subdev, "received libos print from class 0x%x for %u bytes\n",
1288 class, rpc->libos_print_buf_size);
1289
1290 if (class != NV_GSP_MSG_EVENT_UCODE_LIBOS_CLASS_PMU) {
1291 nvkm_warn(subdev,
1292 "received libos print from unknown class 0x%x\n",
1293 class);
1294 return -ENOMSG;
1295 }
1296
1297 if (rpc->libos_print_buf_size > GSP_PAGE_SIZE) {
1298 nvkm_error(subdev, "libos print is too large (%u bytes)\n",
1299 rpc->libos_print_buf_size);
1300 return -E2BIG;
1301 }
1302
1303 memcpy(gsp->blob_pmu.data, rpc->libos_print_buf, rpc->libos_print_buf_size);
1304
1305 return 0;
1306 }
1307
1308 /**
1309 * create_debugfs - create a blob debugfs entry
1310 * @gsp: gsp pointer
1311 * @name: name of this dentry
1312 * @blob: blob wrapper
1313 *
1314 * Creates a debugfs entry for a logging buffer with the name 'name'.
1315 */
create_debugfs(struct nvkm_gsp * gsp,const char * name,struct debugfs_blob_wrapper * blob)1316 static struct dentry *create_debugfs(struct nvkm_gsp *gsp, const char *name,
1317 struct debugfs_blob_wrapper *blob)
1318 {
1319 struct dentry *dent;
1320
1321 dent = debugfs_create_blob(name, 0444, gsp->debugfs.parent, blob);
1322 if (IS_ERR(dent)) {
1323 nvkm_error(&gsp->subdev,
1324 "failed to create %s debugfs entry\n", name);
1325 return NULL;
1326 }
1327
1328 /*
1329 * For some reason, debugfs_create_blob doesn't set the size of the
1330 * dentry, so do that here. See [1]
1331 *
1332 * [1] https://lore.kernel.org/r/linux-fsdevel/20240207200619.3354549-1-ttabi@nvidia.com/
1333 */
1334 i_size_write(d_inode(dent), blob->size);
1335
1336 return dent;
1337 }
1338
1339 /**
1340 * r535_gsp_libos_debugfs_init - create logging debugfs entries
1341 * @gsp: gsp pointer
1342 *
1343 * Create the debugfs entries. This exposes the log buffers to userspace so
1344 * that an external tool can parse it.
1345 *
1346 * The 'logpmu' contains exception dumps from the PMU. It is written via an
1347 * RPC sent from GSP-RM and must be only 4KB. We create it here because it's
1348 * only useful if there is a debugfs entry to expose it. If we get the PMU
1349 * logging RPC and there is no debugfs entry, the RPC is just ignored.
1350 *
1351 * The blob_init, blob_rm, and blob_pmu objects can't be transient
1352 * because debugfs_create_blob doesn't copy them.
1353 *
1354 * NOTE: OpenRM loads the logging elf image and prints the log messages
1355 * in real-time. We may add that capability in the future, but that
1356 * requires loading ELF images that are not distributed with the driver and
1357 * adding the parsing code to Nouveau.
1358 *
1359 * Ideally, this should be part of nouveau_debugfs_init(), but that function
1360 * is called too late. We really want to create these debugfs entries before
1361 * r535_gsp_booter_load() is called, so that if GSP-RM fails to initialize,
1362 * there could still be a log to capture.
1363 */
1364 static void
r535_gsp_libos_debugfs_init(struct nvkm_gsp * gsp)1365 r535_gsp_libos_debugfs_init(struct nvkm_gsp *gsp)
1366 {
1367 struct device *dev = gsp->subdev.device->dev;
1368
1369 /* Create a new debugfs directory with a name unique to this GPU. */
1370 gsp->debugfs.parent = debugfs_create_dir(dev_name(dev), nouveau_debugfs_root);
1371 if (IS_ERR(gsp->debugfs.parent)) {
1372 nvkm_error(&gsp->subdev,
1373 "failed to create %s debugfs root\n", dev_name(dev));
1374 return;
1375 }
1376
1377 gsp->blob_init.data = gsp->loginit.data;
1378 gsp->blob_init.size = gsp->loginit.size;
1379 gsp->blob_intr.data = gsp->logintr.data;
1380 gsp->blob_intr.size = gsp->logintr.size;
1381 gsp->blob_rm.data = gsp->logrm.data;
1382 gsp->blob_rm.size = gsp->logrm.size;
1383
1384 gsp->debugfs.init = create_debugfs(gsp, "loginit", &gsp->blob_init);
1385 if (!gsp->debugfs.init)
1386 goto error;
1387
1388 gsp->debugfs.intr = create_debugfs(gsp, "logintr", &gsp->blob_intr);
1389 if (!gsp->debugfs.intr)
1390 goto error;
1391
1392 gsp->debugfs.rm = create_debugfs(gsp, "logrm", &gsp->blob_rm);
1393 if (!gsp->debugfs.rm)
1394 goto error;
1395
1396 /*
1397 * Since the PMU buffer is copied from an RPC, it doesn't need to be
1398 * a DMA buffer.
1399 */
1400 gsp->blob_pmu.size = GSP_PAGE_SIZE;
1401 gsp->blob_pmu.data = kzalloc(gsp->blob_pmu.size, GFP_KERNEL);
1402 if (!gsp->blob_pmu.data)
1403 goto error;
1404
1405 gsp->debugfs.pmu = create_debugfs(gsp, "logpmu", &gsp->blob_pmu);
1406 if (!gsp->debugfs.pmu) {
1407 kfree(gsp->blob_pmu.data);
1408 goto error;
1409 }
1410
1411 i_size_write(d_inode(gsp->debugfs.init), gsp->blob_init.size);
1412 i_size_write(d_inode(gsp->debugfs.intr), gsp->blob_intr.size);
1413 i_size_write(d_inode(gsp->debugfs.rm), gsp->blob_rm.size);
1414 i_size_write(d_inode(gsp->debugfs.pmu), gsp->blob_pmu.size);
1415
1416 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT,
1417 r535_gsp_msg_libos_print, gsp);
1418
1419 nvkm_debug(&gsp->subdev, "created debugfs GSP-RM logging entries\n");
1420
1421 if (keep_gsp_logging) {
1422 nvkm_info(&gsp->subdev,
1423 "logging buffers will be retained on failure\n");
1424 }
1425
1426 return;
1427
1428 error:
1429 debugfs_remove(gsp->debugfs.parent);
1430 gsp->debugfs.parent = NULL;
1431 }
1432
1433 #endif
1434
1435 static inline u64
r535_gsp_libos_id8(const char * name)1436 r535_gsp_libos_id8(const char *name)
1437 {
1438 u64 id = 0;
1439
1440 for (int i = 0; i < sizeof(id) && *name; i++, name++)
1441 id = (id << 8) | *name;
1442
1443 return id;
1444 }
1445
1446 /**
1447 * create_pte_array() - creates a PTE array of a physically contiguous buffer
1448 * @ptes: pointer to the array
1449 * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned)
1450 * @size: size of the buffer
1451 *
1452 * GSP-RM sometimes expects physically-contiguous buffers to have an array of
1453 * "PTEs" for each page in that buffer. Although in theory that allows for
1454 * the buffer to be physically discontiguous, GSP-RM does not currently
1455 * support that.
1456 *
1457 * In this case, the PTEs are DMA addresses of each page of the buffer. Since
1458 * the buffer is physically contiguous, calculating all the PTEs is simple
1459 * math.
1460 *
1461 * See memdescGetPhysAddrsForGpu()
1462 */
create_pte_array(u64 * ptes,dma_addr_t addr,size_t size)1463 static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size)
1464 {
1465 unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE);
1466 unsigned int i;
1467
1468 for (i = 0; i < num_pages; i++)
1469 ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT);
1470 }
1471
1472 /**
1473 * r535_gsp_libos_init() -- create the libos arguments structure
1474 * @gsp: gsp pointer
1475 *
1476 * The logging buffers are byte queues that contain encoded printf-like
1477 * messages from GSP-RM. They need to be decoded by a special application
1478 * that can parse the buffers.
1479 *
1480 * The 'loginit' buffer contains logs from early GSP-RM init and
1481 * exception dumps. The 'logrm' buffer contains the subsequent logs. Both are
1482 * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.
1483 *
1484 * The physical address map for the log buffer is stored in the buffer
1485 * itself, starting with offset 1. Offset 0 contains the "put" pointer (pp).
1486 * Initially, pp is equal to 0. If the buffer has valid logging data in it,
1487 * then pp points to index into the buffer where the next logging entry will
1488 * be written. Therefore, the logging data is valid if:
1489 * 1 <= pp < sizeof(buffer)/sizeof(u64)
1490 *
1491 * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is
1492 * configured for a larger page size (e.g. 64K pages), we need to give
1493 * the GSP an array of 4K pages. Fortunately, since the buffer is
1494 * physically contiguous, it's simple math to calculate the addresses.
1495 *
1496 * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently
1497 * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the
1498 * buffers to be physically contiguous anyway.
1499 *
1500 * The memory allocated for the arguments must remain until the GSP sends the
1501 * init_done RPC.
1502 *
1503 * See _kgspInitLibosLoggingStructures (allocates memory for buffers)
1504 * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
1505 */
1506 static int
r535_gsp_libos_init(struct nvkm_gsp * gsp)1507 r535_gsp_libos_init(struct nvkm_gsp *gsp)
1508 {
1509 LibosMemoryRegionInitArgument *args;
1510 int ret;
1511
1512 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos);
1513 if (ret)
1514 return ret;
1515
1516 args = gsp->libos.data;
1517
1518 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit);
1519 if (ret)
1520 return ret;
1521
1522 args[0].id8 = r535_gsp_libos_id8("LOGINIT");
1523 args[0].pa = gsp->loginit.addr;
1524 args[0].size = gsp->loginit.size;
1525 args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1526 args[0].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1527 create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size);
1528
1529 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr);
1530 if (ret)
1531 return ret;
1532
1533 args[1].id8 = r535_gsp_libos_id8("LOGINTR");
1534 args[1].pa = gsp->logintr.addr;
1535 args[1].size = gsp->logintr.size;
1536 args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1537 args[1].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1538 create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size);
1539
1540 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm);
1541 if (ret)
1542 return ret;
1543
1544 args[2].id8 = r535_gsp_libos_id8("LOGRM");
1545 args[2].pa = gsp->logrm.addr;
1546 args[2].size = gsp->logrm.size;
1547 args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1548 args[2].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1549 create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size);
1550
1551 ret = r535_gsp_rmargs_init(gsp, false);
1552 if (ret)
1553 return ret;
1554
1555 args[3].id8 = r535_gsp_libos_id8("RMARGS");
1556 args[3].pa = gsp->rmargs.addr;
1557 args[3].size = gsp->rmargs.size;
1558 args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1559 args[3].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1560
1561 #ifdef CONFIG_DEBUG_FS
1562 r535_gsp_libos_debugfs_init(gsp);
1563 #endif
1564
1565 return 0;
1566 }
1567
1568 void
nvkm_gsp_sg_free(struct nvkm_device * device,struct sg_table * sgt)1569 nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt)
1570 {
1571 struct scatterlist *sgl;
1572 int i;
1573
1574 dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
1575
1576 for_each_sgtable_sg(sgt, sgl, i) {
1577 struct page *page = sg_page(sgl);
1578
1579 __free_page(page);
1580 }
1581
1582 sg_free_table(sgt);
1583 }
1584
1585 int
nvkm_gsp_sg(struct nvkm_device * device,u64 size,struct sg_table * sgt)1586 nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)
1587 {
1588 const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE);
1589 struct scatterlist *sgl;
1590 int ret, i;
1591
1592 ret = sg_alloc_table(sgt, pages, GFP_KERNEL);
1593 if (ret)
1594 return ret;
1595
1596 for_each_sgtable_sg(sgt, sgl, i) {
1597 struct page *page = alloc_page(GFP_KERNEL);
1598
1599 if (!page) {
1600 nvkm_gsp_sg_free(device, sgt);
1601 return -ENOMEM;
1602 }
1603
1604 sg_set_page(sgl, page, PAGE_SIZE, 0);
1605 }
1606
1607 ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
1608 if (ret)
1609 nvkm_gsp_sg_free(device, sgt);
1610
1611 return ret;
1612 }
1613
1614 static void
nvkm_gsp_radix3_dtor(struct nvkm_gsp * gsp,struct nvkm_gsp_radix3 * rx3)1615 nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)
1616 {
1617 nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2);
1618 nvkm_gsp_mem_dtor(&rx3->lvl1);
1619 nvkm_gsp_mem_dtor(&rx3->lvl0);
1620 }
1621
1622 /**
1623 * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list
1624 * @gsp: gsp pointer
1625 * @sgt: S/G list to traverse
1626 * @size: size of the image, in bytes
1627 * @rx3: radix3 array to update
1628 *
1629 * The GSP uses a three-level page table, called radix3, to map the firmware.
1630 * Each 64-bit "pointer" in the table is either the bus address of an entry in
1631 * the next table (for levels 0 and 1) or the bus address of the next page in
1632 * the GSP firmware image itself.
1633 *
1634 * Level 0 contains a single entry in one page that points to the first page
1635 * of level 1.
1636 *
1637 * Level 1, since it's also only one page in size, contains up to 512 entries,
1638 * one for each page in Level 2.
1639 *
1640 * Level 2 can be up to 512 pages in size, and each of those entries points to
1641 * the next page of the firmware image. Since there can be up to 512*512
1642 * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB.
1643 *
1644 * Internally, the GSP has its window into system memory, but the base
1645 * physical address of the aperture is not 0. In fact, it varies depending on
1646 * the GPU architecture. Since the GPU is a PCI device, this window is
1647 * accessed via DMA and is therefore bound by IOMMU translation. The end
1648 * result is that GSP-RM must translate the bus addresses in the table to GSP
1649 * physical addresses. All this should happen transparently.
1650 *
1651 * Returns 0 on success, or negative error code
1652 *
1653 * See kgspCreateRadix3_IMPL
1654 */
1655 static int
nvkm_gsp_radix3_sg(struct nvkm_gsp * gsp,struct sg_table * sgt,u64 size,struct nvkm_gsp_radix3 * rx3)1656 nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size,
1657 struct nvkm_gsp_radix3 *rx3)
1658 {
1659 struct sg_dma_page_iter sg_dma_iter;
1660 struct scatterlist *sg;
1661 size_t bufsize;
1662 u64 *pte;
1663 int ret, i, page_idx = 0;
1664
1665 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0);
1666 if (ret)
1667 return ret;
1668
1669 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1);
1670 if (ret)
1671 goto lvl1_fail;
1672
1673 // Allocate level 2
1674 bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
1675 ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2);
1676 if (ret)
1677 goto lvl2_fail;
1678
1679 // Write the bus address of level 1 to level 0
1680 pte = rx3->lvl0.data;
1681 *pte = rx3->lvl1.addr;
1682
1683 // Write the bus address of each page in level 2 to level 1
1684 pte = rx3->lvl1.data;
1685 for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0)
1686 *pte++ = sg_page_iter_dma_address(&sg_dma_iter);
1687
1688 // Finally, write the bus address of each page in sgt to level 2
1689 for_each_sgtable_sg(&rx3->lvl2, sg, i) {
1690 void *sgl_end;
1691
1692 pte = sg_virt(sg);
1693 sgl_end = (void *)pte + sg->length;
1694
1695 for_each_sgtable_dma_page(sgt, &sg_dma_iter, page_idx) {
1696 *pte++ = sg_page_iter_dma_address(&sg_dma_iter);
1697 page_idx++;
1698
1699 // Go to the next scatterlist for level 2 if we've reached the end
1700 if ((void *)pte >= sgl_end)
1701 break;
1702 }
1703 }
1704
1705 if (ret) {
1706 lvl2_fail:
1707 nvkm_gsp_mem_dtor(&rx3->lvl1);
1708 lvl1_fail:
1709 nvkm_gsp_mem_dtor(&rx3->lvl0);
1710 }
1711
1712 return ret;
1713 }
1714
1715 static u32
r535_gsp_sr_data_size(struct nvkm_gsp * gsp)1716 r535_gsp_sr_data_size(struct nvkm_gsp *gsp)
1717 {
1718 GspFwWprMeta *meta = gsp->wpr_meta.data;
1719
1720 return meta->gspFwWprEnd - meta->gspFwWprStart;
1721 }
1722
1723 int
r535_gsp_fini(struct nvkm_gsp * gsp,bool suspend)1724 r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)
1725 {
1726 struct nvkm_rm *rm = gsp->rm;
1727 int ret;
1728
1729 if (suspend) {
1730 u32 len = rm->api->gsp->sr_data_size(gsp);
1731 GspFwSRMeta *sr;
1732
1733 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt);
1734 if (ret)
1735 return ret;
1736
1737 ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3);
1738 if (ret)
1739 return ret;
1740
1741 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta);
1742 if (ret)
1743 return ret;
1744
1745 sr = gsp->sr.meta.data;
1746 sr->magic = GSP_FW_SR_META_MAGIC;
1747 sr->revision = GSP_FW_SR_META_REVISION;
1748 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr;
1749 sr->sizeOfSuspendResumeData = len;
1750
1751 ret = rm->api->fbsr->suspend(gsp);
1752 if (ret) {
1753 nvkm_gsp_mem_dtor(&gsp->sr.meta);
1754 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);
1755 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);
1756 return ret;
1757 }
1758
1759 /*
1760 * TODO: Debug the GSP firmware / RPC handling to find out why
1761 * without this Turing (but none of the other architectures)
1762 * ends up resetting all channels after resume.
1763 */
1764 msleep(50);
1765 }
1766
1767 ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend);
1768 if (WARN_ON(ret))
1769 return ret;
1770
1771 nvkm_msec(gsp->subdev.device, 2000,
1772 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) == 0x80000000)
1773 break;
1774 );
1775
1776 gsp->running = false;
1777 return 0;
1778 }
1779
1780 int
r535_gsp_init(struct nvkm_gsp * gsp)1781 r535_gsp_init(struct nvkm_gsp *gsp)
1782 {
1783 int ret;
1784
1785 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1786
1787 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1788 return -EIO;
1789
1790 ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE);
1791 if (ret)
1792 goto done;
1793
1794 gsp->running = true;
1795
1796 done:
1797 if (gsp->sr.meta.data) {
1798 gsp->rm->api->fbsr->resume(gsp);
1799
1800 nvkm_gsp_mem_dtor(&gsp->sr.meta);
1801 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);
1802 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);
1803 return ret;
1804 }
1805
1806 if (ret == 0)
1807 ret = r535_gsp_postinit(gsp);
1808
1809 return ret;
1810 }
1811
1812 static int
r535_gsp_rm_boot_ctor(struct nvkm_gsp * gsp)1813 r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp)
1814 {
1815 const struct firmware *fw = gsp->fws.bl;
1816 const struct nvfw_bin_hdr *hdr;
1817 RM_RISCV_UCODE_DESC *desc;
1818 int ret;
1819
1820 ret = nvkm_gsp_fwsec_sb_ctor(gsp);
1821 if (ret)
1822 return ret;
1823
1824 hdr = nvfw_bin_hdr(&gsp->subdev, fw->data);
1825 desc = (void *)fw->data + hdr->header_offset;
1826
1827 ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw);
1828 if (ret)
1829 goto dtor_fwsec;
1830
1831 memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size);
1832
1833 gsp->boot.code_offset = desc->monitorCodeOffset;
1834 gsp->boot.data_offset = desc->monitorDataOffset;
1835 gsp->boot.manifest_offset = desc->manifestOffset;
1836 gsp->boot.app_version = desc->appVersion;
1837 return 0;
1838 dtor_fwsec:
1839 nvkm_gsp_fwsec_sb_dtor(gsp);
1840 return ret;
1841 }
1842
1843 static const struct nvkm_firmware_func
1844 r535_gsp_fw = {
1845 .type = NVKM_FIRMWARE_IMG_SGT,
1846 };
1847
1848 static int
r535_gsp_elf_section(struct nvkm_gsp * gsp,const char * name,const u8 ** pdata,u64 * psize)1849 r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize)
1850 {
1851 const u8 *img = gsp->fws.rm->data;
1852 const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img;
1853 const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff];
1854 const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset];
1855
1856 for (int i = 0; i < ehdr->e_shnum; i++, shdr++) {
1857 if (!strcmp(&names[shdr->sh_name], name)) {
1858 *pdata = &img[shdr->sh_offset];
1859 *psize = shdr->sh_size;
1860 return 0;
1861 }
1862 }
1863
1864 nvkm_error(&gsp->subdev, "section '%s' not found\n", name);
1865 return -ENOENT;
1866 }
1867
1868 #ifdef CONFIG_DEBUG_FS
1869
1870 struct r535_gsp_log {
1871 struct nvif_log log;
1872
1873 /*
1874 * Logging buffers in debugfs. The wrapper objects need to remain
1875 * in memory until the dentry is deleted.
1876 */
1877 struct dentry *debugfs_logging_dir;
1878 struct debugfs_blob_wrapper blob_init;
1879 struct debugfs_blob_wrapper blob_intr;
1880 struct debugfs_blob_wrapper blob_rm;
1881 struct debugfs_blob_wrapper blob_pmu;
1882 };
1883
1884 /**
1885 * r535_debugfs_shutdown - delete GSP-RM logging buffers for one GPU
1886 * @_log: nvif_log struct for this GPU
1887 *
1888 * Called when the driver is shutting down, to clean up the retained GSP-RM
1889 * logging buffers.
1890 */
r535_debugfs_shutdown(struct nvif_log * _log)1891 static void r535_debugfs_shutdown(struct nvif_log *_log)
1892 {
1893 struct r535_gsp_log *log = container_of(_log, struct r535_gsp_log, log);
1894
1895 debugfs_remove(log->debugfs_logging_dir);
1896
1897 kfree(log->blob_init.data);
1898 kfree(log->blob_intr.data);
1899 kfree(log->blob_rm.data);
1900 kfree(log->blob_pmu.data);
1901
1902 /* We also need to delete the list object */
1903 kfree(log);
1904 }
1905
1906 /**
1907 * is_empty - return true if the logging buffer was never written to
1908 * @b: blob wrapper with ->data field pointing to logging buffer
1909 *
1910 * The first 64-bit field of loginit, and logintr, and logrm is the 'put'
1911 * pointer, and it is initialized to 0. It's a dword-based index into the
1912 * circular buffer, indicating where the next printf write will be made.
1913 *
1914 * If the pointer is still 0 when GSP-RM is shut down, that means that the
1915 * buffer was never written to, so it can be ignored.
1916 *
1917 * This test also works for logpmu, even though it doesn't have a put pointer.
1918 */
is_empty(const struct debugfs_blob_wrapper * b)1919 static bool is_empty(const struct debugfs_blob_wrapper *b)
1920 {
1921 u64 *put = b->data;
1922
1923 return put ? (*put == 0) : true;
1924 }
1925
1926 /**
1927 * r535_gsp_copy_log - preserve the logging buffers in a blob
1928 * @parent: the top-level dentry for this GPU
1929 * @name: name of debugfs entry to create
1930 * @s: original wrapper object to copy from
1931 * @t: new wrapper object to copy to
1932 *
1933 * When GSP shuts down, the nvkm_gsp object and all its memory is deleted.
1934 * To preserve the logging buffers, the buffers need to be copied, but only
1935 * if they actually have data.
1936 */
r535_gsp_copy_log(struct dentry * parent,const char * name,const struct debugfs_blob_wrapper * s,struct debugfs_blob_wrapper * t)1937 static int r535_gsp_copy_log(struct dentry *parent,
1938 const char *name,
1939 const struct debugfs_blob_wrapper *s,
1940 struct debugfs_blob_wrapper *t)
1941 {
1942 struct dentry *dent;
1943 void *p;
1944
1945 if (is_empty(s))
1946 return 0;
1947
1948 /* The original buffers will be deleted */
1949 p = kmemdup(s->data, s->size, GFP_KERNEL);
1950 if (!p)
1951 return -ENOMEM;
1952
1953 t->data = p;
1954 t->size = s->size;
1955
1956 dent = debugfs_create_blob(name, 0444, parent, t);
1957 if (IS_ERR(dent)) {
1958 kfree(p);
1959 memset(t, 0, sizeof(*t));
1960 return PTR_ERR(dent);
1961 }
1962
1963 i_size_write(d_inode(dent), t->size);
1964
1965 return 0;
1966 }
1967
1968 /**
1969 * r535_gsp_retain_logging - copy logging buffers to new debugfs root
1970 * @gsp: gsp pointer
1971 *
1972 * If keep_gsp_logging is enabled, then we want to preserve the GSP-RM logging
1973 * buffers and their debugfs entries, but all those objects would normally
1974 * deleted if GSP-RM fails to load.
1975 *
1976 * To preserve the logging buffers, we need to:
1977 *
1978 * 1) Allocate new buffers and copy the logs into them, so that the original
1979 * DMA buffers can be released.
1980 *
1981 * 2) Preserve the directories. We don't need to save single dentries because
1982 * we're going to delete the parent when the
1983 *
1984 * If anything fails in this process, then all the dentries need to be
1985 * deleted. We don't need to deallocate the original logging buffers because
1986 * the caller will do that regardless.
1987 */
r535_gsp_retain_logging(struct nvkm_gsp * gsp)1988 static void r535_gsp_retain_logging(struct nvkm_gsp *gsp)
1989 {
1990 struct device *dev = gsp->subdev.device->dev;
1991 struct r535_gsp_log *log = NULL;
1992 int ret;
1993
1994 if (!keep_gsp_logging || !gsp->debugfs.parent) {
1995 /* Nothing to do */
1996 goto exit;
1997 }
1998
1999 /* Check to make sure at least one buffer has data. */
2000 if (is_empty(&gsp->blob_init) && is_empty(&gsp->blob_intr) &&
2001 is_empty(&gsp->blob_rm) && is_empty(&gsp->blob_rm)) {
2002 nvkm_warn(&gsp->subdev, "all logging buffers are empty\n");
2003 goto exit;
2004 }
2005
2006 log = kzalloc(sizeof(*log), GFP_KERNEL);
2007 if (!log)
2008 goto error;
2009
2010 /*
2011 * Since the nvkm_gsp object is going away, the debugfs_blob_wrapper
2012 * objects are also being deleted, which means the dentries will no
2013 * longer be valid. Delete the existing entries so that we can create
2014 * new ones with the same name.
2015 */
2016 debugfs_remove(gsp->debugfs.init);
2017 debugfs_remove(gsp->debugfs.intr);
2018 debugfs_remove(gsp->debugfs.rm);
2019 debugfs_remove(gsp->debugfs.pmu);
2020
2021 ret = r535_gsp_copy_log(gsp->debugfs.parent, "loginit", &gsp->blob_init, &log->blob_init);
2022 if (ret)
2023 goto error;
2024
2025 ret = r535_gsp_copy_log(gsp->debugfs.parent, "logintr", &gsp->blob_intr, &log->blob_intr);
2026 if (ret)
2027 goto error;
2028
2029 ret = r535_gsp_copy_log(gsp->debugfs.parent, "logrm", &gsp->blob_rm, &log->blob_rm);
2030 if (ret)
2031 goto error;
2032
2033 ret = r535_gsp_copy_log(gsp->debugfs.parent, "logpmu", &gsp->blob_pmu, &log->blob_pmu);
2034 if (ret)
2035 goto error;
2036
2037 /* The nvkm_gsp object is going away, so save the dentry */
2038 log->debugfs_logging_dir = gsp->debugfs.parent;
2039
2040 log->log.shutdown = r535_debugfs_shutdown;
2041 list_add(&log->log.entry, &gsp_logs.head);
2042
2043 nvkm_warn(&gsp->subdev,
2044 "logging buffers migrated to /sys/kernel/debug/nouveau/%s\n",
2045 dev_name(dev));
2046
2047 return;
2048
2049 error:
2050 nvkm_warn(&gsp->subdev, "failed to migrate logging buffers\n");
2051
2052 exit:
2053 debugfs_remove(gsp->debugfs.parent);
2054
2055 if (log) {
2056 kfree(log->blob_init.data);
2057 kfree(log->blob_intr.data);
2058 kfree(log->blob_rm.data);
2059 kfree(log->blob_pmu.data);
2060 kfree(log);
2061 }
2062 }
2063
2064 #endif
2065
2066 /**
2067 * r535_gsp_libos_debugfs_fini - cleanup/retain log buffers on shutdown
2068 * @gsp: gsp pointer
2069 *
2070 * If the log buffers are exposed via debugfs, the data for those entries
2071 * needs to be cleaned up when the GSP device shuts down.
2072 */
2073 static void
r535_gsp_libos_debugfs_fini(struct nvkm_gsp __maybe_unused * gsp)2074 r535_gsp_libos_debugfs_fini(struct nvkm_gsp __maybe_unused *gsp)
2075 {
2076 #ifdef CONFIG_DEBUG_FS
2077 r535_gsp_retain_logging(gsp);
2078
2079 /*
2080 * Unlike the other buffers, the PMU blob is a kmalloc'd buffer that
2081 * exists only if the debugfs entries were created.
2082 */
2083 kfree(gsp->blob_pmu.data);
2084 gsp->blob_pmu.data = NULL;
2085 #endif
2086 }
2087
2088 void
r535_gsp_dtor(struct nvkm_gsp * gsp)2089 r535_gsp_dtor(struct nvkm_gsp *gsp)
2090 {
2091 idr_destroy(&gsp->client_id.idr);
2092 mutex_destroy(&gsp->client_id.mutex);
2093
2094 nvkm_gsp_radix3_dtor(gsp, &gsp->radix3);
2095 nvkm_gsp_mem_dtor(&gsp->sig);
2096 nvkm_firmware_dtor(&gsp->fw);
2097
2098 nvkm_falcon_fw_dtor(&gsp->booter.unload);
2099 nvkm_falcon_fw_dtor(&gsp->booter.load);
2100
2101 nvkm_gsp_mem_dtor(&gsp->fmc.args);
2102 kfree(gsp->fmc.sig);
2103 kfree(gsp->fmc.pkey);
2104 kfree(gsp->fmc.hash);
2105 nvkm_gsp_mem_dtor(&gsp->fmc.fw);
2106
2107 mutex_destroy(&gsp->msgq.mutex);
2108 mutex_destroy(&gsp->cmdq.mutex);
2109
2110 nvkm_gsp_dtor_fws(gsp);
2111 nvkm_gsp_fwsec_sb_dtor(gsp);
2112
2113 nvkm_gsp_mem_dtor(&gsp->rmargs);
2114 nvkm_gsp_mem_dtor(&gsp->wpr_meta);
2115 nvkm_gsp_mem_dtor(&gsp->shm.mem);
2116
2117 r535_gsp_libos_debugfs_fini(gsp);
2118
2119 nvkm_gsp_mem_dtor(&gsp->loginit);
2120 nvkm_gsp_mem_dtor(&gsp->logintr);
2121 nvkm_gsp_mem_dtor(&gsp->logrm);
2122 }
2123
2124 static void
r535_gsp_drop_send_user_shared_data(struct nvkm_gsp * gsp)2125 r535_gsp_drop_send_user_shared_data(struct nvkm_gsp *gsp)
2126 {
2127 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL);
2128 }
2129
2130 int
r535_gsp_oneinit(struct nvkm_gsp * gsp)2131 r535_gsp_oneinit(struct nvkm_gsp *gsp)
2132 {
2133 struct nvkm_device *device = gsp->subdev.device;
2134 const struct nvkm_rm_api *rmapi = gsp->rm->api;
2135 const u8 *data;
2136 u64 size;
2137 int ret;
2138
2139 mutex_init(&gsp->cmdq.mutex);
2140 mutex_init(&gsp->msgq.mutex);
2141
2142 /* Load GSP firmware from ELF image into DMA-accessible memory. */
2143 ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size);
2144 if (ret)
2145 return ret;
2146
2147 ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw);
2148 if (ret)
2149 return ret;
2150
2151 /* Load relevant signature from ELF image. */
2152 ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size);
2153 if (ret)
2154 return ret;
2155
2156 ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig);
2157 if (ret)
2158 return ret;
2159
2160 memcpy(gsp->sig.data, data, size);
2161
2162 /* Build radix3 page table for ELF image. */
2163 ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3);
2164 if (ret)
2165 return ret;
2166
2167 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,
2168 r535_gsp_msg_run_cpu_sequencer, gsp);
2169 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp);
2170 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED, rmapi->fifo->rc_triggered, gsp);
2171 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,
2172 r535_gsp_msg_mmu_fault_queued, gsp);
2173 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp);
2174 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL);
2175 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL);
2176 if (rmapi->gsp->drop_send_user_shared_data)
2177 rmapi->gsp->drop_send_user_shared_data(gsp);
2178 if (rmapi->gsp->drop_post_nocat_record)
2179 rmapi->gsp->drop_post_nocat_record(gsp);
2180
2181 ret = r535_gsp_rm_boot_ctor(gsp);
2182 if (ret)
2183 return ret;
2184
2185 /* Release FW images - we've copied them to DMA buffers now. */
2186 nvkm_gsp_dtor_fws(gsp);
2187
2188 ret = r535_gsp_libos_init(gsp);
2189 if (WARN_ON(ret))
2190 return ret;
2191
2192 ret = rmapi->gsp->set_system_info(gsp);
2193 if (WARN_ON(ret))
2194 return ret;
2195
2196 ret = r535_gsp_rpc_set_registry(gsp);
2197 if (WARN_ON(ret))
2198 return ret;
2199
2200 mutex_init(&gsp->client_id.mutex);
2201 idr_init(&gsp->client_id.idr);
2202 return 0;
2203 }
2204
2205 const struct nvkm_rm_api_gsp
2206 r535_gsp = {
2207 .set_rmargs = r535_gsp_set_rmargs,
2208 .set_system_info = r535_gsp_set_system_info,
2209 .get_static_info = r535_gsp_get_static_info,
2210 .xlat_mc_engine_idx = r535_gsp_xlat_mc_engine_idx,
2211 .drop_send_user_shared_data = r535_gsp_drop_send_user_shared_data,
2212 .sr_data_size = r535_gsp_sr_data_size,
2213 };
2214