ivpu_mmu.c (c9da9a1f17bf4fa96b115950fd389c917b583c1c) ivpu_mmu.c (b039f1c4d3727c3e44a7e89ff79e7e8533e1beec)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/highmem.h>
8

--- 58 unchanged lines hidden (view full) ---

67
68#define IVPU_MMU_STRTAB_ENT_SIZE 64
69#define IVPU_MMU_STRTAB_ENT_COUNT 4
70#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2
71#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE
72
73#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */
74#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/highmem.h>
8

--- 58 unchanged lines hidden (view full) ---

67
68#define IVPU_MMU_STRTAB_ENT_SIZE 64
69#define IVPU_MMU_STRTAB_ENT_COUNT 4
70#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2
71#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE
72
73#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */
74#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
75#define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1)
76#define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1)
77#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)
75#define IVPU_MMU_Q_WRAP_MASK GENMASK(IVPU_MMU_Q_COUNT_LOG2, 0)
76#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)
78#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)
77#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)
78#define IVPU_MMU_Q_WRP(val) ((val) & IVPU_MMU_Q_COUNT)
79
80#define IVPU_MMU_CMDQ_CMD_SIZE 16
81#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
82
83#define IVPU_MMU_EVTQ_CMD_SIZE 32
84#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
85
86#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)

--- 383 unchanged lines hidden (view full) ---

470 if (ret)
471 return ret;
472
473 cmdq->cons = cmdq->prod;
474
475 return 0;
476}
477
79
80#define IVPU_MMU_CMDQ_CMD_SIZE 16
81#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
82
83#define IVPU_MMU_EVTQ_CMD_SIZE 32
84#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
85
86#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)

--- 383 unchanged lines hidden (view full) ---

470 if (ret)
471 return ret;
472
473 cmdq->cons = cmdq->prod;
474
475 return 0;
476}
477
478static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q)
479{
480 return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&
481 (IVPU_MMU_Q_WRP(q->prod) != IVPU_MMU_Q_WRP(q->cons)));
482}
483
484static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q)
485{
486 return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&
487 (IVPU_MMU_Q_WRP(q->prod) == IVPU_MMU_Q_WRP(q->cons)));
488}
489
478static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
479{
490static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
491{
480 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
481 u64 *queue_buffer = q->base;
482 int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
492 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
493 u64 *queue_buffer = cmdq->base;
494 int idx = IVPU_MMU_Q_IDX(cmdq->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
483
495
484 if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) {
496 if (ivpu_mmu_queue_is_full(cmdq)) {
485 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
486 return -EBUSY;
487 }
488
489 queue_buffer[idx] = data0;
490 queue_buffer[idx + 1] = data1;
497 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
498 return -EBUSY;
499 }
500
501 queue_buffer[idx] = data0;
502 queue_buffer[idx + 1] = data1;
491 q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
503 cmdq->prod = (cmdq->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
492
493 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
494
495 return 0;
496}
497
498static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
499{

--- 368 unchanged lines hidden (view full) ---

868
869static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
870{
871 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
872 u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
873 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);
874
875 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
504
505 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
506
507 return 0;
508}
509
510static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
511{

--- 368 unchanged lines hidden (view full) ---

880
881static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
882{
883 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
884 u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
885 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);
886
887 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
876 if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT))
888 if (ivpu_mmu_queue_is_empty(evtq))
877 return NULL;
878
879 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
889 return NULL;
890
891 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
880 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, evtq->cons);
881
882 return evt;
883}
884
885void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
886{
887 u32 *event;
888 u32 ssid;
889

--- 4 unchanged lines hidden (view full) ---

894
895 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
896 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) {
897 ivpu_pm_trigger_recovery(vdev, "MMU event");
898 return;
899 }
900
901 ivpu_mmu_user_context_mark_invalid(vdev, ssid);
892 return evt;
893}
894
895void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
896{
897 u32 *event;
898 u32 ssid;
899

--- 4 unchanged lines hidden (view full) ---

904
905 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
906 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) {
907 ivpu_pm_trigger_recovery(vdev, "MMU event");
908 return;
909 }
910
911 ivpu_mmu_user_context_mark_invalid(vdev, ssid);
912 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons);
902 }
903}
904
905void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)
906{
907 u32 *event;
908
909 while ((event = ivpu_mmu_get_event(vdev)) != NULL)

--- 49 unchanged lines hidden --- 		913 }
914}
915
916void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)
917{
918 u32 *event;
919
920 while ((event = ivpu_mmu_get_event(vdev)) != NULL)

--- 49 unchanged lines hidden ---