1fa5d823bSSanjay R Mehta /* SPDX-License-Identifier: GPL-2.0-only */ 2fa5d823bSSanjay R Mehta /* 3fa5d823bSSanjay R Mehta * AMD Passthru DMA device driver 4fa5d823bSSanjay R Mehta * -- Based on the CCP driver 5fa5d823bSSanjay R Mehta * 6fa5d823bSSanjay R Mehta * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. 7fa5d823bSSanjay R Mehta * 8fa5d823bSSanjay R Mehta * Author: Sanjay R Mehta <sanju.mehta@amd.com> 9fa5d823bSSanjay R Mehta * Author: Tom Lendacky <thomas.lendacky@amd.com> 10fa5d823bSSanjay R Mehta * Author: Gary R Hook <gary.hook@amd.com> 11fa5d823bSSanjay R Mehta */ 12fa5d823bSSanjay R Mehta 13fa5d823bSSanjay R Mehta #ifndef __PT_DEV_H__ 14fa5d823bSSanjay R Mehta #define __PT_DEV_H__ 15fa5d823bSSanjay R Mehta 16fa5d823bSSanjay R Mehta #include <linux/device.h> 17b0b4a6b1SSanjay R Mehta #include <linux/dmaengine.h> 18fa5d823bSSanjay R Mehta #include <linux/pci.h> 19fa5d823bSSanjay R Mehta #include <linux/spinlock.h> 20fa5d823bSSanjay R Mehta #include <linux/mutex.h> 21fa5d823bSSanjay R Mehta #include <linux/list.h> 22fa5d823bSSanjay R Mehta #include <linux/wait.h> 23fa5d823bSSanjay R Mehta #include <linux/dmapool.h> 24fa5d823bSSanjay R Mehta 25b0b4a6b1SSanjay R Mehta #include "../virt-dma.h" 26b0b4a6b1SSanjay R Mehta 27fa5d823bSSanjay R Mehta #define MAX_PT_NAME_LEN 16 28fa5d823bSSanjay R Mehta #define MAX_DMAPOOL_NAME_LEN 32 29fa5d823bSSanjay R Mehta 30fa5d823bSSanjay R Mehta #define MAX_HW_QUEUES 1 31fa5d823bSSanjay R Mehta #define MAX_CMD_QLEN 100 32fa5d823bSSanjay R Mehta 33fa5d823bSSanjay R Mehta #define PT_ENGINE_PASSTHRU 5 34fa5d823bSSanjay R Mehta 35fa5d823bSSanjay R Mehta /* Register Mappings */ 36fa5d823bSSanjay R Mehta #define IRQ_MASK_REG 0x040 37fa5d823bSSanjay R Mehta #define IRQ_STATUS_REG 0x200 38fa5d823bSSanjay R Mehta 39fa5d823bSSanjay R Mehta #define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) 40fa5d823bSSanjay R Mehta 41fa5d823bSSanjay R Mehta #define CMD_QUEUE_PRIO_OFFSET 0x00 42fa5d823bSSanjay R Mehta #define CMD_REQID_CONFIG_OFFSET 0x04 43fa5d823bSSanjay R Mehta #define CMD_TIMEOUT_OFFSET 0x08 44fa5d823bSSanjay R Mehta #define CMD_PT_VERSION 0x10 45fa5d823bSSanjay R Mehta 46fa5d823bSSanjay R Mehta #define CMD_Q_CONTROL_BASE 0x0000 47fa5d823bSSanjay R Mehta #define CMD_Q_TAIL_LO_BASE 0x0004 48fa5d823bSSanjay R Mehta #define CMD_Q_HEAD_LO_BASE 0x0008 49fa5d823bSSanjay R Mehta #define CMD_Q_INT_ENABLE_BASE 0x000C 50fa5d823bSSanjay R Mehta #define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 51fa5d823bSSanjay R Mehta 52fa5d823bSSanjay R Mehta #define CMD_Q_STATUS_BASE 0x0100 53fa5d823bSSanjay R Mehta #define CMD_Q_INT_STATUS_BASE 0x0104 54fa5d823bSSanjay R Mehta #define CMD_Q_DMA_STATUS_BASE 0x0108 55fa5d823bSSanjay R Mehta #define CMD_Q_DMA_READ_STATUS_BASE 0x010C 56fa5d823bSSanjay R Mehta #define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110 57fa5d823bSSanjay R Mehta #define CMD_Q_ABORT_BASE 0x0114 58fa5d823bSSanjay R Mehta #define CMD_Q_AX_CACHE_BASE 0x0118 59fa5d823bSSanjay R Mehta 60fa5d823bSSanjay R Mehta #define CMD_CONFIG_OFFSET 0x1120 61fa5d823bSSanjay R Mehta #define CMD_CLK_GATE_CTL_OFFSET 0x6004 62fa5d823bSSanjay R Mehta 63fa5d823bSSanjay R Mehta #define CMD_DESC_DW0_VAL 0x500012 64fa5d823bSSanjay R Mehta 65fa5d823bSSanjay R Mehta /* Address offset for virtual queue registers */ 66fa5d823bSSanjay R Mehta #define CMD_Q_STATUS_INCR 0x1000 67fa5d823bSSanjay R Mehta 68fa5d823bSSanjay R Mehta /* Bit masks */ 69fa5d823bSSanjay R Mehta #define CMD_CONFIG_REQID 0 70fa5d823bSSanjay R Mehta #define CMD_TIMEOUT_DISABLE 0 71fa5d823bSSanjay R Mehta #define CMD_CLK_DYN_GATING_DIS 0 72fa5d823bSSanjay R Mehta #define CMD_CLK_SW_GATE_MODE 0 73fa5d823bSSanjay R Mehta #define CMD_CLK_GATE_CTL 0 74fa5d823bSSanjay R Mehta #define CMD_QUEUE_PRIO GENMASK(2, 1) 75fa5d823bSSanjay R Mehta #define CMD_CONFIG_VHB_EN BIT(0) 76fa5d823bSSanjay R Mehta #define CMD_CLK_DYN_GATING_EN BIT(0) 77fa5d823bSSanjay R Mehta #define CMD_CLK_HW_GATE_MODE BIT(0) 78fa5d823bSSanjay R Mehta #define CMD_CLK_GATE_ON_DELAY BIT(12) 79fa5d823bSSanjay R Mehta #define CMD_CLK_GATE_OFF_DELAY BIT(12) 80fa5d823bSSanjay R Mehta 81fa5d823bSSanjay R Mehta #define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \ 82fa5d823bSSanjay R Mehta CMD_CLK_HW_GATE_MODE | \ 83fa5d823bSSanjay R Mehta CMD_CLK_GATE_ON_DELAY | \ 84fa5d823bSSanjay R Mehta CMD_CLK_DYN_GATING_EN | \ 85fa5d823bSSanjay R Mehta CMD_CLK_GATE_OFF_DELAY) 86fa5d823bSSanjay R Mehta 87fa5d823bSSanjay R Mehta #define CMD_Q_LEN 32 88fa5d823bSSanjay R Mehta #define CMD_Q_RUN BIT(0) 89fa5d823bSSanjay R Mehta #define CMD_Q_HALT BIT(1) 90fa5d823bSSanjay R Mehta #define CMD_Q_MEM_LOCATION BIT(2) 91fa5d823bSSanjay R Mehta #define CMD_Q_SIZE_MASK GENMASK(4, 0) 92fa5d823bSSanjay R Mehta #define CMD_Q_SIZE GENMASK(7, 3) 93fa5d823bSSanjay R Mehta #define CMD_Q_SHIFT GENMASK(1, 0) 94fa5d823bSSanjay R Mehta #define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \ 95fa5d823bSSanjay R Mehta CMD_Q_SIZE_MASK) 96fa5d823bSSanjay R Mehta #define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) 97fa5d823bSSanjay R Mehta #define Q_DESC_SIZE sizeof(struct ptdma_desc) 98fa5d823bSSanjay R Mehta #define Q_SIZE(n) (CMD_Q_LEN * (n)) 99fa5d823bSSanjay R Mehta 100fa5d823bSSanjay R Mehta #define INT_COMPLETION BIT(0) 101fa5d823bSSanjay R Mehta #define INT_ERROR BIT(1) 102fa5d823bSSanjay R Mehta #define INT_QUEUE_STOPPED BIT(2) 103fa5d823bSSanjay R Mehta #define INT_EMPTY_QUEUE BIT(3) 104fa5d823bSSanjay R Mehta #define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) 105fa5d823bSSanjay R Mehta 106fa5d823bSSanjay R Mehta /****** Local Storage Block ******/ 107fa5d823bSSanjay R Mehta #define LSB_START 0 108fa5d823bSSanjay R Mehta #define LSB_END 127 109fa5d823bSSanjay R Mehta #define LSB_COUNT (LSB_END - LSB_START + 1) 110fa5d823bSSanjay R Mehta 111fa5d823bSSanjay R Mehta #define PT_DMAPOOL_MAX_SIZE 64 112fa5d823bSSanjay R Mehta #define PT_DMAPOOL_ALIGN BIT(5) 113fa5d823bSSanjay R Mehta 114fa5d823bSSanjay R Mehta #define PT_PASSTHRU_BLOCKSIZE 512 115fa5d823bSSanjay R Mehta 116fa5d823bSSanjay R Mehta struct pt_device; 117fa5d823bSSanjay R Mehta 118fa5d823bSSanjay R Mehta struct pt_tasklet_data { 119fa5d823bSSanjay R Mehta struct completion completion; 120fa5d823bSSanjay R Mehta struct pt_cmd *cmd; 121fa5d823bSSanjay R Mehta }; 122fa5d823bSSanjay R Mehta 123fa5d823bSSanjay R Mehta /* 124fa5d823bSSanjay R Mehta * struct pt_passthru_engine - pass-through operation 125fa5d823bSSanjay R Mehta * without performing DMA mapping 126fa5d823bSSanjay R Mehta * @mask: mask to be applied to data 127fa5d823bSSanjay R Mehta * @mask_len: length in bytes of mask 128fa5d823bSSanjay R Mehta * @src_dma: data to be used for this operation 129fa5d823bSSanjay R Mehta * @dst_dma: data produced by this operation 130fa5d823bSSanjay R Mehta * @src_len: length in bytes of data used for this operation 131fa5d823bSSanjay R Mehta * 132fa5d823bSSanjay R Mehta * Variables required to be set when calling pt_enqueue_cmd(): 133fa5d823bSSanjay R Mehta * - bit_mod, byte_swap, src, dst, src_len 134fa5d823bSSanjay R Mehta * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP 135fa5d823bSSanjay R Mehta */ 136fa5d823bSSanjay R Mehta struct pt_passthru_engine { 137fa5d823bSSanjay R Mehta dma_addr_t mask; 138fa5d823bSSanjay R Mehta u32 mask_len; /* In bytes */ 139fa5d823bSSanjay R Mehta 140fa5d823bSSanjay R Mehta dma_addr_t src_dma, dst_dma; 141fa5d823bSSanjay R Mehta u64 src_len; /* In bytes */ 142fa5d823bSSanjay R Mehta }; 143fa5d823bSSanjay R Mehta 144fa5d823bSSanjay R Mehta /* 145fa5d823bSSanjay R Mehta * struct pt_cmd - PTDMA operation request 146fa5d823bSSanjay R Mehta * @entry: list element 147fa5d823bSSanjay R Mehta * @work: work element used for callbacks 148fa5d823bSSanjay R Mehta * @pt: PT device to be run on 149fa5d823bSSanjay R Mehta * @ret: operation return code 150fa5d823bSSanjay R Mehta * @flags: cmd processing flags 151fa5d823bSSanjay R Mehta * @engine: PTDMA operation to perform (passthru) 152fa5d823bSSanjay R Mehta * @engine_error: PT engine return code 153fa5d823bSSanjay R Mehta * @passthru: engine specific structures, refer to specific engine struct below 154fa5d823bSSanjay R Mehta * @callback: operation completion callback function 155fa5d823bSSanjay R Mehta * @data: parameter value to be supplied to the callback function 156fa5d823bSSanjay R Mehta * 157fa5d823bSSanjay R Mehta * Variables required to be set when calling pt_enqueue_cmd(): 158fa5d823bSSanjay R Mehta * - engine, callback 159fa5d823bSSanjay R Mehta * - See the operation structures below for what is required for each 160fa5d823bSSanjay R Mehta * operation. 161fa5d823bSSanjay R Mehta */ 162fa5d823bSSanjay R Mehta struct pt_cmd { 163fa5d823bSSanjay R Mehta struct list_head entry; 164fa5d823bSSanjay R Mehta struct work_struct work; 165fa5d823bSSanjay R Mehta struct pt_device *pt; 166fa5d823bSSanjay R Mehta int ret; 167fa5d823bSSanjay R Mehta u32 engine; 168fa5d823bSSanjay R Mehta u32 engine_error; 169fa5d823bSSanjay R Mehta struct pt_passthru_engine passthru; 170fa5d823bSSanjay R Mehta /* Completion callback support */ 171fa5d823bSSanjay R Mehta void (*pt_cmd_callback)(void *data, int err); 172fa5d823bSSanjay R Mehta void *data; 173fa5d823bSSanjay R Mehta }; 174fa5d823bSSanjay R Mehta 175b0b4a6b1SSanjay R Mehta struct pt_dma_desc { 176b0b4a6b1SSanjay R Mehta struct virt_dma_desc vd; 177b0b4a6b1SSanjay R Mehta struct pt_device *pt; 178b0b4a6b1SSanjay R Mehta enum dma_status status; 179b0b4a6b1SSanjay R Mehta size_t len; 180b0b4a6b1SSanjay R Mehta bool issued_to_hw; 181b0b4a6b1SSanjay R Mehta struct pt_cmd pt_cmd; 182b0b4a6b1SSanjay R Mehta }; 183b0b4a6b1SSanjay R Mehta 184b0b4a6b1SSanjay R Mehta struct pt_dma_chan { 185b0b4a6b1SSanjay R Mehta struct virt_dma_chan vc; 186b0b4a6b1SSanjay R Mehta struct pt_device *pt; 187b0b4a6b1SSanjay R Mehta }; 188b0b4a6b1SSanjay R Mehta 189fa5d823bSSanjay R Mehta struct pt_cmd_queue { 190fa5d823bSSanjay R Mehta struct pt_device *pt; 191fa5d823bSSanjay R Mehta 192fa5d823bSSanjay R Mehta /* Queue dma pool */ 193fa5d823bSSanjay R Mehta struct dma_pool *dma_pool; 194fa5d823bSSanjay R Mehta 195*a688efeaSAmit Vadhavana /* Queue base address (not necessarily aligned)*/ 196fa5d823bSSanjay R Mehta struct ptdma_desc *qbase; 197fa5d823bSSanjay R Mehta 198fa5d823bSSanjay R Mehta /* Aligned queue start address (per requirement) */ 19995e5fda3SEric Pilmore spinlock_t q_lock ____cacheline_aligned; 200fa5d823bSSanjay R Mehta unsigned int qidx; 201fa5d823bSSanjay R Mehta 202fa5d823bSSanjay R Mehta unsigned int qsize; 203fa5d823bSSanjay R Mehta dma_addr_t qbase_dma; 204fa5d823bSSanjay R Mehta dma_addr_t qdma_tail; 205fa5d823bSSanjay R Mehta 206fa5d823bSSanjay R Mehta unsigned int active; 207fa5d823bSSanjay R Mehta unsigned int suspended; 208fa5d823bSSanjay R Mehta 209d9650682SIlya Novikov /* Interrupt flag */ 210d9650682SIlya Novikov bool int_en; 211d9650682SIlya Novikov 212fa5d823bSSanjay R Mehta /* Register addresses for queue */ 213fa5d823bSSanjay R Mehta void __iomem *reg_control; 214fa5d823bSSanjay R Mehta u32 qcontrol; /* Cached control register */ 215fa5d823bSSanjay R Mehta 216fa5d823bSSanjay R Mehta /* Status values from job */ 217fa5d823bSSanjay R Mehta u32 int_status; 218fa5d823bSSanjay R Mehta u32 q_status; 219fa5d823bSSanjay R Mehta u32 q_int_status; 220fa5d823bSSanjay R Mehta u32 cmd_error; 221e2fb2e2aSSanjay R Mehta /* Queue Statistics */ 222e2fb2e2aSSanjay R Mehta unsigned long total_pt_ops; 223fa5d823bSSanjay R Mehta } ____cacheline_aligned; 224fa5d823bSSanjay R Mehta 225fa5d823bSSanjay R Mehta struct pt_device { 226fa5d823bSSanjay R Mehta struct list_head entry; 227fa5d823bSSanjay R Mehta 228fa5d823bSSanjay R Mehta unsigned int ord; 229fa5d823bSSanjay R Mehta char name[MAX_PT_NAME_LEN]; 230fa5d823bSSanjay R Mehta 231fa5d823bSSanjay R Mehta struct device *dev; 232fa5d823bSSanjay R Mehta 233fa5d823bSSanjay R Mehta /* Bus specific device information */ 234fa5d823bSSanjay R Mehta struct pt_msix *pt_msix; 235fa5d823bSSanjay R Mehta 236fa5d823bSSanjay R Mehta struct pt_dev_vdata *dev_vdata; 237fa5d823bSSanjay R Mehta 238fa5d823bSSanjay R Mehta unsigned int pt_irq; 239fa5d823bSSanjay R Mehta 240fa5d823bSSanjay R Mehta /* I/O area used for device communication */ 241fa5d823bSSanjay R Mehta void __iomem *io_regs; 242fa5d823bSSanjay R Mehta 243fa5d823bSSanjay R Mehta spinlock_t cmd_lock ____cacheline_aligned; 244fa5d823bSSanjay R Mehta unsigned int cmd_count; 245fa5d823bSSanjay R Mehta struct list_head cmd; 246fa5d823bSSanjay R Mehta 247fa5d823bSSanjay R Mehta /* 248fa5d823bSSanjay R Mehta * The command queue. This represent the queue available on the 249fa5d823bSSanjay R Mehta * PTDMA that are available for processing cmds 250fa5d823bSSanjay R Mehta */ 251fa5d823bSSanjay R Mehta struct pt_cmd_queue cmd_q; 252fa5d823bSSanjay R Mehta 253b0b4a6b1SSanjay R Mehta /* Support for the DMA Engine capabilities */ 254b0b4a6b1SSanjay R Mehta struct dma_device dma_dev; 255b0b4a6b1SSanjay R Mehta struct pt_dma_chan *pt_dma_chan; 256b0b4a6b1SSanjay R Mehta struct kmem_cache *dma_cmd_cache; 257b0b4a6b1SSanjay R Mehta struct kmem_cache *dma_desc_cache; 258b0b4a6b1SSanjay R Mehta 259fa5d823bSSanjay R Mehta wait_queue_head_t lsb_queue; 260fa5d823bSSanjay R Mehta 261e2fb2e2aSSanjay R Mehta /* Device Statistics */ 262e2fb2e2aSSanjay R Mehta unsigned long total_interrupts; 263e2fb2e2aSSanjay R Mehta 264fa5d823bSSanjay R Mehta struct pt_tasklet_data tdata; 265fa5d823bSSanjay R Mehta }; 266fa5d823bSSanjay R Mehta 267fa5d823bSSanjay R Mehta /* 268fa5d823bSSanjay R Mehta * descriptor for PTDMA commands 269fa5d823bSSanjay R Mehta * 8 32-bit words: 270fa5d823bSSanjay R Mehta * word 0: function; engine; control bits 271fa5d823bSSanjay R Mehta * word 1: length of source data 272fa5d823bSSanjay R Mehta * word 2: low 32 bits of source pointer 273fa5d823bSSanjay R Mehta * word 3: upper 16 bits of source pointer; source memory type 274fa5d823bSSanjay R Mehta * word 4: low 32 bits of destination pointer 275fa5d823bSSanjay R Mehta * word 5: upper 16 bits of destination pointer; destination memory type 276fa5d823bSSanjay R Mehta * word 6: reserved 32 bits 277fa5d823bSSanjay R Mehta * word 7: reserved 32 bits 278fa5d823bSSanjay R Mehta */ 279fa5d823bSSanjay R Mehta 280fa5d823bSSanjay R Mehta #define DWORD0_SOC BIT(0) 281fa5d823bSSanjay R Mehta #define DWORD0_IOC BIT(1) 282fa5d823bSSanjay R Mehta 283fa5d823bSSanjay R Mehta struct dword3 { 284fa5d823bSSanjay R Mehta unsigned int src_hi:16; 285fa5d823bSSanjay R Mehta unsigned int src_mem:2; 286fa5d823bSSanjay R Mehta unsigned int lsb_cxt_id:8; 287fa5d823bSSanjay R Mehta unsigned int rsvd1:5; 288fa5d823bSSanjay R Mehta unsigned int fixed:1; 289fa5d823bSSanjay R Mehta }; 290fa5d823bSSanjay R Mehta 291fa5d823bSSanjay R Mehta struct dword5 { 292fa5d823bSSanjay R Mehta unsigned int dst_hi:16; 293fa5d823bSSanjay R Mehta unsigned int dst_mem:2; 294fa5d823bSSanjay R Mehta unsigned int rsvd1:13; 295fa5d823bSSanjay R Mehta unsigned int fixed:1; 296fa5d823bSSanjay R Mehta }; 297fa5d823bSSanjay R Mehta 298fa5d823bSSanjay R Mehta struct ptdma_desc { 299fa5d823bSSanjay R Mehta u32 dw0; 300fa5d823bSSanjay R Mehta u32 length; 301fa5d823bSSanjay R Mehta u32 src_lo; 302fa5d823bSSanjay R Mehta struct dword3 dw3; 303fa5d823bSSanjay R Mehta u32 dst_lo; 304fa5d823bSSanjay R Mehta struct dword5 dw5; 305fa5d823bSSanjay R Mehta __le32 rsvd1; 306fa5d823bSSanjay R Mehta __le32 rsvd2; 307fa5d823bSSanjay R Mehta }; 308fa5d823bSSanjay R Mehta 309fa5d823bSSanjay R Mehta /* Structure to hold PT device data */ 310fa5d823bSSanjay R Mehta struct pt_dev_vdata { 311fa5d823bSSanjay R Mehta const unsigned int bar; 312fa5d823bSSanjay R Mehta }; 313fa5d823bSSanjay R Mehta 314b0b4a6b1SSanjay R Mehta int pt_dmaengine_register(struct pt_device *pt); 315b0b4a6b1SSanjay R Mehta void pt_dmaengine_unregister(struct pt_device *pt); 316b0b4a6b1SSanjay R Mehta 317e2fb2e2aSSanjay R Mehta void ptdma_debugfs_setup(struct pt_device *pt); 318fa5d823bSSanjay R Mehta int pt_core_init(struct pt_device *pt); 319fa5d823bSSanjay R Mehta void pt_core_destroy(struct pt_device *pt); 320fa5d823bSSanjay R Mehta 321fa5d823bSSanjay R Mehta int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, 322fa5d823bSSanjay R Mehta struct pt_passthru_engine *pt_engine); 323fa5d823bSSanjay R Mehta 324d9650682SIlya Novikov void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q); 325fa5d823bSSanjay R Mehta void pt_start_queue(struct pt_cmd_queue *cmd_q); 326fa5d823bSSanjay R Mehta void pt_stop_queue(struct pt_cmd_queue *cmd_q); 327fa5d823bSSanjay R Mehta 328d9650682SIlya Novikov static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) 329d9650682SIlya Novikov { 330d9650682SIlya Novikov iowrite32(0, pt->cmd_q.reg_control + 0x000C); 331d9650682SIlya Novikov } 332d9650682SIlya Novikov 333d9650682SIlya Novikov static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) 334d9650682SIlya Novikov { 335d9650682SIlya Novikov iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); 336d9650682SIlya Novikov } 337fa5d823bSSanjay R Mehta #endif 338