1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2019-2022 HiSilicon Limited. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/dmaengine.h> 6 #include <linux/init.h> 7 #include <linux/iopoll.h> 8 #include <linux/module.h> 9 #include <linux/pci.h> 10 #include <linux/spinlock.h> 11 #include "virt-dma.h" 12 13 /* HiSilicon DMA register common field define */ 14 #define HISI_DMA_Q_SQ_BASE_L 0x0 15 #define HISI_DMA_Q_SQ_BASE_H 0x4 16 #define HISI_DMA_Q_SQ_DEPTH 0x8 17 #define HISI_DMA_Q_SQ_TAIL_PTR 0xc 18 #define HISI_DMA_Q_CQ_BASE_L 0x10 19 #define HISI_DMA_Q_CQ_BASE_H 0x14 20 #define HISI_DMA_Q_CQ_DEPTH 0x18 21 #define HISI_DMA_Q_CQ_HEAD_PTR 0x1c 22 #define HISI_DMA_Q_CTRL0 0x20 23 #define HISI_DMA_Q_CTRL0_QUEUE_EN BIT(0) 24 #define HISI_DMA_Q_CTRL0_QUEUE_PAUSE BIT(4) 25 #define HISI_DMA_Q_CTRL1 0x24 26 #define HISI_DMA_Q_CTRL1_QUEUE_RESET BIT(0) 27 #define HISI_DMA_Q_FSM_STS 0x30 28 #define HISI_DMA_Q_FSM_STS_MASK GENMASK(3, 0) 29 #define HISI_DMA_Q_ERR_INT_NUM0 0x84 30 #define HISI_DMA_Q_ERR_INT_NUM1 0x88 31 #define HISI_DMA_Q_ERR_INT_NUM2 0x8c 32 33 /* HiSilicon IP08 DMA register and field define */ 34 #define HISI_DMA_HIP08_MODE 0x217C 35 #define HISI_DMA_HIP08_Q_BASE 0x0 36 #define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN BIT(2) 37 #define HISI_DMA_HIP08_Q_INT_STS 0x40 38 #define HISI_DMA_HIP08_Q_INT_MSK 0x44 39 #define HISI_DMA_HIP08_Q_INT_STS_MASK GENMASK(14, 0) 40 #define HISI_DMA_HIP08_Q_ERR_INT_NUM3 0x90 41 #define HISI_DMA_HIP08_Q_ERR_INT_NUM4 0x94 42 #define HISI_DMA_HIP08_Q_ERR_INT_NUM5 0x98 43 #define HISI_DMA_HIP08_Q_ERR_INT_NUM6 0x48 44 #define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT BIT(24) 45 46 /* HiSilicon IP09 DMA register and field define */ 47 #define HISI_DMA_HIP09_DMA_FLR_DISABLE 0xA00 48 #define HISI_DMA_HIP09_DMA_FLR_DISABLE_B BIT(0) 49 #define HISI_DMA_HIP09_Q_BASE 0x2000 50 #define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN GENMASK(31, 28) 51 #define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT BIT(26) 52 #define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT BIT(27) 53 #define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE BIT(2) 54 #define HISI_DMA_HIP09_Q_INT_STS 0x40 55 #define HISI_DMA_HIP09_Q_INT_MSK 0x44 56 #define HISI_DMA_HIP09_Q_INT_STS_MASK 0x1 57 #define HISI_DMA_HIP09_Q_ERR_INT_STS 0x48 58 #define HISI_DMA_HIP09_Q_ERR_INT_MSK 0x4C 59 #define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK GENMASK(18, 1) 60 #define HISI_DMA_HIP09_PORT_CFG_REG(port_id) (0x800 + \ 61 (port_id) * 0x20) 62 #define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B BIT(16) 63 64 #define HISI_DMA_HIP09_MAX_PORT_NUM 16 65 66 #define HISI_DMA_HIP08_MSI_NUM 32 67 #define HISI_DMA_HIP08_CHAN_NUM 30 68 #define HISI_DMA_HIP09_MSI_NUM 4 69 #define HISI_DMA_HIP09_CHAN_NUM 4 70 #define HISI_DMA_REVISION_HIP08B 0x21 71 #define HISI_DMA_REVISION_HIP09A 0x30 72 73 #define HISI_DMA_Q_OFFSET 0x100 74 #define HISI_DMA_Q_DEPTH_VAL 1024 75 76 #define PCI_BAR_2 2 77 78 #define HISI_DMA_POLL_Q_STS_DELAY_US 10 79 #define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000 80 81 #define HISI_DMA_MAX_DIR_NAME_LEN 128 82 83 /* 84 * The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they 85 * have the same pci device id but different pci revision. 86 * Unfortunately, they have different register layouts, so two layout 87 * enumerations are defined. 88 */ 89 enum hisi_dma_reg_layout { 90 HISI_DMA_REG_LAYOUT_INVALID = 0, 91 HISI_DMA_REG_LAYOUT_HIP08, 92 HISI_DMA_REG_LAYOUT_HIP09 93 }; 94 95 enum hisi_dma_mode { 96 EP = 0, 97 RC, 98 }; 99 100 enum hisi_dma_chan_status { 101 DISABLE = -1, 102 IDLE = 0, 103 RUN, 104 CPL, 105 PAUSE, 106 HALT, 107 ABORT, 108 WAIT, 109 BUFFCLR, 110 }; 111 112 struct hisi_dma_sqe { 113 __le32 dw0; 114 #define OPCODE_MASK GENMASK(3, 0) 115 #define OPCODE_SMALL_PACKAGE 0x1 116 #define OPCODE_M2M 0x4 117 #define LOCAL_IRQ_EN BIT(8) 118 #define ATTR_SRC_MASK GENMASK(14, 12) 119 __le32 dw1; 120 __le32 dw2; 121 #define ATTR_DST_MASK GENMASK(26, 24) 122 __le32 length; 123 __le64 src_addr; 124 __le64 dst_addr; 125 }; 126 127 struct hisi_dma_cqe { 128 __le32 rsv0; 129 __le32 rsv1; 130 __le16 sq_head; 131 __le16 rsv2; 132 __le16 rsv3; 133 __le16 w0; 134 #define STATUS_MASK GENMASK(15, 1) 135 #define STATUS_SUCC 0x0 136 #define VALID_BIT BIT(0) 137 }; 138 139 struct hisi_dma_desc { 140 struct virt_dma_desc vd; 141 struct hisi_dma_sqe sqe; 142 }; 143 144 struct hisi_dma_chan { 145 struct virt_dma_chan vc; 146 struct hisi_dma_dev *hdma_dev; 147 struct hisi_dma_sqe *sq; 148 struct hisi_dma_cqe *cq; 149 dma_addr_t sq_dma; 150 dma_addr_t cq_dma; 151 u32 sq_tail; 152 u32 cq_head; 153 u32 qp_num; 154 enum hisi_dma_chan_status status; 155 struct hisi_dma_desc *desc; 156 }; 157 158 struct hisi_dma_dev { 159 struct pci_dev *pdev; 160 void __iomem *base; 161 struct dma_device dma_dev; 162 u32 chan_num; 163 u32 chan_depth; 164 enum hisi_dma_reg_layout reg_layout; 165 void __iomem *queue_base; /* queue region start of register */ 166 struct hisi_dma_chan chan[] __counted_by(chan_num); 167 }; 168 169 #ifdef CONFIG_DEBUG_FS 170 171 static const struct debugfs_reg32 hisi_dma_comm_chan_regs[] = { 172 {"DMA_QUEUE_SQ_DEPTH ", 0x0008ull}, 173 {"DMA_QUEUE_SQ_TAIL_PTR ", 0x000Cull}, 174 {"DMA_QUEUE_CQ_DEPTH ", 0x0018ull}, 175 {"DMA_QUEUE_CQ_HEAD_PTR ", 0x001Cull}, 176 {"DMA_QUEUE_CTRL0 ", 0x0020ull}, 177 {"DMA_QUEUE_CTRL1 ", 0x0024ull}, 178 {"DMA_QUEUE_FSM_STS ", 0x0030ull}, 179 {"DMA_QUEUE_SQ_STS ", 0x0034ull}, 180 {"DMA_QUEUE_CQ_TAIL_PTR ", 0x003Cull}, 181 {"DMA_QUEUE_INT_STS ", 0x0040ull}, 182 {"DMA_QUEUE_INT_MSK ", 0x0044ull}, 183 {"DMA_QUEUE_INT_RO ", 0x006Cull}, 184 }; 185 186 static const struct debugfs_reg32 hisi_dma_hip08_chan_regs[] = { 187 {"DMA_QUEUE_BYTE_CNT ", 0x0038ull}, 188 {"DMA_ERR_INT_NUM6 ", 0x0048ull}, 189 {"DMA_QUEUE_DESP0 ", 0x0050ull}, 190 {"DMA_QUEUE_DESP1 ", 0x0054ull}, 191 {"DMA_QUEUE_DESP2 ", 0x0058ull}, 192 {"DMA_QUEUE_DESP3 ", 0x005Cull}, 193 {"DMA_QUEUE_DESP4 ", 0x0074ull}, 194 {"DMA_QUEUE_DESP5 ", 0x0078ull}, 195 {"DMA_QUEUE_DESP6 ", 0x007Cull}, 196 {"DMA_QUEUE_DESP7 ", 0x0080ull}, 197 {"DMA_ERR_INT_NUM0 ", 0x0084ull}, 198 {"DMA_ERR_INT_NUM1 ", 0x0088ull}, 199 {"DMA_ERR_INT_NUM2 ", 0x008Cull}, 200 {"DMA_ERR_INT_NUM3 ", 0x0090ull}, 201 {"DMA_ERR_INT_NUM4 ", 0x0094ull}, 202 {"DMA_ERR_INT_NUM5 ", 0x0098ull}, 203 {"DMA_QUEUE_SQ_STS2 ", 0x00A4ull}, 204 }; 205 206 static const struct debugfs_reg32 hisi_dma_hip09_chan_regs[] = { 207 {"DMA_QUEUE_ERR_INT_STS ", 0x0048ull}, 208 {"DMA_QUEUE_ERR_INT_MSK ", 0x004Cull}, 209 {"DFX_SQ_READ_ERR_PTR ", 0x0068ull}, 210 {"DFX_DMA_ERR_INT_NUM0 ", 0x0084ull}, 211 {"DFX_DMA_ERR_INT_NUM1 ", 0x0088ull}, 212 {"DFX_DMA_ERR_INT_NUM2 ", 0x008Cull}, 213 {"DFX_DMA_QUEUE_SQ_STS2 ", 0x00A4ull}, 214 }; 215 216 static const struct debugfs_reg32 hisi_dma_hip08_comm_regs[] = { 217 {"DMA_ECC_ERR_ADDR ", 0x2004ull}, 218 {"DMA_ECC_ECC_CNT ", 0x2014ull}, 219 {"COMMON_AND_CH_ERR_STS ", 0x2030ull}, 220 {"LOCAL_CPL_ID_STS_0 ", 0x20E0ull}, 221 {"LOCAL_CPL_ID_STS_1 ", 0x20E4ull}, 222 {"LOCAL_CPL_ID_STS_2 ", 0x20E8ull}, 223 {"LOCAL_CPL_ID_STS_3 ", 0x20ECull}, 224 {"LOCAL_TLP_NUM ", 0x2158ull}, 225 {"SQCQ_TLP_NUM ", 0x2164ull}, 226 {"CPL_NUM ", 0x2168ull}, 227 {"INF_BACK_PRESS_STS ", 0x2170ull}, 228 {"DMA_CH_RAS_LEVEL ", 0x2184ull}, 229 {"DMA_CM_RAS_LEVEL ", 0x2188ull}, 230 {"DMA_CH_ERR_STS ", 0x2190ull}, 231 {"DMA_CH_DONE_STS ", 0x2194ull}, 232 {"DMA_SQ_TAG_STS_0 ", 0x21A0ull}, 233 {"DMA_SQ_TAG_STS_1 ", 0x21A4ull}, 234 {"DMA_SQ_TAG_STS_2 ", 0x21A8ull}, 235 {"DMA_SQ_TAG_STS_3 ", 0x21ACull}, 236 {"LOCAL_P_ID_STS_0 ", 0x21B0ull}, 237 {"LOCAL_P_ID_STS_1 ", 0x21B4ull}, 238 {"LOCAL_P_ID_STS_2 ", 0x21B8ull}, 239 {"LOCAL_P_ID_STS_3 ", 0x21BCull}, 240 {"DMA_PREBUFF_INFO_0 ", 0x2200ull}, 241 {"DMA_CM_TABLE_INFO_0 ", 0x2220ull}, 242 {"DMA_CM_CE_RO ", 0x2244ull}, 243 {"DMA_CM_NFE_RO ", 0x2248ull}, 244 {"DMA_CM_FE_RO ", 0x224Cull}, 245 }; 246 247 static const struct debugfs_reg32 hisi_dma_hip09_comm_regs[] = { 248 {"COMMON_AND_CH_ERR_STS ", 0x0030ull}, 249 {"DMA_PORT_IDLE_STS ", 0x0150ull}, 250 {"DMA_CH_RAS_LEVEL ", 0x0184ull}, 251 {"DMA_CM_RAS_LEVEL ", 0x0188ull}, 252 {"DMA_CM_CE_RO ", 0x0244ull}, 253 {"DMA_CM_NFE_RO ", 0x0248ull}, 254 {"DMA_CM_FE_RO ", 0x024Cull}, 255 {"DFX_INF_BACK_PRESS_STS0 ", 0x1A40ull}, 256 {"DFX_INF_BACK_PRESS_STS1 ", 0x1A44ull}, 257 {"DFX_INF_BACK_PRESS_STS2 ", 0x1A48ull}, 258 {"DFX_DMA_WRR_DISABLE ", 0x1A4Cull}, 259 {"DFX_PA_REQ_TLP_NUM ", 0x1C00ull}, 260 {"DFX_PA_BACK_TLP_NUM ", 0x1C04ull}, 261 {"DFX_PA_RETRY_TLP_NUM ", 0x1C08ull}, 262 {"DFX_LOCAL_NP_TLP_NUM ", 0x1C0Cull}, 263 {"DFX_LOCAL_CPL_HEAD_TLP_NUM ", 0x1C10ull}, 264 {"DFX_LOCAL_CPL_DATA_TLP_NUM ", 0x1C14ull}, 265 {"DFX_LOCAL_CPL_EXT_DATA_TLP_NUM ", 0x1C18ull}, 266 {"DFX_LOCAL_P_HEAD_TLP_NUM ", 0x1C1Cull}, 267 {"DFX_LOCAL_P_ACK_TLP_NUM ", 0x1C20ull}, 268 {"DFX_BUF_ALOC_PORT_REQ_NUM ", 0x1C24ull}, 269 {"DFX_BUF_ALOC_PORT_RESULT_NUM ", 0x1C28ull}, 270 {"DFX_BUF_FAIL_SIZE_NUM ", 0x1C2Cull}, 271 {"DFX_BUF_ALOC_SIZE_NUM ", 0x1C30ull}, 272 {"DFX_BUF_NP_RELEASE_SIZE_NUM ", 0x1C34ull}, 273 {"DFX_BUF_P_RELEASE_SIZE_NUM ", 0x1C38ull}, 274 {"DFX_BUF_PORT_RELEASE_SIZE_NUM ", 0x1C3Cull}, 275 {"DFX_DMA_PREBUF_MEM0_ECC_ERR_ADDR ", 0x1CA8ull}, 276 {"DFX_DMA_PREBUF_MEM0_ECC_CNT ", 0x1CACull}, 277 {"DFX_DMA_LOC_NP_OSTB_ECC_ERR_ADDR ", 0x1CB0ull}, 278 {"DFX_DMA_LOC_NP_OSTB_ECC_CNT ", 0x1CB4ull}, 279 {"DFX_DMA_PREBUF_MEM1_ECC_ERR_ADDR ", 0x1CC0ull}, 280 {"DFX_DMA_PREBUF_MEM1_ECC_CNT ", 0x1CC4ull}, 281 {"DMA_CH_DONE_STS ", 0x02E0ull}, 282 {"DMA_CH_ERR_STS ", 0x0320ull}, 283 }; 284 #endif /* CONFIG_DEBUG_FS*/ 285 286 static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev) 287 { 288 if (pdev->revision == HISI_DMA_REVISION_HIP08B) 289 return HISI_DMA_REG_LAYOUT_HIP08; 290 else if (pdev->revision >= HISI_DMA_REVISION_HIP09A) 291 return HISI_DMA_REG_LAYOUT_HIP09; 292 293 return HISI_DMA_REG_LAYOUT_INVALID; 294 } 295 296 static u32 hisi_dma_get_chan_num(struct pci_dev *pdev) 297 { 298 if (pdev->revision == HISI_DMA_REVISION_HIP08B) 299 return HISI_DMA_HIP08_CHAN_NUM; 300 301 return HISI_DMA_HIP09_CHAN_NUM; 302 } 303 304 static u32 hisi_dma_get_msi_num(struct pci_dev *pdev) 305 { 306 if (pdev->revision == HISI_DMA_REVISION_HIP08B) 307 return HISI_DMA_HIP08_MSI_NUM; 308 309 return HISI_DMA_HIP09_MSI_NUM; 310 } 311 312 static u32 hisi_dma_get_queue_base(struct pci_dev *pdev) 313 { 314 if (pdev->revision == HISI_DMA_REVISION_HIP08B) 315 return HISI_DMA_HIP08_Q_BASE; 316 317 return HISI_DMA_HIP09_Q_BASE; 318 } 319 320 static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c) 321 { 322 return container_of(c, struct hisi_dma_chan, vc.chan); 323 } 324 325 static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd) 326 { 327 return container_of(vd, struct hisi_dma_desc, vd); 328 } 329 330 static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index, 331 u32 val) 332 { 333 writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET); 334 } 335 336 static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val) 337 { 338 u32 tmp; 339 340 tmp = readl_relaxed(addr); 341 tmp = val ? tmp | pos : tmp & ~pos; 342 writel_relaxed(tmp, addr); 343 } 344 345 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index, 346 bool pause) 347 { 348 void __iomem *addr; 349 350 addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 + 351 index * HISI_DMA_Q_OFFSET; 352 hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause); 353 } 354 355 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index, 356 bool enable) 357 { 358 void __iomem *addr; 359 360 addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 + 361 index * HISI_DMA_Q_OFFSET; 362 hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable); 363 } 364 365 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) 366 { 367 void __iomem *q_base = hdma_dev->queue_base; 368 369 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) 370 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK, 371 qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK); 372 else { 373 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK, 374 qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK); 375 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK, 376 qp_index, 377 HISI_DMA_HIP09_Q_ERR_INT_STS_MASK); 378 } 379 } 380 381 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) 382 { 383 void __iomem *q_base = hdma_dev->queue_base; 384 385 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { 386 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS, 387 qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK); 388 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK, 389 qp_index, 0); 390 } else { 391 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS, 392 qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK); 393 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS, 394 qp_index, 395 HISI_DMA_HIP09_Q_ERR_INT_STS_MASK); 396 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK, 397 qp_index, 0); 398 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK, 399 qp_index, 0); 400 } 401 } 402 403 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index) 404 { 405 void __iomem *addr; 406 407 addr = hdma_dev->queue_base + 408 HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET; 409 hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1); 410 } 411 412 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index) 413 { 414 void __iomem *q_base = hdma_dev->queue_base; 415 416 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0); 417 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0); 418 } 419 420 static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan, 421 bool disable) 422 { 423 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 424 u32 index = chan->qp_num, tmp; 425 void __iomem *addr; 426 int ret; 427 428 hisi_dma_pause_dma(hdma_dev, index, true); 429 hisi_dma_enable_dma(hdma_dev, index, false); 430 hisi_dma_mask_irq(hdma_dev, index); 431 432 addr = hdma_dev->queue_base + 433 HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET; 434 435 ret = readl_relaxed_poll_timeout(addr, tmp, 436 FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN, 437 HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US); 438 if (ret) { 439 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n"); 440 WARN_ON(1); 441 } 442 443 hisi_dma_do_reset(hdma_dev, index); 444 hisi_dma_reset_qp_point(hdma_dev, index); 445 hisi_dma_pause_dma(hdma_dev, index, false); 446 447 if (!disable) { 448 hisi_dma_enable_dma(hdma_dev, index, true); 449 hisi_dma_unmask_irq(hdma_dev, index); 450 } 451 452 ret = readl_relaxed_poll_timeout(addr, tmp, 453 FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE, 454 HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US); 455 if (ret) { 456 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n"); 457 WARN_ON(1); 458 } 459 } 460 461 static void hisi_dma_free_chan_resources(struct dma_chan *c) 462 { 463 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 464 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 465 466 hisi_dma_reset_or_disable_hw_chan(chan, false); 467 vchan_free_chan_resources(&chan->vc); 468 469 memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth); 470 memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth); 471 chan->sq_tail = 0; 472 chan->cq_head = 0; 473 chan->status = DISABLE; 474 } 475 476 static void hisi_dma_desc_free(struct virt_dma_desc *vd) 477 { 478 kfree(to_hisi_dma_desc(vd)); 479 } 480 481 static struct dma_async_tx_descriptor * 482 hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src, 483 size_t len, unsigned long flags) 484 { 485 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 486 struct hisi_dma_desc *desc; 487 488 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 489 if (!desc) 490 return NULL; 491 492 desc->sqe.length = cpu_to_le32(len); 493 desc->sqe.src_addr = cpu_to_le64(src); 494 desc->sqe.dst_addr = cpu_to_le64(dst); 495 496 return vchan_tx_prep(&chan->vc, &desc->vd, flags); 497 } 498 499 static enum dma_status 500 hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, 501 struct dma_tx_state *txstate) 502 { 503 return dma_cookie_status(c, cookie, txstate); 504 } 505 506 static void hisi_dma_start_transfer(struct hisi_dma_chan *chan) 507 { 508 struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail; 509 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 510 struct hisi_dma_desc *desc; 511 struct virt_dma_desc *vd; 512 513 vd = vchan_next_desc(&chan->vc); 514 if (!vd) { 515 chan->desc = NULL; 516 return; 517 } 518 list_del(&vd->node); 519 desc = to_hisi_dma_desc(vd); 520 chan->desc = desc; 521 522 memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe)); 523 524 /* update other field in sqe */ 525 sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M)); 526 sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN); 527 528 /* make sure data has been updated in sqe */ 529 wmb(); 530 531 /* update sq tail, point to new sqe position */ 532 chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth; 533 534 /* update sq_tail to trigger a new task */ 535 hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR, 536 chan->qp_num, chan->sq_tail); 537 } 538 539 static void hisi_dma_issue_pending(struct dma_chan *c) 540 { 541 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 542 unsigned long flags; 543 544 spin_lock_irqsave(&chan->vc.lock, flags); 545 546 if (vchan_issue_pending(&chan->vc) && !chan->desc) 547 hisi_dma_start_transfer(chan); 548 549 spin_unlock_irqrestore(&chan->vc.lock, flags); 550 } 551 552 static int hisi_dma_terminate_all(struct dma_chan *c) 553 { 554 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 555 unsigned long flags; 556 LIST_HEAD(head); 557 558 spin_lock_irqsave(&chan->vc.lock, flags); 559 560 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true); 561 if (chan->desc) { 562 vchan_terminate_vdesc(&chan->desc->vd); 563 chan->desc = NULL; 564 } 565 566 vchan_get_all_descriptors(&chan->vc, &head); 567 568 spin_unlock_irqrestore(&chan->vc.lock, flags); 569 570 vchan_dma_desc_free_list(&chan->vc, &head); 571 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false); 572 573 return 0; 574 } 575 576 static void hisi_dma_synchronize(struct dma_chan *c) 577 { 578 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 579 580 vchan_synchronize(&chan->vc); 581 } 582 583 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev) 584 { 585 size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth; 586 size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth; 587 struct device *dev = &hdma_dev->pdev->dev; 588 struct hisi_dma_chan *chan; 589 int i; 590 591 for (i = 0; i < hdma_dev->chan_num; i++) { 592 chan = &hdma_dev->chan[i]; 593 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma, 594 GFP_KERNEL); 595 if (!chan->sq) 596 return -ENOMEM; 597 598 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma, 599 GFP_KERNEL); 600 if (!chan->cq) 601 return -ENOMEM; 602 } 603 604 return 0; 605 } 606 607 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index) 608 { 609 struct hisi_dma_chan *chan = &hdma_dev->chan[index]; 610 void __iomem *q_base = hdma_dev->queue_base; 611 u32 hw_depth = hdma_dev->chan_depth - 1; 612 void __iomem *addr; 613 u32 tmp; 614 615 /* set sq, cq base */ 616 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index, 617 lower_32_bits(chan->sq_dma)); 618 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index, 619 upper_32_bits(chan->sq_dma)); 620 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index, 621 lower_32_bits(chan->cq_dma)); 622 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index, 623 upper_32_bits(chan->cq_dma)); 624 625 /* set sq, cq depth */ 626 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth); 627 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth); 628 629 /* init sq tail and cq head */ 630 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0); 631 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0); 632 633 /* init error interrupt stats */ 634 hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0); 635 hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0); 636 hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0); 637 638 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { 639 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3, 640 index, 0); 641 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4, 642 index, 0); 643 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5, 644 index, 0); 645 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6, 646 index, 0); 647 /* 648 * init SQ/CQ direction selecting register. 649 * "0" is to local side and "1" is to remote side. 650 */ 651 addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET; 652 hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0); 653 654 /* 655 * 0 - Continue to next descriptor if error occurs. 656 * 1 - Abort the DMA queue if error occurs. 657 */ 658 hisi_dma_update_bit(addr, 659 HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0); 660 } else { 661 addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET; 662 663 /* 664 * init SQ/CQ direction selecting register. 665 * "0" is to local side and "1" is to remote side. 666 */ 667 hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0); 668 hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0); 669 670 /* 671 * 0 - Continue to next descriptor if error occurs. 672 * 1 - Abort the DMA queue if error occurs. 673 */ 674 675 tmp = readl_relaxed(addr); 676 tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN; 677 writel_relaxed(tmp, addr); 678 679 /* 680 * 0 - dma should process FLR whith CPU. 681 * 1 - dma not process FLR, only cpu process FLR. 682 */ 683 addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE + 684 index * HISI_DMA_Q_OFFSET; 685 hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0); 686 687 addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET; 688 hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1); 689 } 690 } 691 692 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) 693 { 694 hisi_dma_init_hw_qp(hdma_dev, qp_index); 695 hisi_dma_unmask_irq(hdma_dev, qp_index); 696 hisi_dma_enable_dma(hdma_dev, qp_index, true); 697 } 698 699 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) 700 { 701 hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true); 702 } 703 704 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev) 705 { 706 int i; 707 708 for (i = 0; i < hdma_dev->chan_num; i++) { 709 hdma_dev->chan[i].qp_num = i; 710 hdma_dev->chan[i].hdma_dev = hdma_dev; 711 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free; 712 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev); 713 hisi_dma_enable_qp(hdma_dev, i); 714 } 715 } 716 717 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev) 718 { 719 int i; 720 721 for (i = 0; i < hdma_dev->chan_num; i++) { 722 hisi_dma_disable_qp(hdma_dev, i); 723 tasklet_kill(&hdma_dev->chan[i].vc.task); 724 } 725 } 726 727 static irqreturn_t hisi_dma_irq(int irq, void *data) 728 { 729 struct hisi_dma_chan *chan = data; 730 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 731 struct hisi_dma_desc *desc; 732 struct hisi_dma_cqe *cqe; 733 void __iomem *q_base; 734 735 spin_lock(&chan->vc.lock); 736 737 desc = chan->desc; 738 cqe = chan->cq + chan->cq_head; 739 q_base = hdma_dev->queue_base; 740 if (desc) { 741 chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth; 742 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, 743 chan->qp_num, chan->cq_head); 744 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) { 745 vchan_cookie_complete(&desc->vd); 746 hisi_dma_start_transfer(chan); 747 } else { 748 dev_err(&hdma_dev->pdev->dev, "task error!\n"); 749 } 750 } 751 752 spin_unlock(&chan->vc.lock); 753 754 return IRQ_HANDLED; 755 } 756 757 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev) 758 { 759 struct pci_dev *pdev = hdma_dev->pdev; 760 int i, ret; 761 762 for (i = 0; i < hdma_dev->chan_num; i++) { 763 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i), 764 hisi_dma_irq, IRQF_SHARED, "hisi_dma", 765 &hdma_dev->chan[i]); 766 if (ret) 767 return ret; 768 } 769 770 return 0; 771 } 772 773 /* This function enables all hw channels in a device */ 774 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev) 775 { 776 int ret; 777 778 ret = hisi_dma_alloc_qps_mem(hdma_dev); 779 if (ret) { 780 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n"); 781 return ret; 782 } 783 784 ret = hisi_dma_request_qps_irq(hdma_dev); 785 if (ret) { 786 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n"); 787 return ret; 788 } 789 790 hisi_dma_enable_qps(hdma_dev); 791 792 return 0; 793 } 794 795 static void hisi_dma_disable_hw_channels(void *data) 796 { 797 hisi_dma_disable_qps(data); 798 } 799 800 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev, 801 enum hisi_dma_mode mode) 802 { 803 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) 804 writel_relaxed(mode == RC ? 1 : 0, 805 hdma_dev->base + HISI_DMA_HIP08_MODE); 806 } 807 808 static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev) 809 { 810 void __iomem *addr; 811 int i; 812 813 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) { 814 for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) { 815 addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i); 816 hisi_dma_update_bit(addr, 817 HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1); 818 } 819 } 820 } 821 822 static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev) 823 { 824 struct dma_device *dma_dev; 825 826 dma_dev = &hdma_dev->dma_dev; 827 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 828 dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources; 829 dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy; 830 dma_dev->device_tx_status = hisi_dma_tx_status; 831 dma_dev->device_issue_pending = hisi_dma_issue_pending; 832 dma_dev->device_terminate_all = hisi_dma_terminate_all; 833 dma_dev->device_synchronize = hisi_dma_synchronize; 834 dma_dev->directions = BIT(DMA_MEM_TO_MEM); 835 dma_dev->dev = &hdma_dev->pdev->dev; 836 INIT_LIST_HEAD(&dma_dev->channels); 837 } 838 839 /* --- debugfs implementation --- */ 840 #ifdef CONFIG_DEBUG_FS 841 #include <linux/debugfs.h> 842 static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev, 843 u32 *regs_sz) 844 { 845 struct device *dev = &hdma_dev->pdev->dev; 846 struct debugfs_reg32 *regs; 847 u32 regs_sz_comm; 848 849 regs_sz_comm = ARRAY_SIZE(hisi_dma_comm_chan_regs); 850 851 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) 852 *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip08_chan_regs); 853 else 854 *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip09_chan_regs); 855 856 regs = devm_kcalloc(dev, *regs_sz, sizeof(struct debugfs_reg32), 857 GFP_KERNEL); 858 if (!regs) 859 return NULL; 860 memcpy(regs, hisi_dma_comm_chan_regs, sizeof(hisi_dma_comm_chan_regs)); 861 862 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) 863 memcpy(regs + regs_sz_comm, hisi_dma_hip08_chan_regs, 864 sizeof(hisi_dma_hip08_chan_regs)); 865 else 866 memcpy(regs + regs_sz_comm, hisi_dma_hip09_chan_regs, 867 sizeof(hisi_dma_hip09_chan_regs)); 868 869 return regs; 870 } 871 872 static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev) 873 { 874 char dir_name[HISI_DMA_MAX_DIR_NAME_LEN]; 875 struct debugfs_regset32 *regsets; 876 struct debugfs_reg32 *regs; 877 struct dentry *chan_dir; 878 struct device *dev; 879 u32 regs_sz; 880 int ret; 881 int i; 882 883 dev = &hdma_dev->pdev->dev; 884 885 regsets = devm_kcalloc(dev, hdma_dev->chan_num, 886 sizeof(*regsets), GFP_KERNEL); 887 if (!regsets) 888 return -ENOMEM; 889 890 regs = hisi_dma_get_ch_regs(hdma_dev, ®s_sz); 891 if (!regs) 892 return -ENOMEM; 893 894 for (i = 0; i < hdma_dev->chan_num; i++) { 895 regsets[i].regs = regs; 896 regsets[i].nregs = regs_sz; 897 regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET; 898 regsets[i].dev = dev; 899 900 memset(dir_name, 0, HISI_DMA_MAX_DIR_NAME_LEN); 901 ret = sprintf(dir_name, "channel%d", i); 902 if (ret < 0) 903 return ret; 904 905 chan_dir = debugfs_create_dir(dir_name, 906 hdma_dev->dma_dev.dbg_dev_root); 907 debugfs_create_regset32("regs", 0444, chan_dir, ®sets[i]); 908 } 909 910 return 0; 911 } 912 913 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) 914 { 915 struct debugfs_regset32 *regset; 916 struct device *dev; 917 int ret; 918 919 dev = &hdma_dev->pdev->dev; 920 921 if (hdma_dev->dma_dev.dbg_dev_root == NULL) 922 return; 923 924 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 925 if (!regset) 926 return; 927 928 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { 929 regset->regs = hisi_dma_hip08_comm_regs; 930 regset->nregs = ARRAY_SIZE(hisi_dma_hip08_comm_regs); 931 } else { 932 regset->regs = hisi_dma_hip09_comm_regs; 933 regset->nregs = ARRAY_SIZE(hisi_dma_hip09_comm_regs); 934 } 935 regset->base = hdma_dev->base; 936 regset->dev = dev; 937 938 debugfs_create_regset32("regs", 0444, 939 hdma_dev->dma_dev.dbg_dev_root, regset); 940 941 ret = hisi_dma_create_chan_dir(hdma_dev); 942 if (ret < 0) 943 dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n"); 944 } 945 #else 946 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { } 947 #endif /* CONFIG_DEBUG_FS*/ 948 /* --- debugfs implementation --- */ 949 950 static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) 951 { 952 enum hisi_dma_reg_layout reg_layout; 953 struct device *dev = &pdev->dev; 954 struct hisi_dma_dev *hdma_dev; 955 struct dma_device *dma_dev; 956 u32 chan_num; 957 u32 msi_num; 958 int ret; 959 960 reg_layout = hisi_dma_get_reg_layout(pdev); 961 if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) { 962 dev_err(dev, "unsupported device!\n"); 963 return -EINVAL; 964 } 965 966 ret = pcim_enable_device(pdev); 967 if (ret) { 968 dev_err(dev, "failed to enable device mem!\n"); 969 return ret; 970 } 971 972 ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev)); 973 if (ret) { 974 dev_err(dev, "failed to remap I/O region!\n"); 975 return ret; 976 } 977 978 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 979 if (ret) 980 return ret; 981 982 chan_num = hisi_dma_get_chan_num(pdev); 983 hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), 984 GFP_KERNEL); 985 if (!hdma_dev) 986 return -EINVAL; 987 988 hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; 989 hdma_dev->pdev = pdev; 990 hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL; 991 hdma_dev->chan_num = chan_num; 992 hdma_dev->reg_layout = reg_layout; 993 hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev); 994 995 pci_set_drvdata(pdev, hdma_dev); 996 pci_set_master(pdev); 997 998 msi_num = hisi_dma_get_msi_num(pdev); 999 1000 /* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */ 1001 ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI); 1002 if (ret < 0) { 1003 dev_err(dev, "Failed to allocate MSI vectors!\n"); 1004 return ret; 1005 } 1006 1007 hisi_dma_init_dma_dev(hdma_dev); 1008 1009 hisi_dma_set_mode(hdma_dev, RC); 1010 1011 hisi_dma_init_hw(hdma_dev); 1012 1013 ret = hisi_dma_enable_hw_channels(hdma_dev); 1014 if (ret < 0) { 1015 dev_err(dev, "failed to enable hw channel!\n"); 1016 return ret; 1017 } 1018 1019 ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels, 1020 hdma_dev); 1021 if (ret) 1022 return ret; 1023 1024 dma_dev = &hdma_dev->dma_dev; 1025 ret = dmaenginem_async_device_register(dma_dev); 1026 if (ret < 0) { 1027 dev_err(dev, "failed to register device!\n"); 1028 return ret; 1029 } 1030 1031 hisi_dma_create_debugfs(hdma_dev); 1032 1033 return 0; 1034 } 1035 1036 static const struct pci_device_id hisi_dma_pci_tbl[] = { 1037 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) }, 1038 { 0, } 1039 }; 1040 1041 static struct pci_driver hisi_dma_pci_driver = { 1042 .name = "hisi_dma", 1043 .id_table = hisi_dma_pci_tbl, 1044 .probe = hisi_dma_probe, 1045 }; 1046 1047 module_pci_driver(hisi_dma_pci_driver); 1048 1049 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 1050 MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>"); 1051 MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver"); 1052 MODULE_LICENSE("GPL v2"); 1053 MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl); 1054