1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2024 Intel Corporation */ 3 4 #include <linux/bitfield.h> 5 #include <linux/delay.h> 6 #include <linux/overflow.h> 7 #include <linux/regmap.h> 8 #include <linux/scatterlist.h> 9 10 #include "intel-thc-dev.h" 11 #include "intel-thc-dma.h" 12 #include "intel-thc-hw.h" 13 14 static void dma_set_prd_base_addr(struct thc_device *dev, u64 physical_addr, 15 struct thc_dma_configuration *dma_config) 16 { 17 u32 addr_high, addr_low; 18 19 if (!dma_config->is_enabled) 20 return; 21 22 addr_high = upper_32_bits(physical_addr); 23 addr_low = lower_32_bits(physical_addr); 24 25 regmap_write(dev->thc_regmap, dma_config->prd_base_addr_high, addr_high); 26 regmap_write(dev->thc_regmap, dma_config->prd_base_addr_low, addr_low); 27 } 28 29 static void dma_set_start_bit(struct thc_device *dev, 30 struct thc_dma_configuration *dma_config) 31 { 32 u32 ctrl, mask, mbits, data, offset; 33 34 if (!dma_config->is_enabled) 35 return; 36 37 switch (dma_config->dma_channel) { 38 case THC_RXDMA1: 39 case THC_RXDMA2: 40 if (dma_config->dma_channel == THC_RXDMA2) { 41 mbits = FIELD_PREP(THC_M_PRT_DEVINT_CFG_1_THC_M_PRT_INTTYP_DATA_VAL, 42 THC_BITMASK_INTERRUPT_TYPE_DATA); 43 mask = THC_M_PRT_DEVINT_CFG_1_THC_M_PRT_INTTYP_DATA_VAL; 44 regmap_write_bits(dev->thc_regmap, 45 THC_M_PRT_DEVINT_CFG_1_OFFSET, mask, mbits); 46 } 47 48 mbits = THC_M_PRT_READ_DMA_CNTRL_IE_EOF | 49 THC_M_PRT_READ_DMA_CNTRL_SOO | 50 THC_M_PRT_READ_DMA_CNTRL_IE_STALL | 51 THC_M_PRT_READ_DMA_CNTRL_IE_ERROR | 52 THC_M_PRT_READ_DMA_CNTRL_START; 53 54 mask = THC_M_PRT_READ_DMA_CNTRL_TPCWP | mbits; 55 mask |= THC_M_PRT_READ_DMA_CNTRL_INT_SW_DMA_EN; 56 ctrl = FIELD_PREP(THC_M_PRT_READ_DMA_CNTRL_TPCWP, THC_POINTER_WRAPAROUND) | mbits; 57 offset = dma_config->dma_channel == THC_RXDMA1 ? 58 THC_M_PRT_READ_DMA_CNTRL_1_OFFSET : THC_M_PRT_READ_DMA_CNTRL_2_OFFSET; 59 regmap_write_bits(dev->thc_regmap, offset, mask, ctrl); 60 break; 61 62 case THC_SWDMA: 63 mbits = THC_M_PRT_READ_DMA_CNTRL_IE_DMACPL | 64 THC_M_PRT_READ_DMA_CNTRL_IE_IOC | 65 THC_M_PRT_READ_DMA_CNTRL_SOO | 66 THC_M_PRT_READ_DMA_CNTRL_START; 67 68 mask = THC_M_PRT_READ_DMA_CNTRL_TPCWP | mbits; 69 ctrl = FIELD_PREP(THC_M_PRT_READ_DMA_CNTRL_TPCWP, THC_POINTER_WRAPAROUND) | mbits; 70 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_SW_OFFSET, 71 mask, ctrl); 72 break; 73 74 case THC_TXDMA: 75 regmap_write_bits(dev->thc_regmap, THC_M_PRT_WRITE_INT_STS_OFFSET, 76 THC_M_PRT_WRITE_INT_STS_THC_WRDMA_CMPL_STATUS, 77 THC_M_PRT_WRITE_INT_STS_THC_WRDMA_CMPL_STATUS); 78 79 /* Select interrupt or polling method upon Write completion */ 80 if (dev->dma_ctx->use_write_interrupts) 81 data = THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_IE_IOC_DMACPL; 82 else 83 data = 0; 84 85 data |= THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_START; 86 mask = THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_IE_IOC_DMACPL | 87 THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_START; 88 regmap_write_bits(dev->thc_regmap, THC_M_PRT_WRITE_DMA_CNTRL_OFFSET, 89 mask, data); 90 break; 91 92 default: 93 break; 94 } 95 } 96 97 static void dma_set_prd_control(struct thc_device *dev, u8 entry_count, u8 cb_depth, 98 struct thc_dma_configuration *dma_config) 99 { 100 u32 ctrl, mask; 101 102 if (!dma_config->is_enabled) 103 return; 104 105 if (dma_config->dma_channel == THC_TXDMA) { 106 mask = THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_PTEC; 107 ctrl = FIELD_PREP(THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_PTEC, entry_count); 108 } else { 109 mask = THC_M_PRT_RPRD_CNTRL_PTEC | THC_M_PRT_RPRD_CNTRL_PCD; 110 ctrl = FIELD_PREP(THC_M_PRT_RPRD_CNTRL_PTEC, entry_count) | 111 FIELD_PREP(THC_M_PRT_RPRD_CNTRL_PCD, cb_depth); 112 } 113 114 regmap_write_bits(dev->thc_regmap, dma_config->prd_cntrl, mask, ctrl); 115 } 116 117 static void dma_clear_prd_control(struct thc_device *dev, 118 struct thc_dma_configuration *dma_config) 119 { 120 u32 mask; 121 122 if (!dma_config->is_enabled) 123 return; 124 125 if (dma_config->dma_channel == THC_TXDMA) 126 mask = THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_PTEC; 127 else 128 mask = THC_M_PRT_RPRD_CNTRL_PTEC | THC_M_PRT_RPRD_CNTRL_PCD; 129 130 regmap_write_bits(dev->thc_regmap, dma_config->prd_cntrl, mask, 0); 131 } 132 133 static u8 dma_get_read_pointer(struct thc_device *dev, 134 struct thc_dma_configuration *dma_config) 135 { 136 u32 ctrl, read_pointer; 137 138 regmap_read(dev->thc_regmap, dma_config->dma_cntrl, &ctrl); 139 read_pointer = FIELD_GET(THC_M_PRT_READ_DMA_CNTRL_TPCRP, ctrl); 140 141 dev_dbg(dev->dev, "THC_M_PRT_READ_DMA_CNTRL 0x%x offset 0x%x TPCRP 0x%x\n", 142 ctrl, dma_config->dma_cntrl, read_pointer); 143 144 return read_pointer; 145 } 146 147 static u8 dma_get_write_pointer(struct thc_device *dev, 148 struct thc_dma_configuration *dma_config) 149 { 150 u32 ctrl, write_pointer; 151 152 regmap_read(dev->thc_regmap, dma_config->dma_cntrl, &ctrl); 153 write_pointer = FIELD_GET(THC_M_PRT_READ_DMA_CNTRL_TPCWP, ctrl); 154 155 dev_dbg(dev->dev, "THC_M_PRT_READ_DMA_CNTRL 0x%x offset 0x%x TPCWP 0x%x\n", 156 ctrl, dma_config->dma_cntrl, write_pointer); 157 158 return write_pointer; 159 } 160 161 static void dma_set_write_pointer(struct thc_device *dev, u8 value, 162 struct thc_dma_configuration *dma_config) 163 { 164 u32 ctrl, mask; 165 166 mask = THC_M_PRT_READ_DMA_CNTRL_TPCWP; 167 ctrl = FIELD_PREP(THC_M_PRT_READ_DMA_CNTRL_TPCWP, value); 168 regmap_write_bits(dev->thc_regmap, dma_config->dma_cntrl, mask, ctrl); 169 } 170 171 static size_t dma_get_max_packet_size(struct thc_device *dev, 172 struct thc_dma_configuration *dma_config) 173 { 174 return dma_config->max_packet_size; 175 } 176 177 static void dma_set_max_packet_size(struct thc_device *dev, size_t size, 178 struct thc_dma_configuration *dma_config) 179 { 180 if (size) { 181 dma_config->max_packet_size = ALIGN(size, SZ_4K); 182 dma_config->is_enabled = true; 183 } 184 } 185 186 static void thc_copy_one_sgl_to_prd(struct thc_device *dev, 187 struct thc_dma_configuration *config, 188 unsigned int ind) 189 { 190 struct thc_prd_table *prd_tbl; 191 struct scatterlist *sg; 192 int j; 193 194 prd_tbl = &config->prd_tbls[ind]; 195 196 for_each_sg(config->sgls[ind], sg, config->sgls_nent[ind], j) { 197 prd_tbl->entries[j].dest_addr = 198 sg_dma_address(sg) >> THC_ADDRESS_SHIFT; 199 prd_tbl->entries[j].len = sg_dma_len(sg); 200 prd_tbl->entries[j].hw_status = 0; 201 prd_tbl->entries[j].end_of_prd = 0; 202 } 203 204 /* Set the end_of_prd flag in the last filled entry */ 205 if (j > 0) 206 prd_tbl->entries[j - 1].end_of_prd = 1; 207 } 208 209 static void thc_copy_sgls_to_prd(struct thc_device *dev, 210 struct thc_dma_configuration *config) 211 { 212 unsigned int i; 213 214 memset(config->prd_tbls, 0, array_size(PRD_TABLE_SIZE, config->prd_tbl_num)); 215 216 for (i = 0; i < config->prd_tbl_num; i++) 217 thc_copy_one_sgl_to_prd(dev, config, i); 218 } 219 220 static int setup_dma_buffers(struct thc_device *dev, 221 struct thc_dma_configuration *config, 222 enum dma_data_direction dir) 223 { 224 size_t prd_tbls_size = array_size(PRD_TABLE_SIZE, config->prd_tbl_num); 225 unsigned int i, nent = PRD_ENTRIES_NUM; 226 dma_addr_t dma_handle; 227 void *cpu_addr; 228 size_t buf_sz; 229 int count; 230 231 if (!config->is_enabled) 232 return 0; 233 234 memset(config->sgls, 0, sizeof(config->sgls)); 235 memset(config->sgls_nent_pages, 0, sizeof(config->sgls_nent_pages)); 236 memset(config->sgls_nent, 0, sizeof(config->sgls_nent)); 237 238 cpu_addr = dma_alloc_coherent(dev->dev, prd_tbls_size, 239 &dma_handle, GFP_KERNEL); 240 if (!cpu_addr) 241 return -ENOMEM; 242 243 config->prd_tbls = cpu_addr; 244 config->prd_tbls_dma_handle = dma_handle; 245 246 buf_sz = dma_get_max_packet_size(dev, config); 247 248 /* Allocate and map the scatter-gather lists, one for each PRD table */ 249 for (i = 0; i < config->prd_tbl_num; i++) { 250 config->sgls[i] = sgl_alloc(buf_sz, GFP_KERNEL, &nent); 251 if (!config->sgls[i] || nent > PRD_ENTRIES_NUM) { 252 dev_err_once(dev->dev, "sgl_alloc (%uth) failed, nent %u\n", 253 i, nent); 254 return -ENOMEM; 255 } 256 count = dma_map_sg(dev->dev, config->sgls[i], nent, dir); 257 258 config->sgls_nent_pages[i] = nent; 259 config->sgls_nent[i] = count; 260 } 261 262 thc_copy_sgls_to_prd(dev, config); 263 264 return 0; 265 } 266 267 static void thc_reset_dma_settings(struct thc_device *dev) 268 { 269 /* Stop all DMA channels and reset DMA read pointers */ 270 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_1_OFFSET, 271 THC_M_PRT_READ_DMA_CNTRL_START, 0); 272 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_2_OFFSET, 273 THC_M_PRT_READ_DMA_CNTRL_START, 0); 274 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_SW_OFFSET, 275 THC_M_PRT_READ_DMA_CNTRL_START, 0); 276 regmap_write_bits(dev->thc_regmap, THC_M_PRT_WRITE_DMA_CNTRL_OFFSET, 277 THC_M_PRT_WRITE_DMA_CNTRL_THC_WRDMA_START, 0); 278 279 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_1_OFFSET, 280 THC_M_PRT_READ_DMA_CNTRL_TPCPR, 281 THC_M_PRT_READ_DMA_CNTRL_TPCPR); 282 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_2_OFFSET, 283 THC_M_PRT_READ_DMA_CNTRL_TPCPR, 284 THC_M_PRT_READ_DMA_CNTRL_TPCPR); 285 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_SW_OFFSET, 286 THC_M_PRT_READ_DMA_CNTRL_TPCPR, 287 THC_M_PRT_READ_DMA_CNTRL_TPCPR); 288 } 289 290 static void release_dma_buffers(struct thc_device *dev, 291 struct thc_dma_configuration *config) 292 { 293 size_t prd_tbls_size = array_size(PRD_TABLE_SIZE, config->prd_tbl_num); 294 unsigned int i; 295 296 if (!config->is_enabled) 297 return; 298 299 for (i = 0; i < config->prd_tbl_num; i++) { 300 if (!config->sgls[i] || !config->sgls_nent[i]) 301 continue; 302 303 dma_unmap_sg(dev->dev, config->sgls[i], 304 config->sgls_nent_pages[i], 305 config->dir); 306 307 sgl_free(config->sgls[i]); 308 config->sgls[i] = NULL; 309 } 310 311 memset(config->prd_tbls, 0, prd_tbls_size); 312 313 if (config->prd_tbls) { 314 dma_free_coherent(dev->dev, prd_tbls_size, config->prd_tbls, 315 config->prd_tbls_dma_handle); 316 config->prd_tbls = NULL; 317 config->prd_tbls_dma_handle = 0; 318 } 319 } 320 321 struct thc_dma_context *thc_dma_init(struct thc_device *dev) 322 { 323 struct thc_dma_context *dma_ctx; 324 325 dma_ctx = devm_kzalloc(dev->dev, sizeof(*dma_ctx), GFP_KERNEL); 326 if (!dma_ctx) 327 return NULL; 328 329 dev->dma_ctx = dma_ctx; 330 331 dma_ctx->dma_config[THC_RXDMA1].dma_channel = THC_RXDMA1; 332 dma_ctx->dma_config[THC_RXDMA2].dma_channel = THC_RXDMA2; 333 dma_ctx->dma_config[THC_TXDMA].dma_channel = THC_TXDMA; 334 dma_ctx->dma_config[THC_SWDMA].dma_channel = THC_SWDMA; 335 336 dma_ctx->dma_config[THC_RXDMA1].dir = DMA_FROM_DEVICE; 337 dma_ctx->dma_config[THC_RXDMA2].dir = DMA_FROM_DEVICE; 338 dma_ctx->dma_config[THC_TXDMA].dir = DMA_TO_DEVICE; 339 dma_ctx->dma_config[THC_SWDMA].dir = DMA_FROM_DEVICE; 340 341 dma_ctx->dma_config[THC_RXDMA1].prd_tbl_num = PRD_TABLES_NUM; 342 dma_ctx->dma_config[THC_RXDMA2].prd_tbl_num = PRD_TABLES_NUM; 343 dma_ctx->dma_config[THC_TXDMA].prd_tbl_num = 1; 344 dma_ctx->dma_config[THC_SWDMA].prd_tbl_num = 1; 345 346 dma_ctx->dma_config[THC_RXDMA1].prd_base_addr_high = THC_M_PRT_RPRD_BA_HI_1_OFFSET; 347 dma_ctx->dma_config[THC_RXDMA2].prd_base_addr_high = THC_M_PRT_RPRD_BA_HI_2_OFFSET; 348 dma_ctx->dma_config[THC_TXDMA].prd_base_addr_high = THC_M_PRT_WPRD_BA_HI_OFFSET; 349 dma_ctx->dma_config[THC_SWDMA].prd_base_addr_high = THC_M_PRT_RPRD_BA_HI_SW_OFFSET; 350 351 dma_ctx->dma_config[THC_RXDMA1].prd_base_addr_low = THC_M_PRT_RPRD_BA_LOW_1_OFFSET; 352 dma_ctx->dma_config[THC_RXDMA2].prd_base_addr_low = THC_M_PRT_RPRD_BA_LOW_2_OFFSET; 353 dma_ctx->dma_config[THC_TXDMA].prd_base_addr_low = THC_M_PRT_WPRD_BA_LOW_OFFSET; 354 dma_ctx->dma_config[THC_SWDMA].prd_base_addr_low = THC_M_PRT_RPRD_BA_LOW_SW_OFFSET; 355 356 dma_ctx->dma_config[THC_RXDMA1].prd_cntrl = THC_M_PRT_RPRD_CNTRL_1_OFFSET; 357 dma_ctx->dma_config[THC_RXDMA2].prd_cntrl = THC_M_PRT_RPRD_CNTRL_2_OFFSET; 358 dma_ctx->dma_config[THC_TXDMA].prd_cntrl = THC_M_PRT_WRITE_DMA_CNTRL_OFFSET; 359 dma_ctx->dma_config[THC_SWDMA].prd_cntrl = THC_M_PRT_RPRD_CNTRL_SW_OFFSET; 360 361 dma_ctx->dma_config[THC_RXDMA1].dma_cntrl = THC_M_PRT_READ_DMA_CNTRL_1_OFFSET; 362 dma_ctx->dma_config[THC_RXDMA2].dma_cntrl = THC_M_PRT_READ_DMA_CNTRL_2_OFFSET; 363 dma_ctx->dma_config[THC_TXDMA].dma_cntrl = THC_M_PRT_WRITE_DMA_CNTRL_OFFSET; 364 dma_ctx->dma_config[THC_SWDMA].dma_cntrl = THC_M_PRT_READ_DMA_CNTRL_SW_OFFSET; 365 366 /* Enable write DMA completion interrupt by default */ 367 dma_ctx->use_write_interrupts = 1; 368 369 return dma_ctx; 370 } 371 372 /** 373 * thc_dma_set_max_packet_sizes - Set max packet sizes for all DMA engines 374 * 375 * @dev: The pointer of THC private device context 376 * @mps_read1: RxDMA1 max packet size 377 * @mps_read2: RxDMA2 max packet size 378 * @mps_write: TxDMA max packet size 379 * @mps_swdma: Software DMA max packet size 380 * 381 * If mps is not 0, it means the corresponding DMA channel is used, then set 382 * the flag to turn on this channel. 383 * 384 * Return: 0 on success, other error codes on failed. 385 */ 386 int thc_dma_set_max_packet_sizes(struct thc_device *dev, size_t mps_read1, 387 size_t mps_read2, size_t mps_write, 388 size_t mps_swdma) 389 { 390 if (!dev->dma_ctx) { 391 dev_err_once(dev->dev, 392 "Cannot set max packet sizes because DMA context is NULL!\n"); 393 return -EINVAL; 394 } 395 396 dma_set_max_packet_size(dev, mps_read1, &dev->dma_ctx->dma_config[THC_RXDMA1]); 397 dma_set_max_packet_size(dev, mps_read2, &dev->dma_ctx->dma_config[THC_RXDMA2]); 398 dma_set_max_packet_size(dev, mps_write, &dev->dma_ctx->dma_config[THC_TXDMA]); 399 dma_set_max_packet_size(dev, mps_swdma, &dev->dma_ctx->dma_config[THC_SWDMA]); 400 401 return 0; 402 } 403 EXPORT_SYMBOL_NS_GPL(thc_dma_set_max_packet_sizes, "INTEL_THC"); 404 405 /** 406 * thc_dma_allocate - Allocate DMA buffers for all DMA engines 407 * 408 * @dev: The pointer of THC private device context 409 * 410 * Return: 0 on success, other error codes on failed. 411 */ 412 int thc_dma_allocate(struct thc_device *dev) 413 { 414 int ret, chan; 415 416 for (chan = 0; chan < MAX_THC_DMA_CHANNEL; chan++) { 417 ret = setup_dma_buffers(dev, &dev->dma_ctx->dma_config[chan], 418 dev->dma_ctx->dma_config[chan].dir); 419 if (ret < 0) { 420 dev_err_once(dev->dev, "DMA setup failed for DMA channel %d\n", chan); 421 goto release_bufs; 422 } 423 } 424 425 return 0; 426 427 release_bufs: 428 while (chan--) 429 release_dma_buffers(dev, &dev->dma_ctx->dma_config[chan]); 430 431 return ret; 432 } 433 EXPORT_SYMBOL_NS_GPL(thc_dma_allocate, "INTEL_THC"); 434 435 /** 436 * thc_dma_release - Release DMA buffers for all DMA engines 437 * 438 * @dev: The pointer of THC private device context 439 */ 440 void thc_dma_release(struct thc_device *dev) 441 { 442 int chan; 443 444 for (chan = 0; chan < MAX_THC_DMA_CHANNEL; chan++) 445 release_dma_buffers(dev, &dev->dma_ctx->dma_config[chan]); 446 } 447 EXPORT_SYMBOL_NS_GPL(thc_dma_release, "INTEL_THC"); 448 449 static int calc_prd_entries_num(struct thc_prd_table *prd_tbl, 450 size_t mes_len, u8 *nent) 451 { 452 *nent = DIV_ROUND_UP(mes_len, THC_MIN_BYTES_PER_SG_LIST_ENTRY); 453 if (*nent > PRD_ENTRIES_NUM) 454 return -EMSGSIZE; 455 456 return 0; 457 } 458 459 static size_t calc_message_len(struct thc_prd_table *prd_tbl, u8 *nent) 460 { 461 size_t mes_len = 0; 462 unsigned int j; 463 464 for (j = 0; j < PRD_ENTRIES_NUM; j++) { 465 mes_len += prd_tbl->entries[j].len; 466 if (prd_tbl->entries[j].end_of_prd) 467 break; 468 } 469 470 *nent = j + 1; 471 472 return mes_len; 473 } 474 475 /** 476 * thc_dma_configure - Configure DMA settings for all DMA engines 477 * 478 * @dev: The pointer of THC private device context 479 * 480 * Return: 0 on success, other error codes on failed. 481 */ 482 int thc_dma_configure(struct thc_device *dev) 483 { 484 struct thc_dma_context *dma_ctx = dev->dma_ctx; 485 int chan; 486 487 thc_reset_dma_settings(dev); 488 489 if (!dma_ctx) { 490 dev_err_once(dev->dev, "Cannot do DMA configure because DMA context is NULL\n"); 491 return -EINVAL; 492 } 493 494 for (chan = 0; chan < MAX_THC_DMA_CHANNEL; chan++) { 495 dma_set_prd_base_addr(dev, 496 dma_ctx->dma_config[chan].prd_tbls_dma_handle, 497 &dma_ctx->dma_config[chan]); 498 499 dma_set_prd_control(dev, PRD_ENTRIES_NUM - 1, 500 dma_ctx->dma_config[chan].prd_tbl_num - 1, 501 &dma_ctx->dma_config[chan]); 502 } 503 504 /* Start read2 DMA engine */ 505 dma_set_start_bit(dev, &dma_ctx->dma_config[THC_RXDMA2]); 506 507 dev_dbg(dev->dev, "DMA configured successfully!\n"); 508 509 return 0; 510 } 511 EXPORT_SYMBOL_NS_GPL(thc_dma_configure, "INTEL_THC"); 512 513 /** 514 * thc_dma_unconfigure - Unconfigure DMA settings for all DMA engines 515 * 516 * @dev: The pointer of THC private device context 517 */ 518 void thc_dma_unconfigure(struct thc_device *dev) 519 { 520 int chan; 521 522 for (chan = 0; chan < MAX_THC_DMA_CHANNEL; chan++) { 523 dma_set_prd_base_addr(dev, 0, &dev->dma_ctx->dma_config[chan]); 524 dma_clear_prd_control(dev, &dev->dma_ctx->dma_config[chan]); 525 } 526 527 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_1_OFFSET, 528 THC_M_PRT_READ_DMA_CNTRL_START, 0); 529 530 regmap_write_bits(dev->thc_regmap, THC_M_PRT_READ_DMA_CNTRL_2_OFFSET, 531 THC_M_PRT_READ_DMA_CNTRL_START, 0); 532 } 533 EXPORT_SYMBOL_NS_GPL(thc_dma_unconfigure, "INTEL_THC"); 534 535 static int thc_wait_for_dma_pause(struct thc_device *dev, enum thc_dma_channel channel) 536 { 537 u32 ctrl_reg, sts_reg, sts; 538 int ret; 539 540 ctrl_reg = (channel == THC_RXDMA1) ? THC_M_PRT_READ_DMA_CNTRL_1_OFFSET : 541 ((channel == THC_RXDMA2) ? THC_M_PRT_READ_DMA_CNTRL_2_OFFSET : 542 THC_M_PRT_READ_DMA_CNTRL_SW_OFFSET); 543 544 regmap_write_bits(dev->thc_regmap, ctrl_reg, THC_M_PRT_READ_DMA_CNTRL_START, 0); 545 546 sts_reg = (channel == THC_RXDMA1) ? THC_M_PRT_READ_DMA_INT_STS_1_OFFSET : 547 ((channel == THC_RXDMA2) ? THC_M_PRT_READ_DMA_INT_STS_2_OFFSET : 548 THC_M_PRT_READ_DMA_INT_STS_SW_OFFSET); 549 550 ret = regmap_read_poll_timeout(dev->thc_regmap, sts_reg, sts, 551 !(sts & THC_M_PRT_READ_DMA_INT_STS_ACTIVE), 552 THC_DEFAULT_RXDMA_POLLING_US_INTERVAL, 553 THC_DEFAULT_RXDMA_POLLING_US_TIMEOUT); 554 555 if (ret) { 556 dev_err_once(dev->dev, 557 "Timeout while waiting for DMA %d stop\n", channel); 558 return ret; 559 } 560 561 return 0; 562 } 563 564 /** 565 * thc_rxdma_reset - Reset all read DMA engines 566 * 567 * @dev: The pointer of THC private device context 568 * 569 * This is a helper function to reset RxDMA configure. It's typically used 570 * for RxDMA recovery when fatal error happens. 571 * 572 * Return: 0 if successful or error code on failure. 573 */ 574 int thc_rxdma_reset(struct thc_device *dev) 575 { 576 int ret; 577 578 if (mutex_lock_interruptible(&dev->thc_bus_lock)) 579 return -EINTR; 580 581 ret = thc_interrupt_quiesce(dev, true); 582 if (ret) { 583 dev_err(dev->dev, "Quiesce interrupt failed during RxDMA reset\n"); 584 goto end; 585 } 586 587 ret = thc_wait_for_dma_pause(dev, THC_RXDMA1); 588 if (ret) { 589 dev_err(dev->dev, "Wait for RxDMA1 pause failed during RxDMA reset\n"); 590 goto end; 591 } 592 593 ret = thc_wait_for_dma_pause(dev, THC_RXDMA2); 594 if (ret) { 595 dev_err(dev->dev, "Wait for RxDMA2 pause failed during RxDMA reset\n"); 596 goto end; 597 } 598 599 thc_dma_unconfigure(dev); 600 601 ret = thc_dma_configure(dev); 602 if (ret) { 603 dev_err(dev->dev, "Re-config DMA failed during RxDMA reset\n"); 604 goto end; 605 } 606 607 thc_interrupt_quiesce(dev, false); 608 609 end: 610 mutex_unlock(&dev->thc_bus_lock); 611 return ret; 612 } 613 EXPORT_SYMBOL_NS_GPL(thc_rxdma_reset, "INTEL_THC"); 614 615 static int read_dma_buffer(struct thc_device *dev, 616 struct thc_dma_configuration *read_config, 617 u8 prd_table_index, void *read_buff) 618 { 619 struct thc_prd_table *prd_tbl; 620 struct scatterlist *sg; 621 size_t mes_len, ret; 622 u8 nent; 623 624 if (prd_table_index >= read_config->prd_tbl_num) { 625 dev_err_once(dev->dev, "PRD table index %d too big\n", prd_table_index); 626 return -EINVAL; 627 } 628 629 if (!read_config->prd_tbls || !read_config->sgls[prd_table_index]) { 630 dev_err_once(dev->dev, "PRD tables are not ready yet\n"); 631 return -EINVAL; 632 } 633 634 prd_tbl = &read_config->prd_tbls[prd_table_index]; 635 mes_len = calc_message_len(prd_tbl, &nent); 636 if (mes_len > read_config->max_packet_size) { 637 dev_err(dev->dev, 638 "Message length %zu is bigger than buffer length %lu\n", 639 mes_len, read_config->max_packet_size); 640 return -EMSGSIZE; 641 } 642 643 sg = read_config->sgls[prd_table_index]; 644 ret = sg_copy_to_buffer(sg, nent, read_buff, mes_len); 645 if (ret != mes_len) { 646 dev_err_once(dev->dev, "Copied %zu bytes instead of requested %zu\n", 647 ret, mes_len); 648 return -EIO; 649 } 650 651 return mes_len; 652 } 653 654 static void update_write_pointer(struct thc_device *dev, 655 struct thc_dma_configuration *read_config) 656 { 657 u8 write_ptr = dma_get_write_pointer(dev, read_config); 658 659 if (write_ptr + 1 == THC_WRAPAROUND_VALUE_ODD) 660 dma_set_write_pointer(dev, THC_POINTER_WRAPAROUND, read_config); 661 else if (write_ptr + 1 == THC_WRAPAROUND_VALUE_EVEN) 662 dma_set_write_pointer(dev, 0, read_config); 663 else 664 dma_set_write_pointer(dev, write_ptr + 1, read_config); 665 } 666 667 static int is_dma_buf_empty(struct thc_device *dev, 668 struct thc_dma_configuration *read_config, 669 u8 *read_ptr, u8 *write_ptr) 670 { 671 *read_ptr = dma_get_read_pointer(dev, read_config); 672 *write_ptr = dma_get_write_pointer(dev, read_config); 673 674 if ((*read_ptr & THC_POINTER_MASK) == (*write_ptr & THC_POINTER_MASK)) 675 if (*read_ptr != *write_ptr) 676 return true; 677 678 return false; 679 } 680 681 static int thc_dma_read(struct thc_device *dev, 682 struct thc_dma_configuration *read_config, 683 void *read_buff, size_t *read_len, int *read_finished) 684 { 685 u8 read_ptr, write_ptr, prd_table_index; 686 int status; 687 688 if (!is_dma_buf_empty(dev, read_config, &read_ptr, &write_ptr)) { 689 prd_table_index = write_ptr & THC_POINTER_MASK; 690 691 status = read_dma_buffer(dev, read_config, prd_table_index, read_buff); 692 if (status <= 0) { 693 dev_err_once(dev->dev, "read DMA buffer failed %d\n", status); 694 return -EIO; 695 } 696 697 *read_len = status; 698 699 /* Clear the relevant PRD table */ 700 thc_copy_one_sgl_to_prd(dev, read_config, prd_table_index); 701 702 /* Increment the write pointer to let the HW know we have processed this PRD */ 703 update_write_pointer(dev, read_config); 704 } 705 706 /* 707 * This function only reads one frame from PRD table for each call, so we need to 708 * check if all DMAed data is read out and return the flag to the caller. Caller 709 * should repeatedly call thc_dma_read() until all DMAed data is handled. 710 */ 711 if (read_finished) 712 *read_finished = is_dma_buf_empty(dev, read_config, &read_ptr, &write_ptr) ? 1 : 0; 713 714 return 0; 715 } 716 717 /** 718 * thc_rxdma_read - Read data from RXDMA buffer 719 * 720 * @dev: The pointer of THC private device context 721 * @dma_channel: The RXDMA engine of read data source 722 * @read_buff: The pointer of the read data buffer 723 * @read_len: The pointer of the read data length 724 * @read_finished: The pointer of the flag indicating if all pending data has been read out 725 * 726 * Return: 0 on success, other error codes on failed. 727 */ 728 int thc_rxdma_read(struct thc_device *dev, enum thc_dma_channel dma_channel, 729 void *read_buff, size_t *read_len, int *read_finished) 730 { 731 struct thc_dma_configuration *dma_config; 732 int ret; 733 734 dma_config = &dev->dma_ctx->dma_config[dma_channel]; 735 736 if (!dma_config->is_enabled) { 737 dev_err_once(dev->dev, "The DMA channel %d is not enabled", dma_channel); 738 return -EINVAL; 739 } 740 741 if (!read_buff || !read_len) { 742 dev_err(dev->dev, "Invalid input parameters, read_buff %p, read_len %p\n", 743 read_buff, read_len); 744 return -EINVAL; 745 } 746 747 if (dma_channel >= THC_TXDMA) { 748 dev_err(dev->dev, "Unsupported DMA channel for RxDMA read, %d\n", dma_channel); 749 return -EINVAL; 750 } 751 752 ret = thc_dma_read(dev, dma_config, read_buff, read_len, read_finished); 753 754 return ret; 755 } 756 EXPORT_SYMBOL_NS_GPL(thc_rxdma_read, "INTEL_THC"); 757 758 static int thc_swdma_read_start(struct thc_device *dev, void *write_buff, 759 size_t write_len, u32 *prd_tbl_len) 760 { 761 u32 mask, val, data0 = 0, data1 = 0; 762 int ret; 763 764 ret = thc_interrupt_quiesce(dev, true); 765 if (ret) 766 return ret; 767 768 if (thc_wait_for_dma_pause(dev, THC_RXDMA1) || thc_wait_for_dma_pause(dev, THC_RXDMA2)) 769 return -EIO; 770 771 thc_reset_dma_settings(dev); 772 773 /* 774 * Max input size control feature is only available for RxDMA, it must keep disabled 775 * during SWDMA operation, and restore to previous state after SWDMA is done. 776 * Max input size variables in THC device context track hardware state, and keep change 777 * when feature state was changed, so those variables cannot be used to record feature 778 * state after state was changed during SWDMA operation. Here have to use a temp variable 779 * in DMA context to record feature state before SWDMA operation. 780 */ 781 if (dev->i2c_max_rx_size_en) { 782 thc_i2c_rx_max_size_enable(dev, false); 783 dev->dma_ctx->rx_max_size_en = true; 784 } 785 786 /* 787 * Interrupt delay feature is in the same situation with max input size control feature, 788 * needs record feature state before SWDMA. 789 */ 790 if (dev->i2c_int_delay_en) { 791 thc_i2c_rx_int_delay_enable(dev, false); 792 dev->dma_ctx->rx_int_delay_en = true; 793 } 794 795 mask = THC_M_PRT_RPRD_CNTRL_SW_THC_SWDMA_I2C_WBC | 796 THC_M_PRT_RPRD_CNTRL_SW_THC_SWDMA_I2C_RX_DLEN_EN; 797 val = FIELD_PREP(THC_M_PRT_RPRD_CNTRL_SW_THC_SWDMA_I2C_WBC, write_len) | 798 ((!prd_tbl_len) ? THC_M_PRT_RPRD_CNTRL_SW_THC_SWDMA_I2C_RX_DLEN_EN : 0); 799 regmap_write_bits(dev->thc_regmap, THC_M_PRT_RPRD_CNTRL_SW_OFFSET, 800 mask, val); 801 802 if (prd_tbl_len) { 803 mask = THC_M_PRT_SW_DMA_PRD_TABLE_LEN_THC_M_PRT_SW_DMA_PRD_TABLE_LEN; 804 val = FIELD_PREP(THC_M_PRT_SW_DMA_PRD_TABLE_LEN_THC_M_PRT_SW_DMA_PRD_TABLE_LEN, 805 *prd_tbl_len); 806 regmap_write_bits(dev->thc_regmap, THC_M_PRT_SW_DMA_PRD_TABLE_LEN_OFFSET, 807 mask, val); 808 } 809 810 if (write_len <= sizeof(u32)) { 811 for (int i = 0; i < write_len; i++) 812 data0 |= *(((u8 *)write_buff) + i) << (i * 8); 813 814 regmap_write(dev->thc_regmap, THC_M_PRT_SW_SEQ_DATA0_ADDR_OFFSET, data0); 815 } else if (write_len <= 2 * sizeof(u32)) { 816 data0 = *(u32 *)write_buff; 817 regmap_write(dev->thc_regmap, THC_M_PRT_SW_SEQ_DATA0_ADDR_OFFSET, data0); 818 819 for (int i = 0; i < write_len - sizeof(u32); i++) 820 data1 |= *(((u8 *)write_buff) + sizeof(u32) + i) << (i * 8); 821 822 regmap_write(dev->thc_regmap, THC_M_PRT_SW_SEQ_DATA1_OFFSET, data1); 823 } 824 dma_set_start_bit(dev, &dev->dma_ctx->dma_config[THC_SWDMA]); 825 826 return 0; 827 } 828 829 static int thc_swdma_read_completion(struct thc_device *dev) 830 { 831 int ret; 832 833 ret = thc_wait_for_dma_pause(dev, THC_SWDMA); 834 if (ret) 835 return ret; 836 837 /* 838 * Restore max input size control feature to previous state after SWDMA if it was 839 * enabled before SWDMA, and reset temp rx_max_size_en variable for next time. 840 */ 841 if (dev->dma_ctx->rx_max_size_en) { 842 thc_i2c_rx_max_size_enable(dev, true); 843 dev->dma_ctx->rx_max_size_en = false; 844 } 845 846 /* 847 * Restore input interrupt delay feature to previous state after SWDMA if it was 848 * enabled before SWDMA, and reset temp rx_int_delay_en variable for next time. 849 */ 850 if (dev->dma_ctx->rx_int_delay_en) { 851 thc_i2c_rx_int_delay_enable(dev, true); 852 dev->dma_ctx->rx_int_delay_en = false; 853 } 854 855 thc_reset_dma_settings(dev); 856 857 dma_set_start_bit(dev, &dev->dma_ctx->dma_config[THC_RXDMA2]); 858 859 ret = thc_interrupt_quiesce(dev, false); 860 861 return ret; 862 } 863 864 /** 865 * thc_swdma_read - Use software DMA to read data from touch device 866 * 867 * @dev: The pointer of THC private device context 868 * @write_buff: The pointer of write buffer for SWDMA sequence 869 * @write_len: The write data length for SWDMA sequence 870 * @prd_tbl_len: The prd table length of SWDMA engine, can be set to NULL 871 * @read_buff: The pointer of the read data buffer 872 * @read_len: The pointer of the read data length 873 * 874 * Return: 0 on success, other error codes on failed. 875 */ 876 int thc_swdma_read(struct thc_device *dev, void *write_buff, size_t write_len, 877 u32 *prd_tbl_len, void *read_buff, size_t *read_len) 878 { 879 int ret; 880 881 if (!(&dev->dma_ctx->dma_config[THC_SWDMA])->is_enabled) { 882 dev_err_once(dev->dev, "The SWDMA channel is not enabled"); 883 return -EINVAL; 884 } 885 886 if (!read_buff || !read_len) { 887 dev_err(dev->dev, "Invalid input parameters, read_buff %p, read_len %p\n", 888 read_buff, read_len); 889 return -EINVAL; 890 } 891 892 if (mutex_lock_interruptible(&dev->thc_bus_lock)) 893 return -EINTR; 894 895 dev->swdma_done = false; 896 897 ret = thc_swdma_read_start(dev, write_buff, write_len, prd_tbl_len); 898 if (ret) 899 goto end; 900 901 ret = wait_event_interruptible_timeout(dev->swdma_complete_wait, dev->swdma_done, 1 * HZ); 902 if (ret <= 0 || !dev->swdma_done) { 903 dev_err_once(dev->dev, "timeout for waiting SWDMA completion\n"); 904 ret = -ETIMEDOUT; 905 goto end; 906 } 907 908 ret = thc_dma_read(dev, &dev->dma_ctx->dma_config[THC_SWDMA], read_buff, read_len, NULL); 909 if (ret) 910 goto end; 911 912 ret = thc_swdma_read_completion(dev); 913 914 end: 915 mutex_unlock(&dev->thc_bus_lock); 916 return ret; 917 } 918 EXPORT_SYMBOL_NS_GPL(thc_swdma_read, "INTEL_THC"); 919 920 static int write_dma_buffer(struct thc_device *dev, 921 void *buffer, size_t buf_len) 922 { 923 struct thc_dma_configuration *write_config = &dev->dma_ctx->dma_config[THC_TXDMA]; 924 struct thc_prd_table *prd_tbl; 925 struct scatterlist *sg; 926 unsigned long len_left; 927 size_t ret; 928 u8 nent; 929 int i; 930 931 /* There is only one PRD table for write */ 932 prd_tbl = &write_config->prd_tbls[0]; 933 934 if (calc_prd_entries_num(prd_tbl, buf_len, &nent) < 0) { 935 dev_err(dev->dev, "Tx message length too big (%zu)\n", buf_len); 936 return -EOVERFLOW; 937 } 938 939 sg = write_config->sgls[0]; 940 ret = sg_copy_from_buffer(sg, nent, buffer, buf_len); 941 if (ret != buf_len) { 942 dev_err_once(dev->dev, "Copied %zu bytes instead of requested %zu\n", 943 ret, buf_len); 944 return -EIO; 945 } 946 947 prd_tbl = &write_config->prd_tbls[0]; 948 len_left = buf_len; 949 950 for_each_sg(write_config->sgls[0], sg, write_config->sgls_nent[0], i) { 951 if (sg_dma_address(sg) == 0 || sg_dma_len(sg) == 0) { 952 dev_err_once(dev->dev, "SGList: zero address or length\n"); 953 return -EINVAL; 954 } 955 956 prd_tbl->entries[i].dest_addr = 957 sg_dma_address(sg) >> THC_ADDRESS_SHIFT; 958 959 if (len_left < sg_dma_len(sg)) { 960 prd_tbl->entries[i].len = len_left; 961 prd_tbl->entries[i].end_of_prd = 1; 962 break; 963 } 964 965 prd_tbl->entries[i].len = sg_dma_len(sg); 966 prd_tbl->entries[i].end_of_prd = 0; 967 968 len_left -= sg_dma_len(sg); 969 } 970 971 dma_set_prd_control(dev, i, 0, write_config); 972 973 return 0; 974 } 975 976 static void thc_ensure_performance_limitations(struct thc_device *dev) 977 { 978 unsigned long delay_usec = 0; 979 /* 980 * Minimum amount of delay the THC / QUICKSPI driver must wait 981 * between end of write operation and begin of read operation. 982 * This value shall be in 10us multiples. 983 */ 984 if (dev->perf_limit > 0) { 985 delay_usec = dev->perf_limit * 10; 986 udelay(delay_usec); 987 } 988 } 989 990 static void thc_dma_write_completion(struct thc_device *dev) 991 { 992 thc_ensure_performance_limitations(dev); 993 } 994 995 /** 996 * thc_dma_write - Use TXDMA to write data to touch device 997 * 998 * @dev: The pointer of THC private device context 999 * @buffer: The pointer of write data buffer 1000 * @buf_len: The write data length 1001 * 1002 * Return: 0 on success, other error codes on failed. 1003 */ 1004 int thc_dma_write(struct thc_device *dev, void *buffer, size_t buf_len) 1005 { 1006 bool restore_interrupts = false; 1007 u32 sts, ctrl; 1008 int ret; 1009 1010 if (!(&dev->dma_ctx->dma_config[THC_TXDMA])->is_enabled) { 1011 dev_err_once(dev->dev, "The TxDMA channel is not enabled\n"); 1012 return -EINVAL; 1013 } 1014 1015 if (!buffer || buf_len <= 0) { 1016 dev_err(dev->dev, "Invalid input parameters, buffer %p\n, buf_len %zu\n", 1017 buffer, buf_len); 1018 return -EINVAL; 1019 } 1020 1021 regmap_read(dev->thc_regmap, THC_M_PRT_WRITE_INT_STS_OFFSET, &sts); 1022 if (sts & THC_M_PRT_WRITE_INT_STS_THC_WRDMA_ACTIVE) { 1023 dev_err_once(dev->dev, "THC TxDMA is till active and can't start again\n"); 1024 return -EBUSY; 1025 } 1026 1027 if (mutex_lock_interruptible(&dev->thc_bus_lock)) 1028 return -EINTR; 1029 1030 regmap_read(dev->thc_regmap, THC_M_PRT_CONTROL_OFFSET, &ctrl); 1031 1032 ret = write_dma_buffer(dev, buffer, buf_len); 1033 if (ret) 1034 goto end; 1035 1036 if (dev->perf_limit && !(ctrl & THC_M_PRT_CONTROL_THC_DEVINT_QUIESCE_HW_STS)) { 1037 ret = thc_interrupt_quiesce(dev, true); 1038 if (ret) 1039 goto end; 1040 1041 restore_interrupts = true; 1042 } 1043 1044 dev->write_done = false; 1045 1046 dma_set_start_bit(dev, &dev->dma_ctx->dma_config[THC_TXDMA]); 1047 1048 ret = wait_event_interruptible_timeout(dev->write_complete_wait, dev->write_done, 1 * HZ); 1049 if (ret <= 0 || !dev->write_done) { 1050 dev_err_once(dev->dev, "timeout for waiting TxDMA completion\n"); 1051 ret = -ETIMEDOUT; 1052 goto end; 1053 } 1054 1055 thc_dma_write_completion(dev); 1056 mutex_unlock(&dev->thc_bus_lock); 1057 return 0; 1058 1059 end: 1060 mutex_unlock(&dev->thc_bus_lock); 1061 1062 if (restore_interrupts) 1063 ret = thc_interrupt_quiesce(dev, false); 1064 1065 return ret; 1066 } 1067 EXPORT_SYMBOL_NS_GPL(thc_dma_write, "INTEL_THC"); 1068