1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <linux/firmware.h> 13 #include <linux/interrupt.h> 14 #include <linux/module.h> 15 #include <linux/of_platform.h> 16 #include <linux/of_irq.h> 17 #include <linux/pci.h> 18 #include <linux/platform_device.h> 19 #include <linux/workqueue.h> 20 21 #include <crypto/internal/aead.h> 22 #include <crypto/internal/hash.h> 23 #include <crypto/internal/skcipher.h> 24 25 #include "safexcel.h" 26 27 static u32 max_rings = EIP197_MAX_RINGS; 28 module_param(max_rings, uint, 0644); 29 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); 30 31 static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv) 32 { 33 int i; 34 35 /* 36 * Map all interfaces/rings to register index 0 37 * so they can share contexts. Without this, the EIP197 will 38 * assume each interface/ring to be in its own memory domain 39 * i.e. have its own subset of UNIQUE memory addresses. 40 * Which would cause records with the SAME memory address to 41 * use DIFFERENT cache buffers, causing both poor cache utilization 42 * AND serious coherence/invalidation issues. 43 */ 44 for (i = 0; i < 4; i++) 45 writel(0, priv->base + EIP197_FLUE_IFC_LUT(i)); 46 47 /* 48 * Initialize other virtualization regs for cache 49 * These may not be in their reset state ... 50 */ 51 for (i = 0; i < priv->config.rings; i++) { 52 writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i)); 53 writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i)); 54 writel(EIP197_FLUE_CONFIG_MAGIC, 55 priv->base + EIP197_FLUE_CONFIG(i)); 56 } 57 writel(0, priv->base + EIP197_FLUE_OFFSETS); 58 writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET); 59 } 60 61 static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv, 62 u32 addrmid, int *actbank) 63 { 64 u32 val; 65 int curbank; 66 67 curbank = addrmid >> 16; 68 if (curbank != *actbank) { 69 val = readl(priv->base + EIP197_CS_RAM_CTRL); 70 val = (val & ~EIP197_CS_BANKSEL_MASK) | 71 (curbank << EIP197_CS_BANKSEL_OFS); 72 writel(val, priv->base + EIP197_CS_RAM_CTRL); 73 *actbank = curbank; 74 } 75 } 76 77 static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv, 78 int maxbanks, u32 probemask) 79 { 80 u32 val, addrhi, addrlo, addrmid; 81 int actbank; 82 83 /* 84 * And probe the actual size of the physically attached cache data RAM 85 * Using a binary subdivision algorithm downto 32 byte cache lines. 86 */ 87 addrhi = 1 << (16 + maxbanks); 88 addrlo = 0; 89 actbank = min(maxbanks - 1, 0); 90 while ((addrhi - addrlo) > 32) { 91 /* write marker to lowest address in top half */ 92 addrmid = (addrhi + addrlo) >> 1; 93 eip197_trc_cache_banksel(priv, addrmid, &actbank); 94 writel((addrmid | (addrlo << 16)) & probemask, 95 priv->base + EIP197_CLASSIFICATION_RAMS + 96 (addrmid & 0xffff)); 97 98 /* write marker to lowest address in bottom half */ 99 eip197_trc_cache_banksel(priv, addrlo, &actbank); 100 writel((addrlo | (addrhi << 16)) & probemask, 101 priv->base + EIP197_CLASSIFICATION_RAMS + 102 (addrlo & 0xffff)); 103 104 /* read back marker from top half */ 105 eip197_trc_cache_banksel(priv, addrmid, &actbank); 106 val = readl(priv->base + EIP197_CLASSIFICATION_RAMS + 107 (addrmid & 0xffff)); 108 109 if (val == ((addrmid | (addrlo << 16)) & probemask)) { 110 /* read back correct, continue with top half */ 111 addrlo = addrmid; 112 } else { 113 /* not read back correct, continue with bottom half */ 114 addrhi = addrmid; 115 } 116 } 117 return addrhi; 118 } 119 120 static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv, 121 int cs_rc_max, int cs_ht_wc) 122 { 123 int i; 124 u32 htable_offset, val, offset; 125 126 /* Clear all records in administration RAM */ 127 for (i = 0; i < cs_rc_max; i++) { 128 offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; 129 130 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | 131 EIP197_CS_RC_PREV(EIP197_RC_NULL), 132 priv->base + offset); 133 134 val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1); 135 if (i == 0) 136 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); 137 else if (i == cs_rc_max - 1) 138 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); 139 writel(val, priv->base + offset + 4); 140 /* must also initialize the address key due to ECC! */ 141 writel(0, priv->base + offset + 8); 142 writel(0, priv->base + offset + 12); 143 } 144 145 /* Clear the hash table entries */ 146 htable_offset = cs_rc_max * EIP197_CS_RC_SIZE; 147 for (i = 0; i < cs_ht_wc; i++) 148 writel(GENMASK(29, 0), 149 priv->base + EIP197_CLASSIFICATION_RAMS + 150 htable_offset + i * sizeof(u32)); 151 } 152 153 static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) 154 { 155 u32 val, dsize, asize; 156 int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc; 157 int cs_rc_abs_max, cs_ht_sz; 158 int maxbanks; 159 160 /* Setup (dummy) virtualization for cache */ 161 eip197_trc_cache_setupvirt(priv); 162 163 /* 164 * Enable the record cache memory access and 165 * probe the bank select width 166 */ 167 val = readl(priv->base + EIP197_CS_RAM_CTRL); 168 val &= ~EIP197_TRC_ENABLE_MASK; 169 val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK; 170 writel(val, priv->base + EIP197_CS_RAM_CTRL); 171 val = readl(priv->base + EIP197_CS_RAM_CTRL); 172 maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1; 173 174 /* Clear all ECC errors */ 175 writel(0, priv->base + EIP197_TRC_ECCCTRL); 176 177 /* 178 * Make sure the cache memory is accessible by taking record cache into 179 * reset. Need data memory access here, not admin access. 180 */ 181 val = readl(priv->base + EIP197_TRC_PARAMS); 182 val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS; 183 writel(val, priv->base + EIP197_TRC_PARAMS); 184 185 /* Probed data RAM size in bytes */ 186 dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff); 187 188 /* 189 * Now probe the administration RAM size pretty much the same way 190 * Except that only the lower 30 bits are writable and we don't need 191 * bank selects 192 */ 193 val = readl(priv->base + EIP197_TRC_PARAMS); 194 /* admin access now */ 195 val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK); 196 writel(val, priv->base + EIP197_TRC_PARAMS); 197 198 /* Probed admin RAM size in admin words */ 199 asize = eip197_trc_cache_probe(priv, 0, 0xbfffffff) >> 4; 200 201 /* Clear any ECC errors detected while probing! */ 202 writel(0, priv->base + EIP197_TRC_ECCCTRL); 203 204 /* 205 * Determine optimal configuration from RAM sizes 206 * Note that we assume that the physical RAM configuration is sane 207 * Therefore, we don't do any parameter error checking here ... 208 */ 209 210 /* For now, just use a single record format covering everything */ 211 cs_trc_rec_wc = EIP197_CS_TRC_REC_WC; 212 cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC; 213 214 /* 215 * Step #1: How many records will physically fit? 216 * Hard upper limit is 1023! 217 */ 218 cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023); 219 /* Step #2: Need at least 2 words in the admin RAM per record */ 220 cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1)); 221 /* Step #3: Determine log2 of hash table size */ 222 cs_ht_sz = __fls(asize - cs_rc_max) - 2; 223 /* Step #4: determine current size of hash table in dwords */ 224 cs_ht_wc = 16<<cs_ht_sz; /* dwords, not admin words */ 225 /* Step #5: add back excess words and see if we can fit more records */ 226 cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 4)); 227 228 /* Clear the cache RAMs */ 229 eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc); 230 231 /* Disable the record cache memory access */ 232 val = readl(priv->base + EIP197_CS_RAM_CTRL); 233 val &= ~EIP197_TRC_ENABLE_MASK; 234 writel(val, priv->base + EIP197_CS_RAM_CTRL); 235 236 /* Write head and tail pointers of the record free chain */ 237 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | 238 EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1); 239 writel(val, priv->base + EIP197_TRC_FREECHAIN); 240 241 /* Configure the record cache #1 */ 242 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) | 243 EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max); 244 writel(val, priv->base + EIP197_TRC_PARAMS2); 245 246 /* Configure the record cache #2 */ 247 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) | 248 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | 249 EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz); 250 writel(val, priv->base + EIP197_TRC_PARAMS); 251 252 dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n", 253 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc); 254 } 255 256 static void eip197_init_firmware(struct safexcel_crypto_priv *priv) 257 { 258 int pe, i; 259 u32 val; 260 261 for (pe = 0; pe < priv->config.pes; pe++) { 262 /* Configure the token FIFO's */ 263 writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe)); 264 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe)); 265 266 /* Clear the ICE scratchpad memory */ 267 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); 268 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | 269 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | 270 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | 271 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; 272 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); 273 274 /* clear the scratchpad RAM using 32 bit writes only */ 275 for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++) 276 writel(0, EIP197_PE(priv) + 277 EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2)); 278 279 /* Reset the IFPP engine to make its program mem accessible */ 280 writel(EIP197_PE_ICE_x_CTRL_SW_RESET | 281 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | 282 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, 283 EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe)); 284 285 /* Reset the IPUE engine to make its program mem accessible */ 286 writel(EIP197_PE_ICE_x_CTRL_SW_RESET | 287 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | 288 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, 289 EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe)); 290 291 /* Enable access to all IFPP program memories */ 292 writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN, 293 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); 294 } 295 296 } 297 298 static int eip197_write_firmware(struct safexcel_crypto_priv *priv, 299 const struct firmware *fw) 300 { 301 const u32 *data = (const u32 *)fw->data; 302 int i; 303 304 /* Write the firmware */ 305 for (i = 0; i < fw->size / sizeof(u32); i++) 306 writel(be32_to_cpu(data[i]), 307 priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); 308 309 /* Exclude final 2 NOPs from size */ 310 return i - EIP197_FW_TERMINAL_NOPS; 311 } 312 313 /* 314 * If FW is actual production firmware, then poll for its initialization 315 * to complete and check if it is good for the HW, otherwise just return OK. 316 */ 317 static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp) 318 { 319 int pe, pollcnt; 320 u32 base, pollofs; 321 322 if (fpp) 323 pollofs = EIP197_FW_FPP_READY; 324 else 325 pollofs = EIP197_FW_PUE_READY; 326 327 for (pe = 0; pe < priv->config.pes; pe++) { 328 base = EIP197_PE_ICE_SCRATCH_RAM(pe); 329 pollcnt = EIP197_FW_START_POLLCNT; 330 while (pollcnt && 331 (readl_relaxed(EIP197_PE(priv) + base + 332 pollofs) != 1)) { 333 pollcnt--; 334 } 335 if (!pollcnt) { 336 dev_err(priv->dev, "FW(%d) for PE %d failed to start\n", 337 fpp, pe); 338 return false; 339 } 340 } 341 return true; 342 } 343 344 static bool eip197_start_firmware(struct safexcel_crypto_priv *priv, 345 int ipuesz, int ifppsz, int minifw) 346 { 347 int pe; 348 u32 val; 349 350 for (pe = 0; pe < priv->config.pes; pe++) { 351 /* Disable access to all program memory */ 352 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); 353 354 /* Start IFPP microengines */ 355 if (minifw) 356 val = 0; 357 else 358 val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) & 359 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) | 360 EIP197_PE_ICE_UENG_DEBUG_RESET; 361 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe)); 362 363 /* Start IPUE microengines */ 364 if (minifw) 365 val = 0; 366 else 367 val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) & 368 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) | 369 EIP197_PE_ICE_UENG_DEBUG_RESET; 370 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe)); 371 } 372 373 /* For miniFW startup, there is no initialization, so always succeed */ 374 if (minifw) 375 return true; 376 377 /* Wait until all the firmwares have properly started up */ 378 if (!poll_fw_ready(priv, 1)) 379 return false; 380 if (!poll_fw_ready(priv, 0)) 381 return false; 382 383 return true; 384 } 385 386 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) 387 { 388 const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; 389 const struct firmware *fw[FW_NB]; 390 char fw_path[37], *dir = NULL; 391 int i, j, ret = 0, pe; 392 int ipuesz, ifppsz, minifw = 0; 393 394 if (priv->version == EIP197D_MRVL) 395 dir = "eip197d"; 396 else if (priv->version == EIP197B_MRVL || 397 priv->version == EIP197_DEVBRD) 398 dir = "eip197b"; 399 else 400 return -ENODEV; 401 402 retry_fw: 403 for (i = 0; i < FW_NB; i++) { 404 snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]); 405 ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev); 406 if (ret) { 407 if (minifw || priv->version != EIP197B_MRVL) 408 goto release_fw; 409 410 /* Fallback to the old firmware location for the 411 * EIP197b. 412 */ 413 ret = firmware_request_nowarn(&fw[i], fw_name[i], 414 priv->dev); 415 if (ret) 416 goto release_fw; 417 } 418 } 419 420 eip197_init_firmware(priv); 421 422 ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]); 423 424 /* Enable access to IPUE program memories */ 425 for (pe = 0; pe < priv->config.pes; pe++) 426 writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN, 427 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); 428 429 ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]); 430 431 if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) { 432 dev_dbg(priv->dev, "Firmware loaded successfully\n"); 433 return 0; 434 } 435 436 ret = -ENODEV; 437 438 release_fw: 439 for (j = 0; j < i; j++) 440 release_firmware(fw[j]); 441 442 if (!minifw) { 443 /* Retry with minifw path */ 444 dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n"); 445 dir = "eip197_minifw"; 446 minifw = 1; 447 goto retry_fw; 448 } 449 450 dev_dbg(priv->dev, "Firmware load failed.\n"); 451 452 return ret; 453 } 454 455 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) 456 { 457 u32 cd_size_rnd, val; 458 int i, cd_fetch_cnt; 459 460 cd_size_rnd = (priv->config.cd_size + 461 (BIT(priv->hwconfig.hwdataw) - 1)) >> 462 priv->hwconfig.hwdataw; 463 /* determine number of CD's we can fetch into the CD FIFO as 1 block */ 464 if (priv->flags & SAFEXCEL_HW_EIP197) { 465 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */ 466 cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd; 467 cd_fetch_cnt = min_t(uint, cd_fetch_cnt, 468 (priv->config.pes * EIP197_FETCH_DEPTH)); 469 } else { 470 /* for the EIP97, just fetch all that fits minus 1 */ 471 cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) / 472 cd_size_rnd) - 1; 473 } 474 475 for (i = 0; i < priv->config.rings; i++) { 476 /* ring base address */ 477 writel(lower_32_bits(priv->ring[i].cdr.base_dma), 478 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 479 writel(upper_32_bits(priv->ring[i].cdr.base_dma), 480 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 481 482 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | 483 priv->config.cd_size, 484 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); 485 writel(((cd_fetch_cnt * 486 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) | 487 (cd_fetch_cnt * priv->config.cd_offset), 488 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); 489 490 /* Configure DMA tx control */ 491 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); 492 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); 493 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); 494 495 /* clear any pending interrupt */ 496 writel(GENMASK(5, 0), 497 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); 498 } 499 500 return 0; 501 } 502 503 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) 504 { 505 u32 rd_size_rnd, val; 506 int i, rd_fetch_cnt; 507 508 /* determine number of RD's we can fetch into the FIFO as one block */ 509 rd_size_rnd = (EIP197_RD64_FETCH_SIZE + 510 (BIT(priv->hwconfig.hwdataw) - 1)) >> 511 priv->hwconfig.hwdataw; 512 if (priv->flags & SAFEXCEL_HW_EIP197) { 513 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */ 514 rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd; 515 rd_fetch_cnt = min_t(uint, rd_fetch_cnt, 516 (priv->config.pes * EIP197_FETCH_DEPTH)); 517 } else { 518 /* for the EIP97, just fetch all that fits minus 1 */ 519 rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) / 520 rd_size_rnd) - 1; 521 } 522 523 for (i = 0; i < priv->config.rings; i++) { 524 /* ring base address */ 525 writel(lower_32_bits(priv->ring[i].rdr.base_dma), 526 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 527 writel(upper_32_bits(priv->ring[i].rdr.base_dma), 528 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 529 530 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | 531 priv->config.rd_size, 532 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); 533 534 writel(((rd_fetch_cnt * 535 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) | 536 (rd_fetch_cnt * priv->config.rd_offset), 537 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); 538 539 /* Configure DMA tx control */ 540 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); 541 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); 542 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF; 543 writel(val, 544 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); 545 546 /* clear any pending interrupt */ 547 writel(GENMASK(7, 0), 548 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); 549 550 /* enable ring interrupt */ 551 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); 552 val |= EIP197_RDR_IRQ(i); 553 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); 554 } 555 556 return 0; 557 } 558 559 static int safexcel_hw_init(struct safexcel_crypto_priv *priv) 560 { 561 u32 val; 562 int i, ret, pe; 563 564 dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n", 565 priv->config.pes, priv->config.rings); 566 567 /* 568 * For EIP197's only set maximum number of TX commands to 2^5 = 32 569 * Skip for the EIP97 as it does not have this field. 570 */ 571 if (priv->flags & SAFEXCEL_HW_EIP197) { 572 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 573 val |= EIP197_MST_CTRL_TX_MAX_CMD(5); 574 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 575 } 576 577 /* Configure wr/rd cache values */ 578 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | 579 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), 580 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL); 581 582 /* Interrupts reset */ 583 584 /* Disable all global interrupts */ 585 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL); 586 587 /* Clear any pending interrupt */ 588 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); 589 590 /* Processing Engine configuration */ 591 for (pe = 0; pe < priv->config.pes; pe++) { 592 /* Data Fetch Engine configuration */ 593 594 /* Reset all DFE threads */ 595 writel(EIP197_DxE_THR_CTRL_RESET_PE, 596 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); 597 598 if (priv->flags & SAFEXCEL_HW_EIP197) 599 /* Reset HIA input interface arbiter (EIP197 only) */ 600 writel(EIP197_HIA_RA_PE_CTRL_RESET, 601 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); 602 603 /* DMA transfer size to use */ 604 val = EIP197_HIA_DFE_CFG_DIS_DEBUG; 605 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) | 606 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); 607 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | 608 EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); 609 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); 610 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); 611 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe)); 612 613 /* Leave the DFE threads reset state */ 614 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); 615 616 /* Configure the processing engine thresholds */ 617 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | 618 EIP197_PE_IN_xBUF_THRES_MAX(9), 619 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe)); 620 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | 621 EIP197_PE_IN_xBUF_THRES_MAX(7), 622 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe)); 623 624 if (priv->flags & SAFEXCEL_HW_EIP197) 625 /* enable HIA input interface arbiter and rings */ 626 writel(EIP197_HIA_RA_PE_CTRL_EN | 627 GENMASK(priv->config.rings - 1, 0), 628 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); 629 630 /* Data Store Engine configuration */ 631 632 /* Reset all DSE threads */ 633 writel(EIP197_DxE_THR_CTRL_RESET_PE, 634 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); 635 636 /* Wait for all DSE threads to complete */ 637 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) & 638 GENMASK(15, 12)) != GENMASK(15, 12)) 639 ; 640 641 /* DMA transfer size to use */ 642 val = EIP197_HIA_DSE_CFG_DIS_DEBUG; 643 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | 644 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); 645 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); 646 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE; 647 /* FIXME: instability issues can occur for EIP97 but disabling 648 * it impacts performance. 649 */ 650 if (priv->flags & SAFEXCEL_HW_EIP197) 651 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; 652 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe)); 653 654 /* Leave the DSE threads reset state */ 655 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); 656 657 /* Configure the procesing engine thresholds */ 658 writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | 659 EIP197_PE_OUT_DBUF_THRES_MAX(8), 660 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe)); 661 662 /* Processing Engine configuration */ 663 664 /* Token & context configuration */ 665 val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES | 666 EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT | 667 EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT; 668 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe)); 669 670 /* H/W capabilities selection: just enable everything */ 671 writel(EIP197_FUNCTION_ALL, 672 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe)); 673 writel(EIP197_FUNCTION_ALL, 674 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe)); 675 } 676 677 /* Command Descriptor Rings prepare */ 678 for (i = 0; i < priv->config.rings; i++) { 679 /* Clear interrupts for this ring */ 680 writel(GENMASK(31, 0), 681 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i)); 682 683 /* Disable external triggering */ 684 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); 685 686 /* Clear the pending prepared counter */ 687 writel(EIP197_xDR_PREP_CLR_COUNT, 688 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); 689 690 /* Clear the pending processed counter */ 691 writel(EIP197_xDR_PROC_CLR_COUNT, 692 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); 693 694 writel(0, 695 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); 696 writel(0, 697 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); 698 699 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, 700 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); 701 } 702 703 /* Result Descriptor Ring prepare */ 704 for (i = 0; i < priv->config.rings; i++) { 705 /* Disable external triggering*/ 706 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); 707 708 /* Clear the pending prepared counter */ 709 writel(EIP197_xDR_PREP_CLR_COUNT, 710 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); 711 712 /* Clear the pending processed counter */ 713 writel(EIP197_xDR_PROC_CLR_COUNT, 714 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); 715 716 writel(0, 717 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); 718 writel(0, 719 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); 720 721 /* Ring size */ 722 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, 723 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); 724 } 725 726 for (pe = 0; pe < priv->config.pes; pe++) { 727 /* Enable command descriptor rings */ 728 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), 729 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); 730 731 /* Enable result descriptor rings */ 732 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), 733 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); 734 } 735 736 /* Clear any HIA interrupt */ 737 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); 738 739 if (priv->flags & SAFEXCEL_HW_EIP197) { 740 eip197_trc_cache_init(priv); 741 priv->flags |= EIP197_TRC_CACHE; 742 743 ret = eip197_load_firmwares(priv); 744 if (ret) 745 return ret; 746 } 747 748 safexcel_hw_setup_cdesc_rings(priv); 749 safexcel_hw_setup_rdesc_rings(priv); 750 751 return 0; 752 } 753 754 /* Called with ring's lock taken */ 755 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv, 756 int ring) 757 { 758 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ); 759 760 if (!coal) 761 return; 762 763 /* Configure when we want an interrupt */ 764 writel(EIP197_HIA_RDR_THRESH_PKT_MODE | 765 EIP197_HIA_RDR_THRESH_PROC_PKT(coal), 766 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH); 767 } 768 769 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) 770 { 771 struct crypto_async_request *req, *backlog; 772 struct safexcel_context *ctx; 773 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; 774 775 /* If a request wasn't properly dequeued because of a lack of resources, 776 * proceeded it first, 777 */ 778 req = priv->ring[ring].req; 779 backlog = priv->ring[ring].backlog; 780 if (req) 781 goto handle_req; 782 783 while (true) { 784 spin_lock_bh(&priv->ring[ring].queue_lock); 785 backlog = crypto_get_backlog(&priv->ring[ring].queue); 786 req = crypto_dequeue_request(&priv->ring[ring].queue); 787 spin_unlock_bh(&priv->ring[ring].queue_lock); 788 789 if (!req) { 790 priv->ring[ring].req = NULL; 791 priv->ring[ring].backlog = NULL; 792 goto finalize; 793 } 794 795 handle_req: 796 ctx = crypto_tfm_ctx(req->tfm); 797 ret = ctx->send(req, ring, &commands, &results); 798 if (ret) 799 goto request_failed; 800 801 if (backlog) 802 backlog->complete(backlog, -EINPROGRESS); 803 804 /* In case the send() helper did not issue any command to push 805 * to the engine because the input data was cached, continue to 806 * dequeue other requests as this is valid and not an error. 807 */ 808 if (!commands && !results) 809 continue; 810 811 cdesc += commands; 812 rdesc += results; 813 nreq++; 814 } 815 816 request_failed: 817 /* Not enough resources to handle all the requests. Bail out and save 818 * the request and the backlog for the next dequeue call (per-ring). 819 */ 820 priv->ring[ring].req = req; 821 priv->ring[ring].backlog = backlog; 822 823 finalize: 824 if (!nreq) 825 return; 826 827 spin_lock_bh(&priv->ring[ring].lock); 828 829 priv->ring[ring].requests += nreq; 830 831 if (!priv->ring[ring].busy) { 832 safexcel_try_push_requests(priv, ring); 833 priv->ring[ring].busy = true; 834 } 835 836 spin_unlock_bh(&priv->ring[ring].lock); 837 838 /* let the RDR know we have pending descriptors */ 839 writel((rdesc * priv->config.rd_offset) << 2, 840 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); 841 842 /* let the CDR know we have pending descriptors */ 843 writel((cdesc * priv->config.cd_offset) << 2, 844 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); 845 } 846 847 inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, 848 struct safexcel_result_desc *rdesc) 849 { 850 if (likely((!rdesc->descriptor_overflow) && 851 (!rdesc->buffer_overflow) && 852 (!rdesc->result_data.error_code))) 853 return 0; 854 855 if (rdesc->descriptor_overflow) 856 dev_err(priv->dev, "Descriptor overflow detected"); 857 858 if (rdesc->buffer_overflow) 859 dev_err(priv->dev, "Buffer overflow detected"); 860 861 if (rdesc->result_data.error_code & 0x4066) { 862 /* Fatal error (bits 1,2,5,6 & 14) */ 863 dev_err(priv->dev, 864 "result descriptor error (%x)", 865 rdesc->result_data.error_code); 866 return -EIO; 867 } else if (rdesc->result_data.error_code & 868 (BIT(7) | BIT(4) | BIT(3) | BIT(0))) { 869 /* 870 * Give priority over authentication fails: 871 * Blocksize, length & overflow errors, 872 * something wrong with the input! 873 */ 874 return -EINVAL; 875 } else if (rdesc->result_data.error_code & BIT(9)) { 876 /* Authentication failed */ 877 return -EBADMSG; 878 } 879 880 /* All other non-fatal errors */ 881 return -EINVAL; 882 } 883 884 inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, 885 int ring, 886 struct safexcel_result_desc *rdesc, 887 struct crypto_async_request *req) 888 { 889 int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc); 890 891 priv->ring[ring].rdr_req[i] = req; 892 } 893 894 inline struct crypto_async_request * 895 safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring) 896 { 897 int i = safexcel_ring_first_rdr_index(priv, ring); 898 899 return priv->ring[ring].rdr_req[i]; 900 } 901 902 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) 903 { 904 struct safexcel_command_desc *cdesc; 905 906 /* Acknowledge the command descriptors */ 907 do { 908 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); 909 if (IS_ERR(cdesc)) { 910 dev_err(priv->dev, 911 "Could not retrieve the command descriptor\n"); 912 return; 913 } 914 } while (!cdesc->last_seg); 915 } 916 917 void safexcel_inv_complete(struct crypto_async_request *req, int error) 918 { 919 struct safexcel_inv_result *result = req->data; 920 921 if (error == -EINPROGRESS) 922 return; 923 924 result->error = error; 925 complete(&result->completion); 926 } 927 928 int safexcel_invalidate_cache(struct crypto_async_request *async, 929 struct safexcel_crypto_priv *priv, 930 dma_addr_t ctxr_dma, int ring) 931 { 932 struct safexcel_command_desc *cdesc; 933 struct safexcel_result_desc *rdesc; 934 int ret = 0; 935 936 /* Prepare command descriptor */ 937 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma); 938 if (IS_ERR(cdesc)) 939 return PTR_ERR(cdesc); 940 941 cdesc->control_data.type = EIP197_TYPE_EXTENDED; 942 cdesc->control_data.options = 0; 943 cdesc->control_data.refresh = 0; 944 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; 945 946 /* Prepare result descriptor */ 947 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 948 949 if (IS_ERR(rdesc)) { 950 ret = PTR_ERR(rdesc); 951 goto cdesc_rollback; 952 } 953 954 safexcel_rdr_req_set(priv, ring, rdesc, async); 955 956 return ret; 957 958 cdesc_rollback: 959 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 960 961 return ret; 962 } 963 964 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, 965 int ring) 966 { 967 struct crypto_async_request *req; 968 struct safexcel_context *ctx; 969 int ret, i, nreq, ndesc, tot_descs, handled = 0; 970 bool should_complete; 971 972 handle_results: 973 tot_descs = 0; 974 975 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); 976 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; 977 nreq &= EIP197_xDR_PROC_xD_PKT_MASK; 978 if (!nreq) 979 goto requests_left; 980 981 for (i = 0; i < nreq; i++) { 982 req = safexcel_rdr_req_get(priv, ring); 983 984 ctx = crypto_tfm_ctx(req->tfm); 985 ndesc = ctx->handle_result(priv, ring, req, 986 &should_complete, &ret); 987 if (ndesc < 0) { 988 dev_err(priv->dev, "failed to handle result (%d)\n", 989 ndesc); 990 goto acknowledge; 991 } 992 993 if (should_complete) { 994 local_bh_disable(); 995 req->complete(req, ret); 996 local_bh_enable(); 997 } 998 999 tot_descs += ndesc; 1000 handled++; 1001 } 1002 1003 acknowledge: 1004 if (i) 1005 writel(EIP197_xDR_PROC_xD_PKT(i) | 1006 EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), 1007 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); 1008 1009 /* If the number of requests overflowed the counter, try to proceed more 1010 * requests. 1011 */ 1012 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK) 1013 goto handle_results; 1014 1015 requests_left: 1016 spin_lock_bh(&priv->ring[ring].lock); 1017 1018 priv->ring[ring].requests -= handled; 1019 safexcel_try_push_requests(priv, ring); 1020 1021 if (!priv->ring[ring].requests) 1022 priv->ring[ring].busy = false; 1023 1024 spin_unlock_bh(&priv->ring[ring].lock); 1025 } 1026 1027 static void safexcel_dequeue_work(struct work_struct *work) 1028 { 1029 struct safexcel_work_data *data = 1030 container_of(work, struct safexcel_work_data, work); 1031 1032 safexcel_dequeue(data->priv, data->ring); 1033 } 1034 1035 struct safexcel_ring_irq_data { 1036 struct safexcel_crypto_priv *priv; 1037 int ring; 1038 }; 1039 1040 static irqreturn_t safexcel_irq_ring(int irq, void *data) 1041 { 1042 struct safexcel_ring_irq_data *irq_data = data; 1043 struct safexcel_crypto_priv *priv = irq_data->priv; 1044 int ring = irq_data->ring, rc = IRQ_NONE; 1045 u32 status, stat; 1046 1047 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); 1048 if (!status) 1049 return rc; 1050 1051 /* RDR interrupts */ 1052 if (status & EIP197_RDR_IRQ(ring)) { 1053 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); 1054 1055 if (unlikely(stat & EIP197_xDR_ERR)) { 1056 /* 1057 * Fatal error, the RDR is unusable and must be 1058 * reinitialized. This should not happen under 1059 * normal circumstances. 1060 */ 1061 dev_err(priv->dev, "RDR: fatal error.\n"); 1062 } else if (likely(stat & EIP197_xDR_THRESH)) { 1063 rc = IRQ_WAKE_THREAD; 1064 } 1065 1066 /* ACK the interrupts */ 1067 writel(stat & 0xff, 1068 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); 1069 } 1070 1071 /* ACK the interrupts */ 1072 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring)); 1073 1074 return rc; 1075 } 1076 1077 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data) 1078 { 1079 struct safexcel_ring_irq_data *irq_data = data; 1080 struct safexcel_crypto_priv *priv = irq_data->priv; 1081 int ring = irq_data->ring; 1082 1083 safexcel_handle_result_descriptor(priv, ring); 1084 1085 queue_work(priv->ring[ring].workqueue, 1086 &priv->ring[ring].work_data.work); 1087 1088 return IRQ_HANDLED; 1089 } 1090 1091 static int safexcel_request_ring_irq(void *pdev, int irqid, 1092 int is_pci_dev, 1093 irq_handler_t handler, 1094 irq_handler_t threaded_handler, 1095 struct safexcel_ring_irq_data *ring_irq_priv) 1096 { 1097 int ret, irq; 1098 struct device *dev; 1099 1100 if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) { 1101 struct pci_dev *pci_pdev = pdev; 1102 1103 dev = &pci_pdev->dev; 1104 irq = pci_irq_vector(pci_pdev, irqid); 1105 if (irq < 0) { 1106 dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n", 1107 irqid, irq); 1108 return irq; 1109 } 1110 } else if (IS_ENABLED(CONFIG_OF)) { 1111 struct platform_device *plf_pdev = pdev; 1112 char irq_name[6] = {0}; /* "ringX\0" */ 1113 1114 snprintf(irq_name, 6, "ring%d", irqid); 1115 dev = &plf_pdev->dev; 1116 irq = platform_get_irq_byname(plf_pdev, irq_name); 1117 1118 if (irq < 0) { 1119 dev_err(dev, "unable to get IRQ '%s' (err %d)\n", 1120 irq_name, irq); 1121 return irq; 1122 } 1123 } 1124 1125 ret = devm_request_threaded_irq(dev, irq, handler, 1126 threaded_handler, IRQF_ONESHOT, 1127 dev_name(dev), ring_irq_priv); 1128 if (ret) { 1129 dev_err(dev, "unable to request IRQ %d\n", irq); 1130 return ret; 1131 } 1132 1133 return irq; 1134 } 1135 1136 static struct safexcel_alg_template *safexcel_algs[] = { 1137 &safexcel_alg_ecb_des, 1138 &safexcel_alg_cbc_des, 1139 &safexcel_alg_ecb_des3_ede, 1140 &safexcel_alg_cbc_des3_ede, 1141 &safexcel_alg_ecb_aes, 1142 &safexcel_alg_cbc_aes, 1143 &safexcel_alg_cfb_aes, 1144 &safexcel_alg_ofb_aes, 1145 &safexcel_alg_ctr_aes, 1146 &safexcel_alg_md5, 1147 &safexcel_alg_sha1, 1148 &safexcel_alg_sha224, 1149 &safexcel_alg_sha256, 1150 &safexcel_alg_sha384, 1151 &safexcel_alg_sha512, 1152 &safexcel_alg_hmac_md5, 1153 &safexcel_alg_hmac_sha1, 1154 &safexcel_alg_hmac_sha224, 1155 &safexcel_alg_hmac_sha256, 1156 &safexcel_alg_hmac_sha384, 1157 &safexcel_alg_hmac_sha512, 1158 &safexcel_alg_authenc_hmac_sha1_cbc_aes, 1159 &safexcel_alg_authenc_hmac_sha224_cbc_aes, 1160 &safexcel_alg_authenc_hmac_sha256_cbc_aes, 1161 &safexcel_alg_authenc_hmac_sha384_cbc_aes, 1162 &safexcel_alg_authenc_hmac_sha512_cbc_aes, 1163 &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede, 1164 &safexcel_alg_authenc_hmac_sha1_ctr_aes, 1165 &safexcel_alg_authenc_hmac_sha224_ctr_aes, 1166 &safexcel_alg_authenc_hmac_sha256_ctr_aes, 1167 &safexcel_alg_authenc_hmac_sha384_ctr_aes, 1168 &safexcel_alg_authenc_hmac_sha512_ctr_aes, 1169 &safexcel_alg_xts_aes, 1170 &safexcel_alg_gcm, 1171 &safexcel_alg_ccm, 1172 }; 1173 1174 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) 1175 { 1176 int i, j, ret = 0; 1177 1178 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { 1179 safexcel_algs[i]->priv = priv; 1180 1181 /* Do we have all required base algorithms available? */ 1182 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) != 1183 safexcel_algs[i]->algo_mask) 1184 /* No, so don't register this ciphersuite */ 1185 continue; 1186 1187 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 1188 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); 1189 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) 1190 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead); 1191 else 1192 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); 1193 1194 if (ret) 1195 goto fail; 1196 } 1197 1198 return 0; 1199 1200 fail: 1201 for (j = 0; j < i; j++) { 1202 /* Do we have all required base algorithms available? */ 1203 if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) != 1204 safexcel_algs[j]->algo_mask) 1205 /* No, so don't unregister this ciphersuite */ 1206 continue; 1207 1208 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 1209 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); 1210 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD) 1211 crypto_unregister_aead(&safexcel_algs[j]->alg.aead); 1212 else 1213 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); 1214 } 1215 1216 return ret; 1217 } 1218 1219 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) 1220 { 1221 int i; 1222 1223 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { 1224 /* Do we have all required base algorithms available? */ 1225 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) != 1226 safexcel_algs[i]->algo_mask) 1227 /* No, so don't unregister this ciphersuite */ 1228 continue; 1229 1230 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 1231 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); 1232 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) 1233 crypto_unregister_aead(&safexcel_algs[i]->alg.aead); 1234 else 1235 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); 1236 } 1237 } 1238 1239 static void safexcel_configure(struct safexcel_crypto_priv *priv) 1240 { 1241 u32 val, mask = 0; 1242 1243 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); 1244 1245 /* Read number of PEs from the engine */ 1246 if (priv->flags & SAFEXCEL_HW_EIP197) 1247 /* Wider field width for all EIP197 type engines */ 1248 mask = EIP197_N_PES_MASK; 1249 else 1250 /* Narrow field width for EIP97 type engine */ 1251 mask = EIP97_N_PES_MASK; 1252 1253 priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask; 1254 1255 priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); 1256 1257 val = (val & GENMASK(27, 25)) >> 25; 1258 mask = BIT(val) - 1; 1259 1260 priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); 1261 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; 1262 1263 priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32)); 1264 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; 1265 } 1266 1267 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) 1268 { 1269 struct safexcel_register_offsets *offsets = &priv->offsets; 1270 1271 if (priv->flags & SAFEXCEL_HW_EIP197) { 1272 offsets->hia_aic = EIP197_HIA_AIC_BASE; 1273 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE; 1274 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE; 1275 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE; 1276 offsets->hia_dfe = EIP197_HIA_DFE_BASE; 1277 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE; 1278 offsets->hia_dse = EIP197_HIA_DSE_BASE; 1279 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE; 1280 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE; 1281 offsets->pe = EIP197_PE_BASE; 1282 offsets->global = EIP197_GLOBAL_BASE; 1283 } else { 1284 offsets->hia_aic = EIP97_HIA_AIC_BASE; 1285 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE; 1286 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE; 1287 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE; 1288 offsets->hia_dfe = EIP97_HIA_DFE_BASE; 1289 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE; 1290 offsets->hia_dse = EIP97_HIA_DSE_BASE; 1291 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE; 1292 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE; 1293 offsets->pe = EIP97_PE_BASE; 1294 offsets->global = EIP97_GLOBAL_BASE; 1295 } 1296 } 1297 1298 /* 1299 * Generic part of probe routine, shared by platform and PCI driver 1300 * 1301 * Assumes IO resources have been mapped, private data mem has been allocated, 1302 * clocks have been enabled, device pointer has been assigned etc. 1303 * 1304 */ 1305 static int safexcel_probe_generic(void *pdev, 1306 struct safexcel_crypto_priv *priv, 1307 int is_pci_dev) 1308 { 1309 struct device *dev = priv->dev; 1310 u32 peid, version, mask, val, hiaopt; 1311 int i, ret, hwctg; 1312 1313 priv->context_pool = dmam_pool_create("safexcel-context", dev, 1314 sizeof(struct safexcel_context_record), 1315 1, 0); 1316 if (!priv->context_pool) 1317 return -ENOMEM; 1318 1319 /* 1320 * First try the EIP97 HIA version regs 1321 * For the EIP197, this is guaranteed to NOT return any of the test 1322 * values 1323 */ 1324 version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION); 1325 1326 mask = 0; /* do not swap */ 1327 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) { 1328 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version); 1329 } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) { 1330 /* read back byte-swapped, so complement byte swap bits */ 1331 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS; 1332 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version); 1333 } else { 1334 /* So it wasn't an EIP97 ... maybe it's an EIP197? */ 1335 version = readl(priv->base + EIP197_HIA_AIC_BASE + 1336 EIP197_HIA_VERSION); 1337 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) { 1338 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version); 1339 priv->flags |= SAFEXCEL_HW_EIP197; 1340 } else if (EIP197_REG_HI16(version) == 1341 EIP197_HIA_VERSION_BE) { 1342 /* read back byte-swapped, so complement swap bits */ 1343 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS; 1344 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version); 1345 priv->flags |= SAFEXCEL_HW_EIP197; 1346 } else { 1347 return -ENODEV; 1348 } 1349 } 1350 1351 /* Now initialize the reg offsets based on the probing info so far */ 1352 safexcel_init_register_offsets(priv); 1353 1354 /* 1355 * If the version was read byte-swapped, we need to flip the device 1356 * swapping Keep in mind here, though, that what we write will also be 1357 * byte-swapped ... 1358 */ 1359 if (mask) { 1360 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 1361 val = val ^ (mask >> 24); /* toggle byte swap bits */ 1362 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 1363 } 1364 1365 /* 1366 * We're not done probing yet! We may fall through to here if no HIA 1367 * was found at all. So, with the endianness presumably correct now and 1368 * the offsets setup, *really* probe for the EIP97/EIP197. 1369 */ 1370 version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION); 1371 if (((priv->flags & SAFEXCEL_HW_EIP197) && 1372 (EIP197_REG_LO16(version) != EIP197_VERSION_LE)) || 1373 ((!(priv->flags & SAFEXCEL_HW_EIP197) && 1374 (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) { 1375 /* 1376 * We did not find the device that matched our initial probing 1377 * (or our initial probing failed) Report appropriate error. 1378 */ 1379 return -ENODEV; 1380 } 1381 1382 priv->hwconfig.hwver = EIP197_VERSION_MASK(version); 1383 hwctg = version >> 28; 1384 peid = version & 255; 1385 1386 /* Detect EIP96 packet engine and version */ 1387 version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0)); 1388 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { 1389 dev_err(dev, "EIP%d: EIP96 not detected.\n", peid); 1390 return -ENODEV; 1391 } 1392 priv->hwconfig.pever = EIP197_VERSION_MASK(version); 1393 1394 hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS); 1395 1396 if (priv->flags & SAFEXCEL_HW_EIP197) { 1397 /* EIP197 */ 1398 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & 1399 EIP197_HWDATAW_MASK; 1400 priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) & 1401 EIP197_CFSIZE_MASK) + 1402 EIP197_CFSIZE_ADJUST; 1403 priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) & 1404 EIP197_RFSIZE_MASK) + 1405 EIP197_RFSIZE_ADJUST; 1406 } else { 1407 /* EIP97 */ 1408 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & 1409 EIP97_HWDATAW_MASK; 1410 priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) & 1411 EIP97_CFSIZE_MASK; 1412 priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) & 1413 EIP97_RFSIZE_MASK; 1414 } 1415 1416 /* Get supported algorithms from EIP96 transform engine */ 1417 priv->hwconfig.algo_flags = readl(EIP197_PE(priv) + 1418 EIP197_PE_EIP96_OPTIONS(0)); 1419 1420 /* Print single info line describing what we just detected */ 1421 dev_info(priv->dev, "EIP%d:%x(%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n", 1422 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hiaver, 1423 priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize, 1424 priv->hwconfig.hwrfsize, priv->hwconfig.pever, 1425 priv->hwconfig.algo_flags); 1426 1427 safexcel_configure(priv); 1428 1429 if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) { 1430 /* 1431 * Request MSI vectors for global + 1 per ring - 1432 * or just 1 for older dev images 1433 */ 1434 struct pci_dev *pci_pdev = pdev; 1435 1436 ret = pci_alloc_irq_vectors(pci_pdev, 1437 priv->config.rings + 1, 1438 priv->config.rings + 1, 1439 PCI_IRQ_MSI | PCI_IRQ_MSIX); 1440 if (ret < 0) { 1441 dev_err(dev, "Failed to allocate PCI MSI interrupts\n"); 1442 return ret; 1443 } 1444 } 1445 1446 /* Register the ring IRQ handlers and configure the rings */ 1447 priv->ring = devm_kcalloc(dev, priv->config.rings, 1448 sizeof(*priv->ring), 1449 GFP_KERNEL); 1450 if (!priv->ring) 1451 return -ENOMEM; 1452 1453 for (i = 0; i < priv->config.rings; i++) { 1454 char wq_name[9] = {0}; 1455 int irq; 1456 struct safexcel_ring_irq_data *ring_irq; 1457 1458 ret = safexcel_init_ring_descriptors(priv, 1459 &priv->ring[i].cdr, 1460 &priv->ring[i].rdr); 1461 if (ret) { 1462 dev_err(dev, "Failed to initialize rings\n"); 1463 return ret; 1464 } 1465 1466 priv->ring[i].rdr_req = devm_kcalloc(dev, 1467 EIP197_DEFAULT_RING_SIZE, 1468 sizeof(priv->ring[i].rdr_req), 1469 GFP_KERNEL); 1470 if (!priv->ring[i].rdr_req) 1471 return -ENOMEM; 1472 1473 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); 1474 if (!ring_irq) 1475 return -ENOMEM; 1476 1477 ring_irq->priv = priv; 1478 ring_irq->ring = i; 1479 1480 irq = safexcel_request_ring_irq(pdev, 1481 EIP197_IRQ_NUMBER(i, is_pci_dev), 1482 is_pci_dev, 1483 safexcel_irq_ring, 1484 safexcel_irq_ring_thread, 1485 ring_irq); 1486 if (irq < 0) { 1487 dev_err(dev, "Failed to get IRQ ID for ring %d\n", i); 1488 return irq; 1489 } 1490 1491 priv->ring[i].work_data.priv = priv; 1492 priv->ring[i].work_data.ring = i; 1493 INIT_WORK(&priv->ring[i].work_data.work, 1494 safexcel_dequeue_work); 1495 1496 snprintf(wq_name, 9, "wq_ring%d", i); 1497 priv->ring[i].workqueue = 1498 create_singlethread_workqueue(wq_name); 1499 if (!priv->ring[i].workqueue) 1500 return -ENOMEM; 1501 1502 priv->ring[i].requests = 0; 1503 priv->ring[i].busy = false; 1504 1505 crypto_init_queue(&priv->ring[i].queue, 1506 EIP197_DEFAULT_RING_SIZE); 1507 1508 spin_lock_init(&priv->ring[i].lock); 1509 spin_lock_init(&priv->ring[i].queue_lock); 1510 } 1511 1512 atomic_set(&priv->ring_used, 0); 1513 1514 ret = safexcel_hw_init(priv); 1515 if (ret) { 1516 dev_err(dev, "HW init failed (%d)\n", ret); 1517 return ret; 1518 } 1519 1520 ret = safexcel_register_algorithms(priv); 1521 if (ret) { 1522 dev_err(dev, "Failed to register algorithms (%d)\n", ret); 1523 return ret; 1524 } 1525 1526 return 0; 1527 } 1528 1529 static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv) 1530 { 1531 int i; 1532 1533 for (i = 0; i < priv->config.rings; i++) { 1534 /* clear any pending interrupt */ 1535 writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); 1536 writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); 1537 1538 /* Reset the CDR base address */ 1539 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 1540 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 1541 1542 /* Reset the RDR base address */ 1543 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 1544 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 1545 } 1546 } 1547 1548 #if IS_ENABLED(CONFIG_OF) 1549 /* for Device Tree platform driver */ 1550 1551 static int safexcel_probe(struct platform_device *pdev) 1552 { 1553 struct device *dev = &pdev->dev; 1554 struct safexcel_crypto_priv *priv; 1555 int ret; 1556 1557 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 1558 if (!priv) 1559 return -ENOMEM; 1560 1561 priv->dev = dev; 1562 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev); 1563 1564 platform_set_drvdata(pdev, priv); 1565 1566 priv->base = devm_platform_ioremap_resource(pdev, 0); 1567 if (IS_ERR(priv->base)) { 1568 dev_err(dev, "failed to get resource\n"); 1569 return PTR_ERR(priv->base); 1570 } 1571 1572 priv->clk = devm_clk_get(&pdev->dev, NULL); 1573 ret = PTR_ERR_OR_ZERO(priv->clk); 1574 /* The clock isn't mandatory */ 1575 if (ret != -ENOENT) { 1576 if (ret) 1577 return ret; 1578 1579 ret = clk_prepare_enable(priv->clk); 1580 if (ret) { 1581 dev_err(dev, "unable to enable clk (%d)\n", ret); 1582 return ret; 1583 } 1584 } 1585 1586 priv->reg_clk = devm_clk_get(&pdev->dev, "reg"); 1587 ret = PTR_ERR_OR_ZERO(priv->reg_clk); 1588 /* The clock isn't mandatory */ 1589 if (ret != -ENOENT) { 1590 if (ret) 1591 goto err_core_clk; 1592 1593 ret = clk_prepare_enable(priv->reg_clk); 1594 if (ret) { 1595 dev_err(dev, "unable to enable reg clk (%d)\n", ret); 1596 goto err_core_clk; 1597 } 1598 } 1599 1600 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 1601 if (ret) 1602 goto err_reg_clk; 1603 1604 /* Generic EIP97/EIP197 device probing */ 1605 ret = safexcel_probe_generic(pdev, priv, 0); 1606 if (ret) 1607 goto err_reg_clk; 1608 1609 return 0; 1610 1611 err_reg_clk: 1612 clk_disable_unprepare(priv->reg_clk); 1613 err_core_clk: 1614 clk_disable_unprepare(priv->clk); 1615 return ret; 1616 } 1617 1618 static int safexcel_remove(struct platform_device *pdev) 1619 { 1620 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); 1621 int i; 1622 1623 safexcel_unregister_algorithms(priv); 1624 safexcel_hw_reset_rings(priv); 1625 1626 clk_disable_unprepare(priv->clk); 1627 1628 for (i = 0; i < priv->config.rings; i++) 1629 destroy_workqueue(priv->ring[i].workqueue); 1630 1631 return 0; 1632 } 1633 1634 static const struct of_device_id safexcel_of_match_table[] = { 1635 { 1636 .compatible = "inside-secure,safexcel-eip97ies", 1637 .data = (void *)EIP97IES_MRVL, 1638 }, 1639 { 1640 .compatible = "inside-secure,safexcel-eip197b", 1641 .data = (void *)EIP197B_MRVL, 1642 }, 1643 { 1644 .compatible = "inside-secure,safexcel-eip197d", 1645 .data = (void *)EIP197D_MRVL, 1646 }, 1647 /* For backward compatibility and intended for generic use */ 1648 { 1649 .compatible = "inside-secure,safexcel-eip97", 1650 .data = (void *)EIP97IES_MRVL, 1651 }, 1652 { 1653 .compatible = "inside-secure,safexcel-eip197", 1654 .data = (void *)EIP197B_MRVL, 1655 }, 1656 {}, 1657 }; 1658 1659 static struct platform_driver crypto_safexcel = { 1660 .probe = safexcel_probe, 1661 .remove = safexcel_remove, 1662 .driver = { 1663 .name = "crypto-safexcel", 1664 .of_match_table = safexcel_of_match_table, 1665 }, 1666 }; 1667 #endif 1668 1669 #if IS_ENABLED(CONFIG_PCI) 1670 /* PCIE devices - i.e. Inside Secure development boards */ 1671 1672 static int safexcel_pci_probe(struct pci_dev *pdev, 1673 const struct pci_device_id *ent) 1674 { 1675 struct device *dev = &pdev->dev; 1676 struct safexcel_crypto_priv *priv; 1677 void __iomem *pciebase; 1678 int rc; 1679 u32 val; 1680 1681 dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n", 1682 ent->vendor, ent->device, ent->subvendor, 1683 ent->subdevice, ent->driver_data); 1684 1685 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 1686 if (!priv) 1687 return -ENOMEM; 1688 1689 priv->dev = dev; 1690 priv->version = (enum safexcel_eip_version)ent->driver_data; 1691 1692 pci_set_drvdata(pdev, priv); 1693 1694 /* enable the device */ 1695 rc = pcim_enable_device(pdev); 1696 if (rc) { 1697 dev_err(dev, "Failed to enable PCI device\n"); 1698 return rc; 1699 } 1700 1701 /* take ownership of PCI BAR0 */ 1702 rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel"); 1703 if (rc) { 1704 dev_err(dev, "Failed to map IO region for BAR0\n"); 1705 return rc; 1706 } 1707 priv->base = pcim_iomap_table(pdev)[0]; 1708 1709 if (priv->version == EIP197_DEVBRD) { 1710 dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n"); 1711 1712 rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel"); 1713 if (rc) { 1714 dev_err(dev, "Failed to map IO region for BAR4\n"); 1715 return rc; 1716 } 1717 1718 pciebase = pcim_iomap_table(pdev)[2]; 1719 val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR); 1720 if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) { 1721 dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n", 1722 (val & 0xff)); 1723 1724 /* Setup MSI identity map mapping */ 1725 writel(EIP197_XLX_USER_VECT_LUT0_IDENT, 1726 pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR); 1727 writel(EIP197_XLX_USER_VECT_LUT1_IDENT, 1728 pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR); 1729 writel(EIP197_XLX_USER_VECT_LUT2_IDENT, 1730 pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR); 1731 writel(EIP197_XLX_USER_VECT_LUT3_IDENT, 1732 pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR); 1733 1734 /* Enable all device interrupts */ 1735 writel(GENMASK(31, 0), 1736 pciebase + EIP197_XLX_USER_INT_ENB_MSK); 1737 } else { 1738 dev_err(dev, "Unrecognised IRQ block identifier %x\n", 1739 val); 1740 return -ENODEV; 1741 } 1742 1743 /* HW reset FPGA dev board */ 1744 /* assert reset */ 1745 writel(1, priv->base + EIP197_XLX_GPIO_BASE); 1746 wmb(); /* maintain strict ordering for accesses here */ 1747 /* deassert reset */ 1748 writel(0, priv->base + EIP197_XLX_GPIO_BASE); 1749 wmb(); /* maintain strict ordering for accesses here */ 1750 } 1751 1752 /* enable bus mastering */ 1753 pci_set_master(pdev); 1754 1755 /* Generic EIP97/EIP197 device probing */ 1756 rc = safexcel_probe_generic(pdev, priv, 1); 1757 return rc; 1758 } 1759 1760 void safexcel_pci_remove(struct pci_dev *pdev) 1761 { 1762 struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev); 1763 int i; 1764 1765 safexcel_unregister_algorithms(priv); 1766 1767 for (i = 0; i < priv->config.rings; i++) 1768 destroy_workqueue(priv->ring[i].workqueue); 1769 1770 safexcel_hw_reset_rings(priv); 1771 } 1772 1773 static const struct pci_device_id safexcel_pci_ids[] = { 1774 { 1775 PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038, 1776 0x16ae, 0xc522), 1777 .driver_data = EIP197_DEVBRD, 1778 }, 1779 {}, 1780 }; 1781 1782 MODULE_DEVICE_TABLE(pci, safexcel_pci_ids); 1783 1784 static struct pci_driver safexcel_pci_driver = { 1785 .name = "crypto-safexcel", 1786 .id_table = safexcel_pci_ids, 1787 .probe = safexcel_pci_probe, 1788 .remove = safexcel_pci_remove, 1789 }; 1790 #endif 1791 1792 /* Unfortunately, we have to resort to global variables here */ 1793 #if IS_ENABLED(CONFIG_PCI) 1794 int pcireg_rc = -EINVAL; /* Default safe value */ 1795 #endif 1796 #if IS_ENABLED(CONFIG_OF) 1797 int ofreg_rc = -EINVAL; /* Default safe value */ 1798 #endif 1799 1800 static int __init safexcel_init(void) 1801 { 1802 #if IS_ENABLED(CONFIG_PCI) 1803 /* Register PCI driver */ 1804 pcireg_rc = pci_register_driver(&safexcel_pci_driver); 1805 #endif 1806 1807 #if IS_ENABLED(CONFIG_OF) 1808 /* Register platform driver */ 1809 ofreg_rc = platform_driver_register(&crypto_safexcel); 1810 #if IS_ENABLED(CONFIG_PCI) 1811 /* Return success if either PCI or OF registered OK */ 1812 return pcireg_rc ? ofreg_rc : 0; 1813 #else 1814 return ofreg_rc; 1815 #endif 1816 #else 1817 #if IS_ENABLED(CONFIG_PCI) 1818 return pcireg_rc; 1819 #else 1820 return -EINVAL; 1821 #endif 1822 #endif 1823 } 1824 1825 static void __exit safexcel_exit(void) 1826 { 1827 #if IS_ENABLED(CONFIG_OF) 1828 /* Unregister platform driver */ 1829 if (!ofreg_rc) 1830 platform_driver_unregister(&crypto_safexcel); 1831 #endif 1832 1833 #if IS_ENABLED(CONFIG_PCI) 1834 /* Unregister PCI driver if successfully registered before */ 1835 if (!pcireg_rc) 1836 pci_unregister_driver(&safexcel_pci_driver); 1837 #endif 1838 } 1839 1840 module_init(safexcel_init); 1841 module_exit(safexcel_exit); 1842 1843 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>"); 1844 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>"); 1845 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>"); 1846 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197"); 1847 MODULE_LICENSE("GPL v2"); 1848