xref: /linux/drivers/crypto/inside-secure/safexcel.c (revision 78beef629fd95be4ed853b2d37b832f766bd96ca)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2017 Marvell
4  *
5  * Antoine Tenart <antoine.tenart@free-electrons.com>
6  */
7 
8 #include <linux/clk.h>
9 #include <linux/device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/firmware.h>
13 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/of_platform.h>
16 #include <linux/of_irq.h>
17 #include <linux/pci.h>
18 #include <linux/platform_device.h>
19 #include <linux/workqueue.h>
20 
21 #include <crypto/internal/aead.h>
22 #include <crypto/internal/hash.h>
23 #include <crypto/internal/skcipher.h>
24 
25 #include "safexcel.h"
26 
27 static u32 max_rings = EIP197_MAX_RINGS;
28 module_param(max_rings, uint, 0644);
29 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
30 
31 static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
32 {
33 	int i;
34 
35 	/*
36 	 * Map all interfaces/rings to register index 0
37 	 * so they can share contexts. Without this, the EIP197 will
38 	 * assume each interface/ring to be in its own memory domain
39 	 * i.e. have its own subset of UNIQUE memory addresses.
40 	 * Which would cause records with the SAME memory address to
41 	 * use DIFFERENT cache buffers, causing both poor cache utilization
42 	 * AND serious coherence/invalidation issues.
43 	 */
44 	for (i = 0; i < 4; i++)
45 		writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
46 
47 	/*
48 	 * Initialize other virtualization regs for cache
49 	 * These may not be in their reset state ...
50 	 */
51 	for (i = 0; i < priv->config.rings; i++) {
52 		writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
53 		writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
54 		writel(EIP197_FLUE_CONFIG_MAGIC,
55 		       priv->base + EIP197_FLUE_CONFIG(i));
56 	}
57 	writel(0, priv->base + EIP197_FLUE_OFFSETS);
58 	writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
59 }
60 
61 static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
62 				     u32 addrmid, int *actbank)
63 {
64 	u32 val;
65 	int curbank;
66 
67 	curbank = addrmid >> 16;
68 	if (curbank != *actbank) {
69 		val = readl(priv->base + EIP197_CS_RAM_CTRL);
70 		val = (val & ~EIP197_CS_BANKSEL_MASK) |
71 		      (curbank << EIP197_CS_BANKSEL_OFS);
72 		writel(val, priv->base + EIP197_CS_RAM_CTRL);
73 		*actbank = curbank;
74 	}
75 }
76 
77 static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
78 				  int maxbanks, u32 probemask)
79 {
80 	u32 val, addrhi, addrlo, addrmid;
81 	int actbank;
82 
83 	/*
84 	 * And probe the actual size of the physically attached cache data RAM
85 	 * Using a binary subdivision algorithm downto 32 byte cache lines.
86 	 */
87 	addrhi = 1 << (16 + maxbanks);
88 	addrlo = 0;
89 	actbank = min(maxbanks - 1, 0);
90 	while ((addrhi - addrlo) > 32) {
91 		/* write marker to lowest address in top half */
92 		addrmid = (addrhi + addrlo) >> 1;
93 		eip197_trc_cache_banksel(priv, addrmid, &actbank);
94 		writel((addrmid | (addrlo << 16)) & probemask,
95 			priv->base + EIP197_CLASSIFICATION_RAMS +
96 			(addrmid & 0xffff));
97 
98 		/* write marker to lowest address in bottom half */
99 		eip197_trc_cache_banksel(priv, addrlo, &actbank);
100 		writel((addrlo | (addrhi << 16)) & probemask,
101 			priv->base + EIP197_CLASSIFICATION_RAMS +
102 			(addrlo & 0xffff));
103 
104 		/* read back marker from top half */
105 		eip197_trc_cache_banksel(priv, addrmid, &actbank);
106 		val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
107 			    (addrmid & 0xffff));
108 
109 		if (val == ((addrmid | (addrlo << 16)) & probemask)) {
110 			/* read back correct, continue with top half */
111 			addrlo = addrmid;
112 		} else {
113 			/* not read back correct, continue with bottom half */
114 			addrhi = addrmid;
115 		}
116 	}
117 	return addrhi;
118 }
119 
120 static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
121 				   int cs_rc_max, int cs_ht_wc)
122 {
123 	int i;
124 	u32 htable_offset, val, offset;
125 
126 	/* Clear all records in administration RAM */
127 	for (i = 0; i < cs_rc_max; i++) {
128 		offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
129 
130 		writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
131 		       EIP197_CS_RC_PREV(EIP197_RC_NULL),
132 		       priv->base + offset);
133 
134 		val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
135 		if (i == 0)
136 			val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
137 		else if (i == cs_rc_max - 1)
138 			val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
139 		writel(val, priv->base + offset + 4);
140 		/* must also initialize the address key due to ECC! */
141 		writel(0, priv->base + offset + 8);
142 		writel(0, priv->base + offset + 12);
143 	}
144 
145 	/* Clear the hash table entries */
146 	htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
147 	for (i = 0; i < cs_ht_wc; i++)
148 		writel(GENMASK(29, 0),
149 		       priv->base + EIP197_CLASSIFICATION_RAMS +
150 		       htable_offset + i * sizeof(u32));
151 }
152 
153 static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
154 {
155 	u32 val, dsize, asize;
156 	int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
157 	int cs_rc_abs_max, cs_ht_sz;
158 	int maxbanks;
159 
160 	/* Setup (dummy) virtualization for cache */
161 	eip197_trc_cache_setupvirt(priv);
162 
163 	/*
164 	 * Enable the record cache memory access and
165 	 * probe the bank select width
166 	 */
167 	val = readl(priv->base + EIP197_CS_RAM_CTRL);
168 	val &= ~EIP197_TRC_ENABLE_MASK;
169 	val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
170 	writel(val, priv->base + EIP197_CS_RAM_CTRL);
171 	val = readl(priv->base + EIP197_CS_RAM_CTRL);
172 	maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
173 
174 	/* Clear all ECC errors */
175 	writel(0, priv->base + EIP197_TRC_ECCCTRL);
176 
177 	/*
178 	 * Make sure the cache memory is accessible by taking record cache into
179 	 * reset. Need data memory access here, not admin access.
180 	 */
181 	val = readl(priv->base + EIP197_TRC_PARAMS);
182 	val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
183 	writel(val, priv->base + EIP197_TRC_PARAMS);
184 
185 	/* Probed data RAM size in bytes */
186 	dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff);
187 
188 	/*
189 	 * Now probe the administration RAM size pretty much the same way
190 	 * Except that only the lower 30 bits are writable and we don't need
191 	 * bank selects
192 	 */
193 	val = readl(priv->base + EIP197_TRC_PARAMS);
194 	/* admin access now */
195 	val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
196 	writel(val, priv->base + EIP197_TRC_PARAMS);
197 
198 	/* Probed admin RAM size in admin words */
199 	asize = eip197_trc_cache_probe(priv, 0, 0xbfffffff) >> 4;
200 
201 	/* Clear any ECC errors detected while probing! */
202 	writel(0, priv->base + EIP197_TRC_ECCCTRL);
203 
204 	/*
205 	 * Determine optimal configuration from RAM sizes
206 	 * Note that we assume that the physical RAM configuration is sane
207 	 * Therefore, we don't do any parameter error checking here ...
208 	 */
209 
210 	/* For now, just use a single record format covering everything */
211 	cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
212 	cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
213 
214 	/*
215 	 * Step #1: How many records will physically fit?
216 	 * Hard upper limit is 1023!
217 	 */
218 	cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
219 	/* Step #2: Need at least 2 words in the admin RAM per record */
220 	cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
221 	/* Step #3: Determine log2 of hash table size */
222 	cs_ht_sz = __fls(asize - cs_rc_max) - 2;
223 	/* Step #4: determine current size of hash table in dwords */
224 	cs_ht_wc = 16<<cs_ht_sz; /* dwords, not admin words */
225 	/* Step #5: add back excess words and see if we can fit more records */
226 	cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 4));
227 
228 	/* Clear the cache RAMs */
229 	eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
230 
231 	/* Disable the record cache memory access */
232 	val = readl(priv->base + EIP197_CS_RAM_CTRL);
233 	val &= ~EIP197_TRC_ENABLE_MASK;
234 	writel(val, priv->base + EIP197_CS_RAM_CTRL);
235 
236 	/* Write head and tail pointers of the record free chain */
237 	val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
238 	      EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
239 	writel(val, priv->base + EIP197_TRC_FREECHAIN);
240 
241 	/* Configure the record cache #1 */
242 	val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
243 	      EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
244 	writel(val, priv->base + EIP197_TRC_PARAMS2);
245 
246 	/* Configure the record cache #2 */
247 	val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
248 	      EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
249 	      EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
250 	writel(val, priv->base + EIP197_TRC_PARAMS);
251 
252 	dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
253 		 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
254 }
255 
256 static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
257 {
258 	int pe, i;
259 	u32 val;
260 
261 	for (pe = 0; pe < priv->config.pes; pe++) {
262 		/* Configure the token FIFO's */
263 		writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
264 		writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
265 
266 		/* Clear the ICE scratchpad memory */
267 		val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
268 		val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
269 		       EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
270 		       EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
271 		       EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
272 		writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
273 
274 		/* clear the scratchpad RAM using 32 bit writes only */
275 		for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
276 			writel(0, EIP197_PE(priv) +
277 				  EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
278 
279 		/* Reset the IFPP engine to make its program mem accessible */
280 		writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
281 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
282 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
283 		       EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
284 
285 		/* Reset the IPUE engine to make its program mem accessible */
286 		writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
287 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
288 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
289 		       EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
290 
291 		/* Enable access to all IFPP program memories */
292 		writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
293 		       EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
294 	}
295 
296 }
297 
298 static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
299 				  const struct firmware *fw)
300 {
301 	const u32 *data = (const u32 *)fw->data;
302 	int i;
303 
304 	/* Write the firmware */
305 	for (i = 0; i < fw->size / sizeof(u32); i++)
306 		writel(be32_to_cpu(data[i]),
307 		       priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
308 
309 	/* Exclude final 2 NOPs from size */
310 	return i - EIP197_FW_TERMINAL_NOPS;
311 }
312 
313 /*
314  * If FW is actual production firmware, then poll for its initialization
315  * to complete and check if it is good for the HW, otherwise just return OK.
316  */
317 static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
318 {
319 	int pe, pollcnt;
320 	u32 base, pollofs;
321 
322 	if (fpp)
323 		pollofs  = EIP197_FW_FPP_READY;
324 	else
325 		pollofs  = EIP197_FW_PUE_READY;
326 
327 	for (pe = 0; pe < priv->config.pes; pe++) {
328 		base = EIP197_PE_ICE_SCRATCH_RAM(pe);
329 		pollcnt = EIP197_FW_START_POLLCNT;
330 		while (pollcnt &&
331 		       (readl_relaxed(EIP197_PE(priv) + base +
332 			      pollofs) != 1)) {
333 			pollcnt--;
334 		}
335 		if (!pollcnt) {
336 			dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
337 				fpp, pe);
338 			return false;
339 		}
340 	}
341 	return true;
342 }
343 
344 static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
345 				  int ipuesz, int ifppsz, int minifw)
346 {
347 	int pe;
348 	u32 val;
349 
350 	for (pe = 0; pe < priv->config.pes; pe++) {
351 		/* Disable access to all program memory */
352 		writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
353 
354 		/* Start IFPP microengines */
355 		if (minifw)
356 			val = 0;
357 		else
358 			val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
359 					EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
360 				EIP197_PE_ICE_UENG_DEBUG_RESET;
361 		writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
362 
363 		/* Start IPUE microengines */
364 		if (minifw)
365 			val = 0;
366 		else
367 			val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
368 					EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
369 				EIP197_PE_ICE_UENG_DEBUG_RESET;
370 		writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
371 	}
372 
373 	/* For miniFW startup, there is no initialization, so always succeed */
374 	if (minifw)
375 		return true;
376 
377 	/* Wait until all the firmwares have properly started up */
378 	if (!poll_fw_ready(priv, 1))
379 		return false;
380 	if (!poll_fw_ready(priv, 0))
381 		return false;
382 
383 	return true;
384 }
385 
386 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
387 {
388 	const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
389 	const struct firmware *fw[FW_NB];
390 	char fw_path[37], *dir = NULL;
391 	int i, j, ret = 0, pe;
392 	int ipuesz, ifppsz, minifw = 0;
393 
394 	if (priv->version == EIP197D_MRVL)
395 		dir = "eip197d";
396 	else if (priv->version == EIP197B_MRVL ||
397 		 priv->version == EIP197_DEVBRD)
398 		dir = "eip197b";
399 	else
400 		return -ENODEV;
401 
402 retry_fw:
403 	for (i = 0; i < FW_NB; i++) {
404 		snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
405 		ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
406 		if (ret) {
407 			if (minifw || priv->version != EIP197B_MRVL)
408 				goto release_fw;
409 
410 			/* Fallback to the old firmware location for the
411 			 * EIP197b.
412 			 */
413 			ret = firmware_request_nowarn(&fw[i], fw_name[i],
414 						      priv->dev);
415 			if (ret)
416 				goto release_fw;
417 		}
418 	}
419 
420 	eip197_init_firmware(priv);
421 
422 	ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
423 
424 	/* Enable access to IPUE program memories */
425 	for (pe = 0; pe < priv->config.pes; pe++)
426 		writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
427 		       EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
428 
429 	ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
430 
431 	if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
432 		dev_dbg(priv->dev, "Firmware loaded successfully\n");
433 		return 0;
434 	}
435 
436 	ret = -ENODEV;
437 
438 release_fw:
439 	for (j = 0; j < i; j++)
440 		release_firmware(fw[j]);
441 
442 	if (!minifw) {
443 		/* Retry with minifw path */
444 		dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
445 		dir = "eip197_minifw";
446 		minifw = 1;
447 		goto retry_fw;
448 	}
449 
450 	dev_dbg(priv->dev, "Firmware load failed.\n");
451 
452 	return ret;
453 }
454 
455 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
456 {
457 	u32 cd_size_rnd, val;
458 	int i, cd_fetch_cnt;
459 
460 	cd_size_rnd  = (priv->config.cd_size +
461 			(BIT(priv->hwconfig.hwdataw) - 1)) >>
462 		       priv->hwconfig.hwdataw;
463 	/* determine number of CD's we can fetch into the CD FIFO as 1 block */
464 	if (priv->flags & SAFEXCEL_HW_EIP197) {
465 		/* EIP197: try to fetch enough in 1 go to keep all pipes busy */
466 		cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
467 		cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
468 				     (priv->config.pes * EIP197_FETCH_DEPTH));
469 	} else {
470 		/* for the EIP97, just fetch all that fits minus 1 */
471 		cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
472 				cd_size_rnd) - 1;
473 	}
474 
475 	for (i = 0; i < priv->config.rings; i++) {
476 		/* ring base address */
477 		writel(lower_32_bits(priv->ring[i].cdr.base_dma),
478 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
479 		writel(upper_32_bits(priv->ring[i].cdr.base_dma),
480 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
481 
482 		writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
483 		       priv->config.cd_size,
484 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
485 		writel(((cd_fetch_cnt *
486 			 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
487 		       (cd_fetch_cnt * priv->config.cd_offset),
488 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
489 
490 		/* Configure DMA tx control */
491 		val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
492 		val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
493 		writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
494 
495 		/* clear any pending interrupt */
496 		writel(GENMASK(5, 0),
497 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
498 	}
499 
500 	return 0;
501 }
502 
503 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
504 {
505 	u32 rd_size_rnd, val;
506 	int i, rd_fetch_cnt;
507 
508 	/* determine number of RD's we can fetch into the FIFO as one block */
509 	rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
510 		       (BIT(priv->hwconfig.hwdataw) - 1)) >>
511 		      priv->hwconfig.hwdataw;
512 	if (priv->flags & SAFEXCEL_HW_EIP197) {
513 		/* EIP197: try to fetch enough in 1 go to keep all pipes busy */
514 		rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
515 		rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
516 				     (priv->config.pes * EIP197_FETCH_DEPTH));
517 	} else {
518 		/* for the EIP97, just fetch all that fits minus 1 */
519 		rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
520 				rd_size_rnd) - 1;
521 	}
522 
523 	for (i = 0; i < priv->config.rings; i++) {
524 		/* ring base address */
525 		writel(lower_32_bits(priv->ring[i].rdr.base_dma),
526 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
527 		writel(upper_32_bits(priv->ring[i].rdr.base_dma),
528 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
529 
530 		writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
531 		       priv->config.rd_size,
532 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
533 
534 		writel(((rd_fetch_cnt *
535 			 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
536 		       (rd_fetch_cnt * priv->config.rd_offset),
537 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
538 
539 		/* Configure DMA tx control */
540 		val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
541 		val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
542 		val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
543 		writel(val,
544 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
545 
546 		/* clear any pending interrupt */
547 		writel(GENMASK(7, 0),
548 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
549 
550 		/* enable ring interrupt */
551 		val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
552 		val |= EIP197_RDR_IRQ(i);
553 		writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
554 	}
555 
556 	return 0;
557 }
558 
559 static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
560 {
561 	u32 val;
562 	int i, ret, pe;
563 
564 	dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
565 		priv->config.pes, priv->config.rings);
566 
567 	/*
568 	 * For EIP197's only set maximum number of TX commands to 2^5 = 32
569 	 * Skip for the EIP97 as it does not have this field.
570 	 */
571 	if (priv->flags & SAFEXCEL_HW_EIP197) {
572 		val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
573 		val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
574 		writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
575 	}
576 
577 	/* Configure wr/rd cache values */
578 	writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
579 	       EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
580 	       EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
581 
582 	/* Interrupts reset */
583 
584 	/* Disable all global interrupts */
585 	writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
586 
587 	/* Clear any pending interrupt */
588 	writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
589 
590 	/* Processing Engine configuration */
591 	for (pe = 0; pe < priv->config.pes; pe++) {
592 		/* Data Fetch Engine configuration */
593 
594 		/* Reset all DFE threads */
595 		writel(EIP197_DxE_THR_CTRL_RESET_PE,
596 		       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
597 
598 		if (priv->flags & SAFEXCEL_HW_EIP197)
599 			/* Reset HIA input interface arbiter (EIP197 only) */
600 			writel(EIP197_HIA_RA_PE_CTRL_RESET,
601 			       EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
602 
603 		/* DMA transfer size to use */
604 		val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
605 		val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
606 		       EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
607 		val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
608 		       EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
609 		val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
610 		val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
611 		writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
612 
613 		/* Leave the DFE threads reset state */
614 		writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
615 
616 		/* Configure the processing engine thresholds */
617 		writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
618 		       EIP197_PE_IN_xBUF_THRES_MAX(9),
619 		       EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
620 		writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
621 		       EIP197_PE_IN_xBUF_THRES_MAX(7),
622 		       EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
623 
624 		if (priv->flags & SAFEXCEL_HW_EIP197)
625 			/* enable HIA input interface arbiter and rings */
626 			writel(EIP197_HIA_RA_PE_CTRL_EN |
627 			       GENMASK(priv->config.rings - 1, 0),
628 			       EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
629 
630 		/* Data Store Engine configuration */
631 
632 		/* Reset all DSE threads */
633 		writel(EIP197_DxE_THR_CTRL_RESET_PE,
634 		       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
635 
636 		/* Wait for all DSE threads to complete */
637 		while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
638 			GENMASK(15, 12)) != GENMASK(15, 12))
639 			;
640 
641 		/* DMA transfer size to use */
642 		val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
643 		val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) |
644 		       EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
645 		val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
646 		val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
647 		/* FIXME: instability issues can occur for EIP97 but disabling
648 		 * it impacts performance.
649 		 */
650 		if (priv->flags & SAFEXCEL_HW_EIP197)
651 			val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
652 		writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
653 
654 		/* Leave the DSE threads reset state */
655 		writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
656 
657 		/* Configure the procesing engine thresholds */
658 		writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) |
659 		       EIP197_PE_OUT_DBUF_THRES_MAX(8),
660 		       EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
661 
662 		/* Processing Engine configuration */
663 
664 		/* Token & context configuration */
665 		val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
666 		      EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
667 		      EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
668 		writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
669 
670 		/* H/W capabilities selection: just enable everything */
671 		writel(EIP197_FUNCTION_ALL,
672 		       EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
673 		writel(EIP197_FUNCTION_ALL,
674 		       EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
675 	}
676 
677 	/* Command Descriptor Rings prepare */
678 	for (i = 0; i < priv->config.rings; i++) {
679 		/* Clear interrupts for this ring */
680 		writel(GENMASK(31, 0),
681 		       EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
682 
683 		/* Disable external triggering */
684 		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
685 
686 		/* Clear the pending prepared counter */
687 		writel(EIP197_xDR_PREP_CLR_COUNT,
688 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
689 
690 		/* Clear the pending processed counter */
691 		writel(EIP197_xDR_PROC_CLR_COUNT,
692 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
693 
694 		writel(0,
695 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
696 		writel(0,
697 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
698 
699 		writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
700 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
701 	}
702 
703 	/* Result Descriptor Ring prepare */
704 	for (i = 0; i < priv->config.rings; i++) {
705 		/* Disable external triggering*/
706 		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
707 
708 		/* Clear the pending prepared counter */
709 		writel(EIP197_xDR_PREP_CLR_COUNT,
710 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
711 
712 		/* Clear the pending processed counter */
713 		writel(EIP197_xDR_PROC_CLR_COUNT,
714 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
715 
716 		writel(0,
717 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
718 		writel(0,
719 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
720 
721 		/* Ring size */
722 		writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
723 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
724 	}
725 
726 	for (pe = 0; pe < priv->config.pes; pe++) {
727 		/* Enable command descriptor rings */
728 		writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
729 		       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
730 
731 		/* Enable result descriptor rings */
732 		writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
733 		       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
734 	}
735 
736 	/* Clear any HIA interrupt */
737 	writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
738 
739 	if (priv->flags & SAFEXCEL_HW_EIP197) {
740 		eip197_trc_cache_init(priv);
741 		priv->flags |= EIP197_TRC_CACHE;
742 
743 		ret = eip197_load_firmwares(priv);
744 		if (ret)
745 			return ret;
746 	}
747 
748 	safexcel_hw_setup_cdesc_rings(priv);
749 	safexcel_hw_setup_rdesc_rings(priv);
750 
751 	return 0;
752 }
753 
754 /* Called with ring's lock taken */
755 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
756 				       int ring)
757 {
758 	int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
759 
760 	if (!coal)
761 		return;
762 
763 	/* Configure when we want an interrupt */
764 	writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
765 	       EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
766 	       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
767 }
768 
769 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
770 {
771 	struct crypto_async_request *req, *backlog;
772 	struct safexcel_context *ctx;
773 	int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
774 
775 	/* If a request wasn't properly dequeued because of a lack of resources,
776 	 * proceeded it first,
777 	 */
778 	req = priv->ring[ring].req;
779 	backlog = priv->ring[ring].backlog;
780 	if (req)
781 		goto handle_req;
782 
783 	while (true) {
784 		spin_lock_bh(&priv->ring[ring].queue_lock);
785 		backlog = crypto_get_backlog(&priv->ring[ring].queue);
786 		req = crypto_dequeue_request(&priv->ring[ring].queue);
787 		spin_unlock_bh(&priv->ring[ring].queue_lock);
788 
789 		if (!req) {
790 			priv->ring[ring].req = NULL;
791 			priv->ring[ring].backlog = NULL;
792 			goto finalize;
793 		}
794 
795 handle_req:
796 		ctx = crypto_tfm_ctx(req->tfm);
797 		ret = ctx->send(req, ring, &commands, &results);
798 		if (ret)
799 			goto request_failed;
800 
801 		if (backlog)
802 			backlog->complete(backlog, -EINPROGRESS);
803 
804 		/* In case the send() helper did not issue any command to push
805 		 * to the engine because the input data was cached, continue to
806 		 * dequeue other requests as this is valid and not an error.
807 		 */
808 		if (!commands && !results)
809 			continue;
810 
811 		cdesc += commands;
812 		rdesc += results;
813 		nreq++;
814 	}
815 
816 request_failed:
817 	/* Not enough resources to handle all the requests. Bail out and save
818 	 * the request and the backlog for the next dequeue call (per-ring).
819 	 */
820 	priv->ring[ring].req = req;
821 	priv->ring[ring].backlog = backlog;
822 
823 finalize:
824 	if (!nreq)
825 		return;
826 
827 	spin_lock_bh(&priv->ring[ring].lock);
828 
829 	priv->ring[ring].requests += nreq;
830 
831 	if (!priv->ring[ring].busy) {
832 		safexcel_try_push_requests(priv, ring);
833 		priv->ring[ring].busy = true;
834 	}
835 
836 	spin_unlock_bh(&priv->ring[ring].lock);
837 
838 	/* let the RDR know we have pending descriptors */
839 	writel((rdesc * priv->config.rd_offset) << 2,
840 	       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
841 
842 	/* let the CDR know we have pending descriptors */
843 	writel((cdesc * priv->config.cd_offset) << 2,
844 	       EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
845 }
846 
847 inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
848 				       struct safexcel_result_desc *rdesc)
849 {
850 	if (likely((!rdesc->descriptor_overflow) &&
851 		   (!rdesc->buffer_overflow) &&
852 		   (!rdesc->result_data.error_code)))
853 		return 0;
854 
855 	if (rdesc->descriptor_overflow)
856 		dev_err(priv->dev, "Descriptor overflow detected");
857 
858 	if (rdesc->buffer_overflow)
859 		dev_err(priv->dev, "Buffer overflow detected");
860 
861 	if (rdesc->result_data.error_code & 0x4066) {
862 		/* Fatal error (bits 1,2,5,6 & 14) */
863 		dev_err(priv->dev,
864 			"result descriptor error (%x)",
865 			rdesc->result_data.error_code);
866 		return -EIO;
867 	} else if (rdesc->result_data.error_code &
868 		   (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
869 		/*
870 		 * Give priority over authentication fails:
871 		 * Blocksize, length & overflow errors,
872 		 * something wrong with the input!
873 		 */
874 		return -EINVAL;
875 	} else if (rdesc->result_data.error_code & BIT(9)) {
876 		/* Authentication failed */
877 		return -EBADMSG;
878 	}
879 
880 	/* All other non-fatal errors */
881 	return -EINVAL;
882 }
883 
884 inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
885 				 int ring,
886 				 struct safexcel_result_desc *rdesc,
887 				 struct crypto_async_request *req)
888 {
889 	int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
890 
891 	priv->ring[ring].rdr_req[i] = req;
892 }
893 
894 inline struct crypto_async_request *
895 safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
896 {
897 	int i = safexcel_ring_first_rdr_index(priv, ring);
898 
899 	return priv->ring[ring].rdr_req[i];
900 }
901 
902 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
903 {
904 	struct safexcel_command_desc *cdesc;
905 
906 	/* Acknowledge the command descriptors */
907 	do {
908 		cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
909 		if (IS_ERR(cdesc)) {
910 			dev_err(priv->dev,
911 				"Could not retrieve the command descriptor\n");
912 			return;
913 		}
914 	} while (!cdesc->last_seg);
915 }
916 
917 void safexcel_inv_complete(struct crypto_async_request *req, int error)
918 {
919 	struct safexcel_inv_result *result = req->data;
920 
921 	if (error == -EINPROGRESS)
922 		return;
923 
924 	result->error = error;
925 	complete(&result->completion);
926 }
927 
928 int safexcel_invalidate_cache(struct crypto_async_request *async,
929 			      struct safexcel_crypto_priv *priv,
930 			      dma_addr_t ctxr_dma, int ring)
931 {
932 	struct safexcel_command_desc *cdesc;
933 	struct safexcel_result_desc *rdesc;
934 	int ret = 0;
935 
936 	/* Prepare command descriptor */
937 	cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
938 	if (IS_ERR(cdesc))
939 		return PTR_ERR(cdesc);
940 
941 	cdesc->control_data.type = EIP197_TYPE_EXTENDED;
942 	cdesc->control_data.options = 0;
943 	cdesc->control_data.refresh = 0;
944 	cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
945 
946 	/* Prepare result descriptor */
947 	rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
948 
949 	if (IS_ERR(rdesc)) {
950 		ret = PTR_ERR(rdesc);
951 		goto cdesc_rollback;
952 	}
953 
954 	safexcel_rdr_req_set(priv, ring, rdesc, async);
955 
956 	return ret;
957 
958 cdesc_rollback:
959 	safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
960 
961 	return ret;
962 }
963 
964 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
965 						     int ring)
966 {
967 	struct crypto_async_request *req;
968 	struct safexcel_context *ctx;
969 	int ret, i, nreq, ndesc, tot_descs, handled = 0;
970 	bool should_complete;
971 
972 handle_results:
973 	tot_descs = 0;
974 
975 	nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
976 	nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
977 	nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
978 	if (!nreq)
979 		goto requests_left;
980 
981 	for (i = 0; i < nreq; i++) {
982 		req = safexcel_rdr_req_get(priv, ring);
983 
984 		ctx = crypto_tfm_ctx(req->tfm);
985 		ndesc = ctx->handle_result(priv, ring, req,
986 					   &should_complete, &ret);
987 		if (ndesc < 0) {
988 			dev_err(priv->dev, "failed to handle result (%d)\n",
989 				ndesc);
990 			goto acknowledge;
991 		}
992 
993 		if (should_complete) {
994 			local_bh_disable();
995 			req->complete(req, ret);
996 			local_bh_enable();
997 		}
998 
999 		tot_descs += ndesc;
1000 		handled++;
1001 	}
1002 
1003 acknowledge:
1004 	if (i)
1005 		writel(EIP197_xDR_PROC_xD_PKT(i) |
1006 		       EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
1007 		       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1008 
1009 	/* If the number of requests overflowed the counter, try to proceed more
1010 	 * requests.
1011 	 */
1012 	if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
1013 		goto handle_results;
1014 
1015 requests_left:
1016 	spin_lock_bh(&priv->ring[ring].lock);
1017 
1018 	priv->ring[ring].requests -= handled;
1019 	safexcel_try_push_requests(priv, ring);
1020 
1021 	if (!priv->ring[ring].requests)
1022 		priv->ring[ring].busy = false;
1023 
1024 	spin_unlock_bh(&priv->ring[ring].lock);
1025 }
1026 
1027 static void safexcel_dequeue_work(struct work_struct *work)
1028 {
1029 	struct safexcel_work_data *data =
1030 			container_of(work, struct safexcel_work_data, work);
1031 
1032 	safexcel_dequeue(data->priv, data->ring);
1033 }
1034 
1035 struct safexcel_ring_irq_data {
1036 	struct safexcel_crypto_priv *priv;
1037 	int ring;
1038 };
1039 
1040 static irqreturn_t safexcel_irq_ring(int irq, void *data)
1041 {
1042 	struct safexcel_ring_irq_data *irq_data = data;
1043 	struct safexcel_crypto_priv *priv = irq_data->priv;
1044 	int ring = irq_data->ring, rc = IRQ_NONE;
1045 	u32 status, stat;
1046 
1047 	status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
1048 	if (!status)
1049 		return rc;
1050 
1051 	/* RDR interrupts */
1052 	if (status & EIP197_RDR_IRQ(ring)) {
1053 		stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1054 
1055 		if (unlikely(stat & EIP197_xDR_ERR)) {
1056 			/*
1057 			 * Fatal error, the RDR is unusable and must be
1058 			 * reinitialized. This should not happen under
1059 			 * normal circumstances.
1060 			 */
1061 			dev_err(priv->dev, "RDR: fatal error.\n");
1062 		} else if (likely(stat & EIP197_xDR_THRESH)) {
1063 			rc = IRQ_WAKE_THREAD;
1064 		}
1065 
1066 		/* ACK the interrupts */
1067 		writel(stat & 0xff,
1068 		       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1069 	}
1070 
1071 	/* ACK the interrupts */
1072 	writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
1073 
1074 	return rc;
1075 }
1076 
1077 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
1078 {
1079 	struct safexcel_ring_irq_data *irq_data = data;
1080 	struct safexcel_crypto_priv *priv = irq_data->priv;
1081 	int ring = irq_data->ring;
1082 
1083 	safexcel_handle_result_descriptor(priv, ring);
1084 
1085 	queue_work(priv->ring[ring].workqueue,
1086 		   &priv->ring[ring].work_data.work);
1087 
1088 	return IRQ_HANDLED;
1089 }
1090 
1091 static int safexcel_request_ring_irq(void *pdev, int irqid,
1092 				     int is_pci_dev,
1093 				     irq_handler_t handler,
1094 				     irq_handler_t threaded_handler,
1095 				     struct safexcel_ring_irq_data *ring_irq_priv)
1096 {
1097 	int ret, irq;
1098 	struct device *dev;
1099 
1100 	if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
1101 		struct pci_dev *pci_pdev = pdev;
1102 
1103 		dev = &pci_pdev->dev;
1104 		irq = pci_irq_vector(pci_pdev, irqid);
1105 		if (irq < 0) {
1106 			dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
1107 				irqid, irq);
1108 			return irq;
1109 		}
1110 	} else if (IS_ENABLED(CONFIG_OF)) {
1111 		struct platform_device *plf_pdev = pdev;
1112 		char irq_name[6] = {0}; /* "ringX\0" */
1113 
1114 		snprintf(irq_name, 6, "ring%d", irqid);
1115 		dev = &plf_pdev->dev;
1116 		irq = platform_get_irq_byname(plf_pdev, irq_name);
1117 
1118 		if (irq < 0) {
1119 			dev_err(dev, "unable to get IRQ '%s' (err %d)\n",
1120 				irq_name, irq);
1121 			return irq;
1122 		}
1123 	}
1124 
1125 	ret = devm_request_threaded_irq(dev, irq, handler,
1126 					threaded_handler, IRQF_ONESHOT,
1127 					dev_name(dev), ring_irq_priv);
1128 	if (ret) {
1129 		dev_err(dev, "unable to request IRQ %d\n", irq);
1130 		return ret;
1131 	}
1132 
1133 	return irq;
1134 }
1135 
1136 static struct safexcel_alg_template *safexcel_algs[] = {
1137 	&safexcel_alg_ecb_des,
1138 	&safexcel_alg_cbc_des,
1139 	&safexcel_alg_ecb_des3_ede,
1140 	&safexcel_alg_cbc_des3_ede,
1141 	&safexcel_alg_ecb_aes,
1142 	&safexcel_alg_cbc_aes,
1143 	&safexcel_alg_cfb_aes,
1144 	&safexcel_alg_ofb_aes,
1145 	&safexcel_alg_ctr_aes,
1146 	&safexcel_alg_md5,
1147 	&safexcel_alg_sha1,
1148 	&safexcel_alg_sha224,
1149 	&safexcel_alg_sha256,
1150 	&safexcel_alg_sha384,
1151 	&safexcel_alg_sha512,
1152 	&safexcel_alg_hmac_md5,
1153 	&safexcel_alg_hmac_sha1,
1154 	&safexcel_alg_hmac_sha224,
1155 	&safexcel_alg_hmac_sha256,
1156 	&safexcel_alg_hmac_sha384,
1157 	&safexcel_alg_hmac_sha512,
1158 	&safexcel_alg_authenc_hmac_sha1_cbc_aes,
1159 	&safexcel_alg_authenc_hmac_sha224_cbc_aes,
1160 	&safexcel_alg_authenc_hmac_sha256_cbc_aes,
1161 	&safexcel_alg_authenc_hmac_sha384_cbc_aes,
1162 	&safexcel_alg_authenc_hmac_sha512_cbc_aes,
1163 	&safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
1164 	&safexcel_alg_authenc_hmac_sha1_ctr_aes,
1165 	&safexcel_alg_authenc_hmac_sha224_ctr_aes,
1166 	&safexcel_alg_authenc_hmac_sha256_ctr_aes,
1167 	&safexcel_alg_authenc_hmac_sha384_ctr_aes,
1168 	&safexcel_alg_authenc_hmac_sha512_ctr_aes,
1169 	&safexcel_alg_xts_aes,
1170 	&safexcel_alg_gcm,
1171 	&safexcel_alg_ccm,
1172 };
1173 
1174 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
1175 {
1176 	int i, j, ret = 0;
1177 
1178 	for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1179 		safexcel_algs[i]->priv = priv;
1180 
1181 		/* Do we have all required base algorithms available? */
1182 		if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1183 		    safexcel_algs[i]->algo_mask)
1184 			/* No, so don't register this ciphersuite */
1185 			continue;
1186 
1187 		if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1188 			ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
1189 		else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1190 			ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
1191 		else
1192 			ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
1193 
1194 		if (ret)
1195 			goto fail;
1196 	}
1197 
1198 	return 0;
1199 
1200 fail:
1201 	for (j = 0; j < i; j++) {
1202 		/* Do we have all required base algorithms available? */
1203 		if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
1204 		    safexcel_algs[j]->algo_mask)
1205 			/* No, so don't unregister this ciphersuite */
1206 			continue;
1207 
1208 		if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1209 			crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
1210 		else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
1211 			crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
1212 		else
1213 			crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
1214 	}
1215 
1216 	return ret;
1217 }
1218 
1219 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
1220 {
1221 	int i;
1222 
1223 	for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1224 		/* Do we have all required base algorithms available? */
1225 		if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1226 		    safexcel_algs[i]->algo_mask)
1227 			/* No, so don't unregister this ciphersuite */
1228 			continue;
1229 
1230 		if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1231 			crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
1232 		else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1233 			crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
1234 		else
1235 			crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
1236 	}
1237 }
1238 
1239 static void safexcel_configure(struct safexcel_crypto_priv *priv)
1240 {
1241 	u32 val, mask = 0;
1242 
1243 	val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
1244 
1245 	/* Read number of PEs from the engine */
1246 	if (priv->flags & SAFEXCEL_HW_EIP197)
1247 		/* Wider field width for all EIP197 type engines */
1248 		mask = EIP197_N_PES_MASK;
1249 	else
1250 		/* Narrow field width for EIP97 type engine */
1251 		mask = EIP97_N_PES_MASK;
1252 
1253 	priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask;
1254 
1255 	priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
1256 
1257 	val = (val & GENMASK(27, 25)) >> 25;
1258 	mask = BIT(val) - 1;
1259 
1260 	priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
1261 	priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
1262 
1263 	priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
1264 	priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
1265 }
1266 
1267 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
1268 {
1269 	struct safexcel_register_offsets *offsets = &priv->offsets;
1270 
1271 	if (priv->flags & SAFEXCEL_HW_EIP197) {
1272 		offsets->hia_aic	= EIP197_HIA_AIC_BASE;
1273 		offsets->hia_aic_g	= EIP197_HIA_AIC_G_BASE;
1274 		offsets->hia_aic_r	= EIP197_HIA_AIC_R_BASE;
1275 		offsets->hia_aic_xdr	= EIP197_HIA_AIC_xDR_BASE;
1276 		offsets->hia_dfe	= EIP197_HIA_DFE_BASE;
1277 		offsets->hia_dfe_thr	= EIP197_HIA_DFE_THR_BASE;
1278 		offsets->hia_dse	= EIP197_HIA_DSE_BASE;
1279 		offsets->hia_dse_thr	= EIP197_HIA_DSE_THR_BASE;
1280 		offsets->hia_gen_cfg	= EIP197_HIA_GEN_CFG_BASE;
1281 		offsets->pe		= EIP197_PE_BASE;
1282 		offsets->global		= EIP197_GLOBAL_BASE;
1283 	} else {
1284 		offsets->hia_aic	= EIP97_HIA_AIC_BASE;
1285 		offsets->hia_aic_g	= EIP97_HIA_AIC_G_BASE;
1286 		offsets->hia_aic_r	= EIP97_HIA_AIC_R_BASE;
1287 		offsets->hia_aic_xdr	= EIP97_HIA_AIC_xDR_BASE;
1288 		offsets->hia_dfe	= EIP97_HIA_DFE_BASE;
1289 		offsets->hia_dfe_thr	= EIP97_HIA_DFE_THR_BASE;
1290 		offsets->hia_dse	= EIP97_HIA_DSE_BASE;
1291 		offsets->hia_dse_thr	= EIP97_HIA_DSE_THR_BASE;
1292 		offsets->hia_gen_cfg	= EIP97_HIA_GEN_CFG_BASE;
1293 		offsets->pe		= EIP97_PE_BASE;
1294 		offsets->global		= EIP97_GLOBAL_BASE;
1295 	}
1296 }
1297 
1298 /*
1299  * Generic part of probe routine, shared by platform and PCI driver
1300  *
1301  * Assumes IO resources have been mapped, private data mem has been allocated,
1302  * clocks have been enabled, device pointer has been assigned etc.
1303  *
1304  */
1305 static int safexcel_probe_generic(void *pdev,
1306 				  struct safexcel_crypto_priv *priv,
1307 				  int is_pci_dev)
1308 {
1309 	struct device *dev = priv->dev;
1310 	u32 peid, version, mask, val, hiaopt;
1311 	int i, ret, hwctg;
1312 
1313 	priv->context_pool = dmam_pool_create("safexcel-context", dev,
1314 					      sizeof(struct safexcel_context_record),
1315 					      1, 0);
1316 	if (!priv->context_pool)
1317 		return -ENOMEM;
1318 
1319 	/*
1320 	 * First try the EIP97 HIA version regs
1321 	 * For the EIP197, this is guaranteed to NOT return any of the test
1322 	 * values
1323 	 */
1324 	version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
1325 
1326 	mask = 0;  /* do not swap */
1327 	if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1328 		priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1329 	} else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
1330 		/* read back byte-swapped, so complement byte swap bits */
1331 		mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1332 		priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1333 	} else {
1334 		/* So it wasn't an EIP97 ... maybe it's an EIP197? */
1335 		version = readl(priv->base + EIP197_HIA_AIC_BASE +
1336 				EIP197_HIA_VERSION);
1337 		if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1338 			priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1339 			priv->flags |= SAFEXCEL_HW_EIP197;
1340 		} else if (EIP197_REG_HI16(version) ==
1341 			   EIP197_HIA_VERSION_BE) {
1342 			/* read back byte-swapped, so complement swap bits */
1343 			mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1344 			priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1345 			priv->flags |= SAFEXCEL_HW_EIP197;
1346 		} else {
1347 			return -ENODEV;
1348 		}
1349 	}
1350 
1351 	/* Now initialize the reg offsets based on the probing info so far */
1352 	safexcel_init_register_offsets(priv);
1353 
1354 	/*
1355 	 * If the version was read byte-swapped, we need to flip the device
1356 	 * swapping Keep in mind here, though, that what we write will also be
1357 	 * byte-swapped ...
1358 	 */
1359 	if (mask) {
1360 		val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1361 		val = val ^ (mask >> 24); /* toggle byte swap bits */
1362 		writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1363 	}
1364 
1365 	/*
1366 	 * We're not done probing yet! We may fall through to here if no HIA
1367 	 * was found at all. So, with the endianness presumably correct now and
1368 	 * the offsets setup, *really* probe for the EIP97/EIP197.
1369 	 */
1370 	version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
1371 	if (((priv->flags & SAFEXCEL_HW_EIP197) &&
1372 	     (EIP197_REG_LO16(version) != EIP197_VERSION_LE)) ||
1373 	    ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
1374 	     (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
1375 		/*
1376 		 * We did not find the device that matched our initial probing
1377 		 * (or our initial probing failed) Report appropriate error.
1378 		 */
1379 		return -ENODEV;
1380 	}
1381 
1382 	priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
1383 	hwctg = version >> 28;
1384 	peid = version & 255;
1385 
1386 	/* Detect EIP96 packet engine and version */
1387 	version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
1388 	if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1389 		dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
1390 		return -ENODEV;
1391 	}
1392 	priv->hwconfig.pever = EIP197_VERSION_MASK(version);
1393 
1394 	hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
1395 
1396 	if (priv->flags & SAFEXCEL_HW_EIP197) {
1397 		/* EIP197 */
1398 		priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1399 					  EIP197_HWDATAW_MASK;
1400 		priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
1401 					   EIP197_CFSIZE_MASK) +
1402 					  EIP197_CFSIZE_ADJUST;
1403 		priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
1404 					   EIP197_RFSIZE_MASK) +
1405 					  EIP197_RFSIZE_ADJUST;
1406 	} else {
1407 		/* EIP97 */
1408 		priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1409 					  EIP97_HWDATAW_MASK;
1410 		priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
1411 					  EIP97_CFSIZE_MASK;
1412 		priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
1413 					  EIP97_RFSIZE_MASK;
1414 	}
1415 
1416 	/* Get supported algorithms from EIP96 transform engine */
1417 	priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
1418 				    EIP197_PE_EIP96_OPTIONS(0));
1419 
1420 	/* Print single info line describing what we just detected */
1421 	dev_info(priv->dev, "EIP%d:%x(%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n",
1422 		 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hiaver,
1423 		 priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize,
1424 		 priv->hwconfig.hwrfsize, priv->hwconfig.pever,
1425 		 priv->hwconfig.algo_flags);
1426 
1427 	safexcel_configure(priv);
1428 
1429 	if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
1430 		/*
1431 		 * Request MSI vectors for global + 1 per ring -
1432 		 * or just 1 for older dev images
1433 		 */
1434 		struct pci_dev *pci_pdev = pdev;
1435 
1436 		ret = pci_alloc_irq_vectors(pci_pdev,
1437 					    priv->config.rings + 1,
1438 					    priv->config.rings + 1,
1439 					    PCI_IRQ_MSI | PCI_IRQ_MSIX);
1440 		if (ret < 0) {
1441 			dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
1442 			return ret;
1443 		}
1444 	}
1445 
1446 	/* Register the ring IRQ handlers and configure the rings */
1447 	priv->ring = devm_kcalloc(dev, priv->config.rings,
1448 				  sizeof(*priv->ring),
1449 				  GFP_KERNEL);
1450 	if (!priv->ring)
1451 		return -ENOMEM;
1452 
1453 	for (i = 0; i < priv->config.rings; i++) {
1454 		char wq_name[9] = {0};
1455 		int irq;
1456 		struct safexcel_ring_irq_data *ring_irq;
1457 
1458 		ret = safexcel_init_ring_descriptors(priv,
1459 						     &priv->ring[i].cdr,
1460 						     &priv->ring[i].rdr);
1461 		if (ret) {
1462 			dev_err(dev, "Failed to initialize rings\n");
1463 			return ret;
1464 		}
1465 
1466 		priv->ring[i].rdr_req = devm_kcalloc(dev,
1467 			EIP197_DEFAULT_RING_SIZE,
1468 			sizeof(priv->ring[i].rdr_req),
1469 			GFP_KERNEL);
1470 		if (!priv->ring[i].rdr_req)
1471 			return -ENOMEM;
1472 
1473 		ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
1474 		if (!ring_irq)
1475 			return -ENOMEM;
1476 
1477 		ring_irq->priv = priv;
1478 		ring_irq->ring = i;
1479 
1480 		irq = safexcel_request_ring_irq(pdev,
1481 						EIP197_IRQ_NUMBER(i, is_pci_dev),
1482 						is_pci_dev,
1483 						safexcel_irq_ring,
1484 						safexcel_irq_ring_thread,
1485 						ring_irq);
1486 		if (irq < 0) {
1487 			dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
1488 			return irq;
1489 		}
1490 
1491 		priv->ring[i].work_data.priv = priv;
1492 		priv->ring[i].work_data.ring = i;
1493 		INIT_WORK(&priv->ring[i].work_data.work,
1494 			  safexcel_dequeue_work);
1495 
1496 		snprintf(wq_name, 9, "wq_ring%d", i);
1497 		priv->ring[i].workqueue =
1498 			create_singlethread_workqueue(wq_name);
1499 		if (!priv->ring[i].workqueue)
1500 			return -ENOMEM;
1501 
1502 		priv->ring[i].requests = 0;
1503 		priv->ring[i].busy = false;
1504 
1505 		crypto_init_queue(&priv->ring[i].queue,
1506 				  EIP197_DEFAULT_RING_SIZE);
1507 
1508 		spin_lock_init(&priv->ring[i].lock);
1509 		spin_lock_init(&priv->ring[i].queue_lock);
1510 	}
1511 
1512 	atomic_set(&priv->ring_used, 0);
1513 
1514 	ret = safexcel_hw_init(priv);
1515 	if (ret) {
1516 		dev_err(dev, "HW init failed (%d)\n", ret);
1517 		return ret;
1518 	}
1519 
1520 	ret = safexcel_register_algorithms(priv);
1521 	if (ret) {
1522 		dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1523 		return ret;
1524 	}
1525 
1526 	return 0;
1527 }
1528 
1529 static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
1530 {
1531 	int i;
1532 
1533 	for (i = 0; i < priv->config.rings; i++) {
1534 		/* clear any pending interrupt */
1535 		writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
1536 		writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
1537 
1538 		/* Reset the CDR base address */
1539 		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1540 		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1541 
1542 		/* Reset the RDR base address */
1543 		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1544 		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1545 	}
1546 }
1547 
1548 #if IS_ENABLED(CONFIG_OF)
1549 /* for Device Tree platform driver */
1550 
1551 static int safexcel_probe(struct platform_device *pdev)
1552 {
1553 	struct device *dev = &pdev->dev;
1554 	struct safexcel_crypto_priv *priv;
1555 	int ret;
1556 
1557 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
1558 	if (!priv)
1559 		return -ENOMEM;
1560 
1561 	priv->dev = dev;
1562 	priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
1563 
1564 	platform_set_drvdata(pdev, priv);
1565 
1566 	priv->base = devm_platform_ioremap_resource(pdev, 0);
1567 	if (IS_ERR(priv->base)) {
1568 		dev_err(dev, "failed to get resource\n");
1569 		return PTR_ERR(priv->base);
1570 	}
1571 
1572 	priv->clk = devm_clk_get(&pdev->dev, NULL);
1573 	ret = PTR_ERR_OR_ZERO(priv->clk);
1574 	/* The clock isn't mandatory */
1575 	if  (ret != -ENOENT) {
1576 		if (ret)
1577 			return ret;
1578 
1579 		ret = clk_prepare_enable(priv->clk);
1580 		if (ret) {
1581 			dev_err(dev, "unable to enable clk (%d)\n", ret);
1582 			return ret;
1583 		}
1584 	}
1585 
1586 	priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
1587 	ret = PTR_ERR_OR_ZERO(priv->reg_clk);
1588 	/* The clock isn't mandatory */
1589 	if  (ret != -ENOENT) {
1590 		if (ret)
1591 			goto err_core_clk;
1592 
1593 		ret = clk_prepare_enable(priv->reg_clk);
1594 		if (ret) {
1595 			dev_err(dev, "unable to enable reg clk (%d)\n", ret);
1596 			goto err_core_clk;
1597 		}
1598 	}
1599 
1600 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1601 	if (ret)
1602 		goto err_reg_clk;
1603 
1604 	/* Generic EIP97/EIP197 device probing */
1605 	ret = safexcel_probe_generic(pdev, priv, 0);
1606 	if (ret)
1607 		goto err_reg_clk;
1608 
1609 	return 0;
1610 
1611 err_reg_clk:
1612 	clk_disable_unprepare(priv->reg_clk);
1613 err_core_clk:
1614 	clk_disable_unprepare(priv->clk);
1615 	return ret;
1616 }
1617 
1618 static int safexcel_remove(struct platform_device *pdev)
1619 {
1620 	struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1621 	int i;
1622 
1623 	safexcel_unregister_algorithms(priv);
1624 	safexcel_hw_reset_rings(priv);
1625 
1626 	clk_disable_unprepare(priv->clk);
1627 
1628 	for (i = 0; i < priv->config.rings; i++)
1629 		destroy_workqueue(priv->ring[i].workqueue);
1630 
1631 	return 0;
1632 }
1633 
1634 static const struct of_device_id safexcel_of_match_table[] = {
1635 	{
1636 		.compatible = "inside-secure,safexcel-eip97ies",
1637 		.data = (void *)EIP97IES_MRVL,
1638 	},
1639 	{
1640 		.compatible = "inside-secure,safexcel-eip197b",
1641 		.data = (void *)EIP197B_MRVL,
1642 	},
1643 	{
1644 		.compatible = "inside-secure,safexcel-eip197d",
1645 		.data = (void *)EIP197D_MRVL,
1646 	},
1647 	/* For backward compatibility and intended for generic use */
1648 	{
1649 		.compatible = "inside-secure,safexcel-eip97",
1650 		.data = (void *)EIP97IES_MRVL,
1651 	},
1652 	{
1653 		.compatible = "inside-secure,safexcel-eip197",
1654 		.data = (void *)EIP197B_MRVL,
1655 	},
1656 	{},
1657 };
1658 
1659 static struct platform_driver  crypto_safexcel = {
1660 	.probe		= safexcel_probe,
1661 	.remove		= safexcel_remove,
1662 	.driver		= {
1663 		.name	= "crypto-safexcel",
1664 		.of_match_table = safexcel_of_match_table,
1665 	},
1666 };
1667 #endif
1668 
1669 #if IS_ENABLED(CONFIG_PCI)
1670 /* PCIE devices - i.e. Inside Secure development boards */
1671 
1672 static int safexcel_pci_probe(struct pci_dev *pdev,
1673 			       const struct pci_device_id *ent)
1674 {
1675 	struct device *dev = &pdev->dev;
1676 	struct safexcel_crypto_priv *priv;
1677 	void __iomem *pciebase;
1678 	int rc;
1679 	u32 val;
1680 
1681 	dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
1682 		ent->vendor, ent->device, ent->subvendor,
1683 		ent->subdevice, ent->driver_data);
1684 
1685 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1686 	if (!priv)
1687 		return -ENOMEM;
1688 
1689 	priv->dev = dev;
1690 	priv->version = (enum safexcel_eip_version)ent->driver_data;
1691 
1692 	pci_set_drvdata(pdev, priv);
1693 
1694 	/* enable the device */
1695 	rc = pcim_enable_device(pdev);
1696 	if (rc) {
1697 		dev_err(dev, "Failed to enable PCI device\n");
1698 		return rc;
1699 	}
1700 
1701 	/* take ownership of PCI BAR0 */
1702 	rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
1703 	if (rc) {
1704 		dev_err(dev, "Failed to map IO region for BAR0\n");
1705 		return rc;
1706 	}
1707 	priv->base = pcim_iomap_table(pdev)[0];
1708 
1709 	if (priv->version == EIP197_DEVBRD) {
1710 		dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
1711 
1712 		rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
1713 		if (rc) {
1714 			dev_err(dev, "Failed to map IO region for BAR4\n");
1715 			return rc;
1716 		}
1717 
1718 		pciebase = pcim_iomap_table(pdev)[2];
1719 		val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
1720 		if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
1721 			dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
1722 				(val & 0xff));
1723 
1724 			/* Setup MSI identity map mapping */
1725 			writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
1726 			       pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
1727 			writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
1728 			       pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
1729 			writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
1730 			       pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
1731 			writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
1732 			       pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
1733 
1734 			/* Enable all device interrupts */
1735 			writel(GENMASK(31, 0),
1736 			       pciebase + EIP197_XLX_USER_INT_ENB_MSK);
1737 		} else {
1738 			dev_err(dev, "Unrecognised IRQ block identifier %x\n",
1739 				val);
1740 			return -ENODEV;
1741 		}
1742 
1743 		/* HW reset FPGA dev board */
1744 		/* assert reset */
1745 		writel(1, priv->base + EIP197_XLX_GPIO_BASE);
1746 		wmb(); /* maintain strict ordering for accesses here */
1747 		/* deassert reset */
1748 		writel(0, priv->base + EIP197_XLX_GPIO_BASE);
1749 		wmb(); /* maintain strict ordering for accesses here */
1750 	}
1751 
1752 	/* enable bus mastering */
1753 	pci_set_master(pdev);
1754 
1755 	/* Generic EIP97/EIP197 device probing */
1756 	rc = safexcel_probe_generic(pdev, priv, 1);
1757 	return rc;
1758 }
1759 
1760 void safexcel_pci_remove(struct pci_dev *pdev)
1761 {
1762 	struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
1763 	int i;
1764 
1765 	safexcel_unregister_algorithms(priv);
1766 
1767 	for (i = 0; i < priv->config.rings; i++)
1768 		destroy_workqueue(priv->ring[i].workqueue);
1769 
1770 	safexcel_hw_reset_rings(priv);
1771 }
1772 
1773 static const struct pci_device_id safexcel_pci_ids[] = {
1774 	{
1775 		PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
1776 			       0x16ae, 0xc522),
1777 		.driver_data = EIP197_DEVBRD,
1778 	},
1779 	{},
1780 };
1781 
1782 MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
1783 
1784 static struct pci_driver safexcel_pci_driver = {
1785 	.name          = "crypto-safexcel",
1786 	.id_table      = safexcel_pci_ids,
1787 	.probe         = safexcel_pci_probe,
1788 	.remove        = safexcel_pci_remove,
1789 };
1790 #endif
1791 
1792 static int __init safexcel_init(void)
1793 {
1794 	int rc;
1795 
1796 #if IS_ENABLED(CONFIG_OF)
1797 		/* Register platform driver */
1798 		platform_driver_register(&crypto_safexcel);
1799 #endif
1800 
1801 #if IS_ENABLED(CONFIG_PCI)
1802 		/* Register PCI driver */
1803 		rc = pci_register_driver(&safexcel_pci_driver);
1804 #endif
1805 
1806 	return 0;
1807 }
1808 
1809 static void __exit safexcel_exit(void)
1810 {
1811 #if IS_ENABLED(CONFIG_OF)
1812 		/* Unregister platform driver */
1813 		platform_driver_unregister(&crypto_safexcel);
1814 #endif
1815 
1816 #if IS_ENABLED(CONFIG_PCI)
1817 		/* Unregister PCI driver if successfully registered before */
1818 		pci_unregister_driver(&safexcel_pci_driver);
1819 #endif
1820 }
1821 
1822 module_init(safexcel_init);
1823 module_exit(safexcel_exit);
1824 
1825 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1826 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1827 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1828 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
1829 MODULE_LICENSE("GPL v2");
1830