xref: /linux/drivers/mtd/nand/raw/atmel/pmecc.c (revision 4f2c0a4acffbec01079c28f839422e64ddeff004)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright 2017 ATMEL
4  * Copyright 2017 Free Electrons
5  *
6  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
7  *
8  * Derived from the atmel_nand.c driver which contained the following
9  * copyrights:
10  *
11  *   Copyright 2003 Rick Bronson
12  *
13  *   Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
14  *	Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
15  *
16  *   Derived from drivers/mtd/spia.c (removed in v3.8)
17  *	Copyright 2000 Steven J. Hill (sjhill@cotw.com)
18  *
19  *   Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
20  *	Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
21  *
22  *   Derived from Das U-Boot source code
23  *	(u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
24  *      Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
25  *
26  *   Add Programmable Multibit ECC support for various AT91 SoC
27  *	Copyright 2012 ATMEL, Hong Xu
28  *
29  *   Add Nand Flash Controller support for SAMA5 SoC
30  *	Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
31  *
32  * The PMECC is an hardware assisted BCH engine, which means part of the
33  * ECC algorithm is left to the software. The hardware/software repartition
34  * is explained in the "PMECC Controller Functional Description" chapter in
35  * Atmel datasheets, and some of the functions in this file are directly
36  * implementing the algorithms described in the "Software Implementation"
37  * sub-section.
38  *
39  * TODO: it seems that the software BCH implementation in lib/bch.c is already
40  * providing some of the logic we are implementing here. It would be smart
41  * to expose the needed lib/bch.c helpers/functions and re-use them here.
42  */
43 
44 #include <linux/genalloc.h>
45 #include <linux/iopoll.h>
46 #include <linux/module.h>
47 #include <linux/mtd/rawnand.h>
48 #include <linux/of_irq.h>
49 #include <linux/of_platform.h>
50 #include <linux/platform_device.h>
51 #include <linux/slab.h>
52 
53 #include "pmecc.h"
54 
55 /* Galois field dimension */
56 #define PMECC_GF_DIMENSION_13			13
57 #define PMECC_GF_DIMENSION_14			14
58 
59 /* Primitive Polynomial used by PMECC */
60 #define PMECC_GF_13_PRIMITIVE_POLY		0x201b
61 #define PMECC_GF_14_PRIMITIVE_POLY		0x4443
62 
63 #define PMECC_LOOKUP_TABLE_SIZE_512		0x2000
64 #define PMECC_LOOKUP_TABLE_SIZE_1024		0x4000
65 
66 /* Time out value for reading PMECC status register */
67 #define PMECC_MAX_TIMEOUT_MS			100
68 
69 /* PMECC Register Definitions */
70 #define ATMEL_PMECC_CFG				0x0
71 #define PMECC_CFG_BCH_STRENGTH(x)		(x)
72 #define PMECC_CFG_BCH_STRENGTH_MASK		GENMASK(2, 0)
73 #define PMECC_CFG_SECTOR512			(0 << 4)
74 #define PMECC_CFG_SECTOR1024			(1 << 4)
75 #define PMECC_CFG_NSECTORS(x)			((fls(x) - 1) << 8)
76 #define PMECC_CFG_READ_OP			(0 << 12)
77 #define PMECC_CFG_WRITE_OP			(1 << 12)
78 #define PMECC_CFG_SPARE_ENABLE			BIT(16)
79 #define PMECC_CFG_AUTO_ENABLE			BIT(20)
80 
81 #define ATMEL_PMECC_SAREA			0x4
82 #define ATMEL_PMECC_SADDR			0x8
83 #define ATMEL_PMECC_EADDR			0xc
84 
85 #define ATMEL_PMECC_CLK				0x10
86 #define PMECC_CLK_133MHZ			(2 << 0)
87 
88 #define ATMEL_PMECC_CTRL			0x14
89 #define PMECC_CTRL_RST				BIT(0)
90 #define PMECC_CTRL_DATA				BIT(1)
91 #define PMECC_CTRL_USER				BIT(2)
92 #define PMECC_CTRL_ENABLE			BIT(4)
93 #define PMECC_CTRL_DISABLE			BIT(5)
94 
95 #define ATMEL_PMECC_SR				0x18
96 #define PMECC_SR_BUSY				BIT(0)
97 #define PMECC_SR_ENABLE				BIT(4)
98 
99 #define ATMEL_PMECC_IER				0x1c
100 #define ATMEL_PMECC_IDR				0x20
101 #define ATMEL_PMECC_IMR				0x24
102 #define ATMEL_PMECC_ISR				0x28
103 #define PMECC_ERROR_INT				BIT(0)
104 
105 #define ATMEL_PMECC_ECC(sector, n)		\
106 	((((sector) + 1) * 0x40) + (n))
107 
108 #define ATMEL_PMECC_REM(sector, n)		\
109 	((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
110 
111 /* PMERRLOC Register Definitions */
112 #define ATMEL_PMERRLOC_ELCFG			0x0
113 #define PMERRLOC_ELCFG_SECTOR_512		(0 << 0)
114 #define PMERRLOC_ELCFG_SECTOR_1024		(1 << 0)
115 #define PMERRLOC_ELCFG_NUM_ERRORS(n)		((n) << 16)
116 
117 #define ATMEL_PMERRLOC_ELPRIM			0x4
118 #define ATMEL_PMERRLOC_ELEN			0x8
119 #define ATMEL_PMERRLOC_ELDIS			0xc
120 #define PMERRLOC_DISABLE			BIT(0)
121 
122 #define ATMEL_PMERRLOC_ELSR			0x10
123 #define PMERRLOC_ELSR_BUSY			BIT(0)
124 
125 #define ATMEL_PMERRLOC_ELIER			0x14
126 #define ATMEL_PMERRLOC_ELIDR			0x18
127 #define ATMEL_PMERRLOC_ELIMR			0x1c
128 #define ATMEL_PMERRLOC_ELISR			0x20
129 #define PMERRLOC_ERR_NUM_MASK			GENMASK(12, 8)
130 #define PMERRLOC_CALC_DONE			BIT(0)
131 
132 #define ATMEL_PMERRLOC_SIGMA(x)			(((x) * 0x4) + 0x28)
133 
134 #define ATMEL_PMERRLOC_EL(offs, x)		(((x) * 0x4) + (offs))
135 
136 struct atmel_pmecc_gf_tables {
137 	u16 *alpha_to;
138 	u16 *index_of;
139 };
140 
141 struct atmel_pmecc_caps {
142 	const int *strengths;
143 	int nstrengths;
144 	int el_offset;
145 	bool correct_erased_chunks;
146 };
147 
148 struct atmel_pmecc {
149 	struct device *dev;
150 	const struct atmel_pmecc_caps *caps;
151 
152 	struct {
153 		void __iomem *base;
154 		void __iomem *errloc;
155 	} regs;
156 
157 	struct mutex lock;
158 };
159 
160 struct atmel_pmecc_user_conf_cache {
161 	u32 cfg;
162 	u32 sarea;
163 	u32 saddr;
164 	u32 eaddr;
165 };
166 
167 struct atmel_pmecc_user {
168 	struct atmel_pmecc_user_conf_cache cache;
169 	struct atmel_pmecc *pmecc;
170 	const struct atmel_pmecc_gf_tables *gf_tables;
171 	int eccbytes;
172 	s16 *partial_syn;
173 	s16 *si;
174 	s16 *lmu;
175 	s16 *smu;
176 	s32 *mu;
177 	s32 *dmu;
178 	s32 *delta;
179 	u32 isr;
180 };
181 
182 static DEFINE_MUTEX(pmecc_gf_tables_lock);
183 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
184 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
185 
deg(unsigned int poly)186 static inline int deg(unsigned int poly)
187 {
188 	/* polynomial degree is the most-significant bit index */
189 	return fls(poly) - 1;
190 }
191 
atmel_pmecc_build_gf_tables(int mm,unsigned int poly,struct atmel_pmecc_gf_tables * gf_tables)192 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
193 				       struct atmel_pmecc_gf_tables *gf_tables)
194 {
195 	unsigned int i, x = 1;
196 	const unsigned int k = BIT(deg(poly));
197 	unsigned int nn = BIT(mm) - 1;
198 
199 	/* primitive polynomial must be of degree m */
200 	if (k != (1u << mm))
201 		return -EINVAL;
202 
203 	for (i = 0; i < nn; i++) {
204 		gf_tables->alpha_to[i] = x;
205 		gf_tables->index_of[x] = i;
206 		if (i && (x == 1))
207 			/* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
208 			return -EINVAL;
209 		x <<= 1;
210 		if (x & k)
211 			x ^= poly;
212 	}
213 	gf_tables->alpha_to[nn] = 1;
214 	gf_tables->index_of[0] = 0;
215 
216 	return 0;
217 }
218 
219 static const struct atmel_pmecc_gf_tables *
atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req * req)220 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
221 {
222 	struct atmel_pmecc_gf_tables *gf_tables;
223 	unsigned int poly, degree, table_size;
224 	int ret;
225 
226 	if (req->ecc.sectorsize == 512) {
227 		degree = PMECC_GF_DIMENSION_13;
228 		poly = PMECC_GF_13_PRIMITIVE_POLY;
229 		table_size = PMECC_LOOKUP_TABLE_SIZE_512;
230 	} else {
231 		degree = PMECC_GF_DIMENSION_14;
232 		poly = PMECC_GF_14_PRIMITIVE_POLY;
233 		table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
234 	}
235 
236 	gf_tables = kzalloc(sizeof(*gf_tables) +
237 			    (2 * table_size * sizeof(u16)),
238 			    GFP_KERNEL);
239 	if (!gf_tables)
240 		return ERR_PTR(-ENOMEM);
241 
242 	gf_tables->alpha_to = (void *)(gf_tables + 1);
243 	gf_tables->index_of = gf_tables->alpha_to + table_size;
244 
245 	ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
246 	if (ret) {
247 		kfree(gf_tables);
248 		return ERR_PTR(ret);
249 	}
250 
251 	return gf_tables;
252 }
253 
254 static const struct atmel_pmecc_gf_tables *
atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req * req)255 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
256 {
257 	const struct atmel_pmecc_gf_tables **gf_tables, *ret;
258 
259 	mutex_lock(&pmecc_gf_tables_lock);
260 	if (req->ecc.sectorsize == 512)
261 		gf_tables = &pmecc_gf_tables_512;
262 	else
263 		gf_tables = &pmecc_gf_tables_1024;
264 
265 	ret = *gf_tables;
266 
267 	if (!ret) {
268 		ret = atmel_pmecc_create_gf_tables(req);
269 		if (!IS_ERR(ret))
270 			*gf_tables = ret;
271 	}
272 	mutex_unlock(&pmecc_gf_tables_lock);
273 
274 	return ret;
275 }
276 
atmel_pmecc_prepare_user_req(struct atmel_pmecc * pmecc,struct atmel_pmecc_user_req * req)277 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
278 					struct atmel_pmecc_user_req *req)
279 {
280 	int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
281 
282 	if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
283 		return -EINVAL;
284 
285 	if (req->ecc.ooboffset >= 0 &&
286 	    req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
287 		return -EINVAL;
288 
289 	if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
290 		if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
291 			return -EINVAL;
292 
293 		if (req->pagesize > 512)
294 			req->ecc.sectorsize = 1024;
295 		else
296 			req->ecc.sectorsize = 512;
297 	}
298 
299 	if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
300 		return -EINVAL;
301 
302 	if (req->pagesize % req->ecc.sectorsize)
303 		return -EINVAL;
304 
305 	req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
306 
307 	max_eccbytes = req->ecc.bytes;
308 
309 	for (i = 0; i < pmecc->caps->nstrengths; i++) {
310 		int nbytes, strength = pmecc->caps->strengths[i];
311 
312 		if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
313 		    strength < req->ecc.strength)
314 			continue;
315 
316 		nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
317 				      8);
318 		nbytes *= req->ecc.nsectors;
319 
320 		if (nbytes > max_eccbytes)
321 			break;
322 
323 		eccstrength = strength;
324 		eccbytes = nbytes;
325 
326 		if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
327 			break;
328 	}
329 
330 	if (!eccstrength)
331 		return -EINVAL;
332 
333 	req->ecc.bytes = eccbytes;
334 	req->ecc.strength = eccstrength;
335 
336 	if (req->ecc.ooboffset < 0)
337 		req->ecc.ooboffset = req->oobsize - eccbytes;
338 
339 	return 0;
340 }
341 
342 struct atmel_pmecc_user *
atmel_pmecc_create_user(struct atmel_pmecc * pmecc,struct atmel_pmecc_user_req * req)343 atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
344 			struct atmel_pmecc_user_req *req)
345 {
346 	struct atmel_pmecc_user *user;
347 	const struct atmel_pmecc_gf_tables *gf_tables;
348 	int strength, size, ret;
349 
350 	ret = atmel_pmecc_prepare_user_req(pmecc, req);
351 	if (ret)
352 		return ERR_PTR(ret);
353 
354 	size = sizeof(*user);
355 	size = ALIGN(size, sizeof(u16));
356 	/* Reserve space for partial_syn, si and smu */
357 	size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
358 		(2 + req->ecc.strength + 2);
359 	/* Reserve space for lmu. */
360 	size += (req->ecc.strength + 1) * sizeof(u16);
361 	/* Reserve space for mu, dmu and delta. */
362 	size = ALIGN(size, sizeof(s32));
363 	size += (req->ecc.strength + 1) * sizeof(s32) * 3;
364 
365 	user = kzalloc(size, GFP_KERNEL);
366 	if (!user)
367 		return ERR_PTR(-ENOMEM);
368 
369 	user->pmecc = pmecc;
370 
371 	user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
372 	user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
373 	user->lmu = user->si + ((2 * req->ecc.strength) + 1);
374 	user->smu = user->lmu + (req->ecc.strength + 1);
375 	user->mu = (s32 *)PTR_ALIGN(user->smu +
376 				    (((2 * req->ecc.strength) + 1) *
377 				     (req->ecc.strength + 2)),
378 				    sizeof(s32));
379 	user->dmu = user->mu + req->ecc.strength + 1;
380 	user->delta = user->dmu + req->ecc.strength + 1;
381 
382 	gf_tables = atmel_pmecc_get_gf_tables(req);
383 	if (IS_ERR(gf_tables)) {
384 		kfree(user);
385 		return ERR_CAST(gf_tables);
386 	}
387 
388 	user->gf_tables = gf_tables;
389 
390 	user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
391 
392 	for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
393 		if (pmecc->caps->strengths[strength] == req->ecc.strength)
394 			break;
395 	}
396 
397 	user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
398 			  PMECC_CFG_NSECTORS(req->ecc.nsectors);
399 
400 	if (req->ecc.sectorsize == 1024)
401 		user->cache.cfg |= PMECC_CFG_SECTOR1024;
402 
403 	user->cache.sarea = req->oobsize - 1;
404 	user->cache.saddr = req->ecc.ooboffset;
405 	user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
406 
407 	return user;
408 }
409 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
410 
atmel_pmecc_destroy_user(struct atmel_pmecc_user * user)411 void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user)
412 {
413 	kfree(user);
414 }
415 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user);
416 
get_strength(struct atmel_pmecc_user * user)417 static int get_strength(struct atmel_pmecc_user *user)
418 {
419 	const int *strengths = user->pmecc->caps->strengths;
420 
421 	return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
422 }
423 
get_sectorsize(struct atmel_pmecc_user * user)424 static int get_sectorsize(struct atmel_pmecc_user *user)
425 {
426 	return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
427 }
428 
atmel_pmecc_gen_syndrome(struct atmel_pmecc_user * user,int sector)429 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
430 {
431 	int strength = get_strength(user);
432 	u32 value;
433 	int i;
434 
435 	/* Fill odd syndromes */
436 	for (i = 0; i < strength; i++) {
437 		value = readl_relaxed(user->pmecc->regs.base +
438 				      ATMEL_PMECC_REM(sector, i / 2));
439 		if (i & 1)
440 			value >>= 16;
441 
442 		user->partial_syn[(2 * i) + 1] = value;
443 	}
444 }
445 
atmel_pmecc_substitute(struct atmel_pmecc_user * user)446 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
447 {
448 	int degree = get_sectorsize(user) == 512 ? 13 : 14;
449 	int cw_len = BIT(degree) - 1;
450 	int strength = get_strength(user);
451 	s16 *alpha_to = user->gf_tables->alpha_to;
452 	s16 *index_of = user->gf_tables->index_of;
453 	s16 *partial_syn = user->partial_syn;
454 	s16 *si;
455 	int i, j;
456 
457 	/*
458 	 * si[] is a table that holds the current syndrome value,
459 	 * an element of that table belongs to the field
460 	 */
461 	si = user->si;
462 
463 	memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
464 
465 	/* Computation 2t syndromes based on S(x) */
466 	/* Odd syndromes */
467 	for (i = 1; i < 2 * strength; i += 2) {
468 		for (j = 0; j < degree; j++) {
469 			if (partial_syn[i] & BIT(j))
470 				si[i] = alpha_to[i * j] ^ si[i];
471 		}
472 	}
473 	/* Even syndrome = (Odd syndrome) ** 2 */
474 	for (i = 2, j = 1; j <= strength; i = ++j << 1) {
475 		if (si[j] == 0) {
476 			si[i] = 0;
477 		} else {
478 			s16 tmp;
479 
480 			tmp = index_of[si[j]];
481 			tmp = (tmp * 2) % cw_len;
482 			si[i] = alpha_to[tmp];
483 		}
484 	}
485 }
486 
atmel_pmecc_get_sigma(struct atmel_pmecc_user * user)487 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
488 {
489 	s16 *lmu = user->lmu;
490 	s16 *si = user->si;
491 	s32 *mu = user->mu;
492 	s32 *dmu = user->dmu;
493 	s32 *delta = user->delta;
494 	int degree = get_sectorsize(user) == 512 ? 13 : 14;
495 	int cw_len = BIT(degree) - 1;
496 	int strength = get_strength(user);
497 	int num = 2 * strength + 1;
498 	s16 *index_of = user->gf_tables->index_of;
499 	s16 *alpha_to = user->gf_tables->alpha_to;
500 	int i, j, k;
501 	u32 dmu_0_count, tmp;
502 	s16 *smu = user->smu;
503 
504 	/* index of largest delta */
505 	int ro;
506 	int largest;
507 	int diff;
508 
509 	dmu_0_count = 0;
510 
511 	/* First Row */
512 
513 	/* Mu */
514 	mu[0] = -1;
515 
516 	memset(smu, 0, sizeof(s16) * num);
517 	smu[0] = 1;
518 
519 	/* discrepancy set to 1 */
520 	dmu[0] = 1;
521 	/* polynom order set to 0 */
522 	lmu[0] = 0;
523 	delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
524 
525 	/* Second Row */
526 
527 	/* Mu */
528 	mu[1] = 0;
529 	/* Sigma(x) set to 1 */
530 	memset(&smu[num], 0, sizeof(s16) * num);
531 	smu[num] = 1;
532 
533 	/* discrepancy set to S1 */
534 	dmu[1] = si[1];
535 
536 	/* polynom order set to 0 */
537 	lmu[1] = 0;
538 
539 	delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
540 
541 	/* Init the Sigma(x) last row */
542 	memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
543 
544 	for (i = 1; i <= strength; i++) {
545 		mu[i + 1] = i << 1;
546 		/* Begin Computing Sigma (Mu+1) and L(mu) */
547 		/* check if discrepancy is set to 0 */
548 		if (dmu[i] == 0) {
549 			dmu_0_count++;
550 
551 			tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
552 			if ((strength - (lmu[i] >> 1) - 1) & 0x1)
553 				tmp += 2;
554 			else
555 				tmp += 1;
556 
557 			if (dmu_0_count == tmp) {
558 				for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
559 					smu[(strength + 1) * num + j] =
560 							smu[i * num + j];
561 
562 				lmu[strength + 1] = lmu[i];
563 				return;
564 			}
565 
566 			/* copy polynom */
567 			for (j = 0; j <= lmu[i] >> 1; j++)
568 				smu[(i + 1) * num + j] = smu[i * num + j];
569 
570 			/* copy previous polynom order to the next */
571 			lmu[i + 1] = lmu[i];
572 		} else {
573 			ro = 0;
574 			largest = -1;
575 			/* find largest delta with dmu != 0 */
576 			for (j = 0; j < i; j++) {
577 				if ((dmu[j]) && (delta[j] > largest)) {
578 					largest = delta[j];
579 					ro = j;
580 				}
581 			}
582 
583 			/* compute difference */
584 			diff = (mu[i] - mu[ro]);
585 
586 			/* Compute degree of the new smu polynomial */
587 			if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
588 				lmu[i + 1] = lmu[i];
589 			else
590 				lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
591 
592 			/* Init smu[i+1] with 0 */
593 			for (k = 0; k < num; k++)
594 				smu[(i + 1) * num + k] = 0;
595 
596 			/* Compute smu[i+1] */
597 			for (k = 0; k <= lmu[ro] >> 1; k++) {
598 				s16 a, b, c;
599 
600 				if (!(smu[ro * num + k] && dmu[i]))
601 					continue;
602 
603 				a = index_of[dmu[i]];
604 				b = index_of[dmu[ro]];
605 				c = index_of[smu[ro * num + k]];
606 				tmp = a + (cw_len - b) + c;
607 				a = alpha_to[tmp % cw_len];
608 				smu[(i + 1) * num + (k + diff)] = a;
609 			}
610 
611 			for (k = 0; k <= lmu[i] >> 1; k++)
612 				smu[(i + 1) * num + k] ^= smu[i * num + k];
613 		}
614 
615 		/* End Computing Sigma (Mu+1) and L(mu) */
616 		/* In either case compute delta */
617 		delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
618 
619 		/* Do not compute discrepancy for the last iteration */
620 		if (i >= strength)
621 			continue;
622 
623 		for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
624 			tmp = 2 * (i - 1);
625 			if (k == 0) {
626 				dmu[i + 1] = si[tmp + 3];
627 			} else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
628 				s16 a, b, c;
629 
630 				a = index_of[smu[(i + 1) * num + k]];
631 				b = si[2 * (i - 1) + 3 - k];
632 				c = index_of[b];
633 				tmp = a + c;
634 				tmp %= cw_len;
635 				dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
636 			}
637 		}
638 	}
639 }
640 
atmel_pmecc_err_location(struct atmel_pmecc_user * user)641 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
642 {
643 	int sector_size = get_sectorsize(user);
644 	int degree = sector_size == 512 ? 13 : 14;
645 	struct atmel_pmecc *pmecc = user->pmecc;
646 	int strength = get_strength(user);
647 	int ret, roots_nbr, i, err_nbr = 0;
648 	int num = (2 * strength) + 1;
649 	s16 *smu = user->smu;
650 	u32 val;
651 
652 	writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
653 
654 	for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
655 		writel_relaxed(smu[(strength + 1) * num + i],
656 			       pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
657 		err_nbr++;
658 	}
659 
660 	val = (err_nbr - 1) << 16;
661 	if (sector_size == 1024)
662 		val |= 1;
663 
664 	writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
665 	writel((sector_size * 8) + (degree * strength),
666 	       pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
667 
668 	ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
669 					 ATMEL_PMERRLOC_ELISR,
670 					 val, val & PMERRLOC_CALC_DONE, 0,
671 					 PMECC_MAX_TIMEOUT_MS * 1000);
672 	if (ret) {
673 		dev_err(pmecc->dev,
674 			"PMECC: Timeout to calculate error location.\n");
675 		return ret;
676 	}
677 
678 	roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
679 	/* Number of roots == degree of smu hence <= cap */
680 	if (roots_nbr == user->lmu[strength + 1] >> 1)
681 		return err_nbr - 1;
682 
683 	/*
684 	 * Number of roots does not match the degree of smu
685 	 * unable to correct error.
686 	 */
687 	return -EBADMSG;
688 }
689 
atmel_pmecc_correct_sector(struct atmel_pmecc_user * user,int sector,void * data,void * ecc)690 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
691 			       void *data, void *ecc)
692 {
693 	struct atmel_pmecc *pmecc = user->pmecc;
694 	int sectorsize = get_sectorsize(user);
695 	int eccbytes = user->eccbytes;
696 	int i, nerrors;
697 
698 	if (!(user->isr & BIT(sector)))
699 		return 0;
700 
701 	atmel_pmecc_gen_syndrome(user, sector);
702 	atmel_pmecc_substitute(user);
703 	atmel_pmecc_get_sigma(user);
704 
705 	nerrors = atmel_pmecc_err_location(user);
706 	if (nerrors < 0)
707 		return nerrors;
708 
709 	for (i = 0; i < nerrors; i++) {
710 		const char *area;
711 		int byte, bit;
712 		u32 errpos;
713 		u8 *ptr;
714 
715 		errpos = readl_relaxed(pmecc->regs.errloc +
716 				ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
717 		errpos--;
718 
719 		byte = errpos / 8;
720 		bit = errpos % 8;
721 
722 		if (byte < sectorsize) {
723 			ptr = data + byte;
724 			area = "data";
725 		} else if (byte < sectorsize + eccbytes) {
726 			ptr = ecc + byte - sectorsize;
727 			area = "ECC";
728 		} else {
729 			dev_dbg(pmecc->dev,
730 				"Invalid errpos value (%d, max is %d)\n",
731 				errpos, (sectorsize + eccbytes) * 8);
732 			return -EINVAL;
733 		}
734 
735 		dev_dbg(pmecc->dev,
736 			"Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
737 			area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
738 
739 		*ptr ^= BIT(bit);
740 	}
741 
742 	return nerrors;
743 }
744 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
745 
atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user * user)746 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
747 {
748 	return user->pmecc->caps->correct_erased_chunks;
749 }
750 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
751 
atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user * user,int sector,void * ecc)752 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
753 					int sector, void *ecc)
754 {
755 	struct atmel_pmecc *pmecc = user->pmecc;
756 	u8 *ptr = ecc;
757 	int i;
758 
759 	for (i = 0; i < user->eccbytes; i++)
760 		ptr[i] = readb_relaxed(pmecc->regs.base +
761 				       ATMEL_PMECC_ECC(sector, i));
762 }
763 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
764 
atmel_pmecc_reset(struct atmel_pmecc * pmecc)765 void atmel_pmecc_reset(struct atmel_pmecc *pmecc)
766 {
767 	writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
768 	writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
769 }
770 EXPORT_SYMBOL_GPL(atmel_pmecc_reset);
771 
atmel_pmecc_enable(struct atmel_pmecc_user * user,int op)772 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
773 {
774 	struct atmel_pmecc *pmecc = user->pmecc;
775 	u32 cfg;
776 
777 	if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
778 		dev_err(pmecc->dev, "Bad ECC operation!");
779 		return -EINVAL;
780 	}
781 
782 	mutex_lock(&user->pmecc->lock);
783 
784 	cfg = user->cache.cfg;
785 	if (op == NAND_ECC_WRITE)
786 		cfg |= PMECC_CFG_WRITE_OP;
787 	else
788 		cfg |= PMECC_CFG_AUTO_ENABLE;
789 
790 	writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
791 	writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
792 	writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
793 	writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
794 
795 	writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
796 	writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
797 
798 	return 0;
799 }
800 EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
801 
atmel_pmecc_disable(struct atmel_pmecc_user * user)802 void atmel_pmecc_disable(struct atmel_pmecc_user *user)
803 {
804 	atmel_pmecc_reset(user->pmecc);
805 	mutex_unlock(&user->pmecc->lock);
806 }
807 EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
808 
atmel_pmecc_wait_rdy(struct atmel_pmecc_user * user)809 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
810 {
811 	struct atmel_pmecc *pmecc = user->pmecc;
812 	u32 status;
813 	int ret;
814 
815 	ret = readl_relaxed_poll_timeout(pmecc->regs.base +
816 					 ATMEL_PMECC_SR,
817 					 status, !(status & PMECC_SR_BUSY), 0,
818 					 PMECC_MAX_TIMEOUT_MS * 1000);
819 	if (ret) {
820 		dev_err(pmecc->dev,
821 			"Timeout while waiting for PMECC ready.\n");
822 		return ret;
823 	}
824 
825 	user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
826 
827 	return 0;
828 }
829 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
830 
atmel_pmecc_create(struct platform_device * pdev,const struct atmel_pmecc_caps * caps,int pmecc_res_idx,int errloc_res_idx)831 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,
832 					const struct atmel_pmecc_caps *caps,
833 					int pmecc_res_idx, int errloc_res_idx)
834 {
835 	struct device *dev = &pdev->dev;
836 	struct atmel_pmecc *pmecc;
837 
838 	pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
839 	if (!pmecc)
840 		return ERR_PTR(-ENOMEM);
841 
842 	pmecc->caps = caps;
843 	pmecc->dev = dev;
844 	mutex_init(&pmecc->lock);
845 
846 	pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx);
847 	if (IS_ERR(pmecc->regs.base))
848 		return ERR_CAST(pmecc->regs.base);
849 
850 	pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx);
851 	if (IS_ERR(pmecc->regs.errloc))
852 		return ERR_CAST(pmecc->regs.errloc);
853 
854 	/* Disable all interrupts before registering the PMECC handler. */
855 	writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
856 	atmel_pmecc_reset(pmecc);
857 
858 	return pmecc;
859 }
860 
devm_atmel_pmecc_put(struct device * dev,void * res)861 static void devm_atmel_pmecc_put(struct device *dev, void *res)
862 {
863 	struct atmel_pmecc **pmecc = res;
864 
865 	put_device((*pmecc)->dev);
866 }
867 
atmel_pmecc_get_by_node(struct device * userdev,struct device_node * np)868 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,
869 						   struct device_node *np)
870 {
871 	struct platform_device *pdev;
872 	struct atmel_pmecc *pmecc, **ptr;
873 	int ret;
874 
875 	pdev = of_find_device_by_node(np);
876 	if (!pdev)
877 		return ERR_PTR(-EPROBE_DEFER);
878 	pmecc = platform_get_drvdata(pdev);
879 	if (!pmecc) {
880 		ret = -EPROBE_DEFER;
881 		goto err_put_device;
882 	}
883 
884 	ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
885 	if (!ptr) {
886 		ret = -ENOMEM;
887 		goto err_put_device;
888 	}
889 
890 	*ptr = pmecc;
891 
892 	devres_add(userdev, ptr);
893 
894 	return pmecc;
895 
896 err_put_device:
897 	put_device(&pdev->dev);
898 	return ERR_PTR(ret);
899 }
900 
901 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
902 
903 static struct atmel_pmecc_caps at91sam9g45_caps = {
904 	.strengths = atmel_pmecc_strengths,
905 	.nstrengths = 5,
906 	.el_offset = 0x8c,
907 };
908 
909 static struct atmel_pmecc_caps sama5d4_caps = {
910 	.strengths = atmel_pmecc_strengths,
911 	.nstrengths = 5,
912 	.el_offset = 0x8c,
913 	.correct_erased_chunks = true,
914 };
915 
916 static struct atmel_pmecc_caps sama5d2_caps = {
917 	.strengths = atmel_pmecc_strengths,
918 	.nstrengths = 6,
919 	.el_offset = 0xac,
920 	.correct_erased_chunks = true,
921 };
922 
923 static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = {
924 	{ .compatible = "atmel,sama5d4-nand", &sama5d4_caps },
925 	{ .compatible = "atmel,sama5d2-nand", &sama5d2_caps },
926 	{ /* sentinel */ }
927 };
928 
devm_atmel_pmecc_get(struct device * userdev)929 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)
930 {
931 	struct atmel_pmecc *pmecc;
932 	struct device_node *np;
933 
934 	if (!userdev)
935 		return ERR_PTR(-EINVAL);
936 
937 	if (!userdev->of_node)
938 		return NULL;
939 
940 	np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);
941 	if (np) {
942 		pmecc = atmel_pmecc_get_by_node(userdev, np);
943 		of_node_put(np);
944 	} else {
945 		/*
946 		 * Support old DT bindings: in this case the PMECC iomem
947 		 * resources are directly defined in the user pdev at position
948 		 * 1 and 2. Extract all relevant information from there.
949 		 */
950 		struct platform_device *pdev = to_platform_device(userdev);
951 		const struct atmel_pmecc_caps *caps;
952 		const struct of_device_id *match;
953 
954 		/* No PMECC engine available. */
955 		if (!of_property_read_bool(userdev->of_node,
956 					   "atmel,has-pmecc"))
957 			return NULL;
958 
959 		caps = &at91sam9g45_caps;
960 
961 		/* Find the caps associated to the NAND dev node. */
962 		match = of_match_node(atmel_pmecc_legacy_match,
963 				      userdev->of_node);
964 		if (match && match->data)
965 			caps = match->data;
966 
967 		pmecc = atmel_pmecc_create(pdev, caps, 1, 2);
968 	}
969 
970 	return pmecc;
971 }
972 EXPORT_SYMBOL(devm_atmel_pmecc_get);
973 
974 static const struct of_device_id atmel_pmecc_match[] = {
975 	{ .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },
976 	{ .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },
977 	{ .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },
978 	{ /* sentinel */ }
979 };
980 MODULE_DEVICE_TABLE(of, atmel_pmecc_match);
981 
atmel_pmecc_probe(struct platform_device * pdev)982 static int atmel_pmecc_probe(struct platform_device *pdev)
983 {
984 	struct device *dev = &pdev->dev;
985 	const struct atmel_pmecc_caps *caps;
986 	struct atmel_pmecc *pmecc;
987 
988 	caps = of_device_get_match_data(&pdev->dev);
989 	if (!caps) {
990 		dev_err(dev, "Invalid caps\n");
991 		return -EINVAL;
992 	}
993 
994 	pmecc = atmel_pmecc_create(pdev, caps, 0, 1);
995 	if (IS_ERR(pmecc))
996 		return PTR_ERR(pmecc);
997 
998 	platform_set_drvdata(pdev, pmecc);
999 
1000 	return 0;
1001 }
1002 
1003 static struct platform_driver atmel_pmecc_driver = {
1004 	.driver = {
1005 		.name = "atmel-pmecc",
1006 		.of_match_table = atmel_pmecc_match,
1007 	},
1008 	.probe = atmel_pmecc_probe,
1009 };
1010 module_platform_driver(atmel_pmecc_driver);
1011 
1012 MODULE_LICENSE("GPL");
1013 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1014 MODULE_DESCRIPTION("PMECC engine driver");
1015 MODULE_ALIAS("platform:atmel_pmecc");
1016