xref: /linux/drivers/mtd/nand/raw/meson_nand.c (revision 9f2c9170934eace462499ba0bfe042cc72900173)
1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2 /*
3  * Amlogic Meson Nand Flash Controller Driver
4  *
5  * Copyright (c) 2018 Amlogic, inc.
6  * Author: Liang Yang <liang.yang@amlogic.com>
7  */
8 
9 #include <linux/platform_device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/clk.h>
13 #include <linux/clk-provider.h>
14 #include <linux/mtd/rawnand.h>
15 #include <linux/mtd/mtd.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/regmap.h>
18 #include <linux/slab.h>
19 #include <linux/module.h>
20 #include <linux/iopoll.h>
21 #include <linux/of.h>
22 #include <linux/of_device.h>
23 #include <linux/sched/task_stack.h>
24 
25 #define NFC_REG_CMD		0x00
26 #define NFC_CMD_IDLE		(0xc << 14)
27 #define NFC_CMD_CLE		(0x5 << 14)
28 #define NFC_CMD_ALE		(0x6 << 14)
29 #define NFC_CMD_ADL		((0 << 16) | (3 << 20))
30 #define NFC_CMD_ADH		((1 << 16) | (3 << 20))
31 #define NFC_CMD_AIL		((2 << 16) | (3 << 20))
32 #define NFC_CMD_AIH		((3 << 16) | (3 << 20))
33 #define NFC_CMD_SEED		((8 << 16) | (3 << 20))
34 #define NFC_CMD_M2N		((0 << 17) | (2 << 20))
35 #define NFC_CMD_N2M		((1 << 17) | (2 << 20))
36 #define NFC_CMD_RB		BIT(20)
37 #define NFC_CMD_SCRAMBLER_ENABLE	BIT(19)
38 #define NFC_CMD_SCRAMBLER_DISABLE	0
39 #define NFC_CMD_SHORTMODE_DISABLE	0
40 #define NFC_CMD_RB_INT		BIT(14)
41 
42 #define NFC_CMD_GET_SIZE(x)	(((x) >> 22) & GENMASK(4, 0))
43 
44 #define NFC_REG_CFG		0x04
45 #define NFC_REG_DADR		0x08
46 #define NFC_REG_IADR		0x0c
47 #define NFC_REG_BUF		0x10
48 #define NFC_REG_INFO		0x14
49 #define NFC_REG_DC		0x18
50 #define NFC_REG_ADR		0x1c
51 #define NFC_REG_DL		0x20
52 #define NFC_REG_DH		0x24
53 #define NFC_REG_CADR		0x28
54 #define NFC_REG_SADR		0x2c
55 #define NFC_REG_PINS		0x30
56 #define NFC_REG_VER		0x38
57 
58 #define NFC_RB_IRQ_EN		BIT(21)
59 
60 #define CLK_DIV_SHIFT		0
61 #define CLK_DIV_WIDTH		6
62 
63 #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)	\
64 	(								\
65 		(cmd_dir)			|			\
66 		((ran) << 19)			|			\
67 		((bch) << 14)			|			\
68 		((short_mode) << 13)		|			\
69 		(((page_size) & 0x7f) << 6)	|			\
70 		((pages) & 0x3f)					\
71 	)
72 
73 #define GENCMDDADDRL(adl, addr)		((adl) | ((addr) & 0xffff))
74 #define GENCMDDADDRH(adh, addr)		((adh) | (((addr) >> 16) & 0xffff))
75 #define GENCMDIADDRL(ail, addr)		((ail) | ((addr) & 0xffff))
76 #define GENCMDIADDRH(aih, addr)		((aih) | (((addr) >> 16) & 0xffff))
77 
78 #define DMA_DIR(dir)		((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
79 
80 #define ECC_CHECK_RETURN_FF	(-1)
81 
82 #define NAND_CE0		(0xe << 10)
83 #define NAND_CE1		(0xd << 10)
84 
85 #define DMA_BUSY_TIMEOUT	0x100000
86 #define CMD_FIFO_EMPTY_TIMEOUT	1000
87 
88 #define MAX_CE_NUM		2
89 
90 /* eMMC clock register, misc control */
91 #define CLK_SELECT_NAND		BIT(31)
92 
93 #define NFC_CLK_CYCLE		6
94 
95 /* nand flash controller delay 3 ns */
96 #define NFC_DEFAULT_DELAY	3000
97 
98 #define ROW_ADDER(page, index)	(((page) >> (8 * (index))) & 0xff)
99 #define MAX_CYCLE_ADDRS		5
100 #define DIRREAD			1
101 #define DIRWRITE		0
102 
103 #define ECC_PARITY_BCH8_512B	14
104 #define ECC_COMPLETE            BIT(31)
105 #define ECC_ERR_CNT(x)		(((x) >> 24) & GENMASK(5, 0))
106 #define ECC_ZERO_CNT(x)		(((x) >> 16) & GENMASK(5, 0))
107 #define ECC_UNCORRECTABLE	0x3f
108 
109 #define PER_INFO_BYTE		8
110 
111 struct meson_nfc_nand_chip {
112 	struct list_head node;
113 	struct nand_chip nand;
114 	unsigned long clk_rate;
115 	unsigned long level1_divider;
116 	u32 bus_timing;
117 	u32 twb;
118 	u32 tadl;
119 	u32 tbers_max;
120 
121 	u32 bch_mode;
122 	u8 *data_buf;
123 	__le64 *info_buf;
124 	u32 nsels;
125 	u8 sels[];
126 };
127 
128 struct meson_nand_ecc {
129 	u32 bch;
130 	u32 strength;
131 };
132 
133 struct meson_nfc_data {
134 	const struct nand_ecc_caps *ecc_caps;
135 };
136 
137 struct meson_nfc_param {
138 	u32 chip_select;
139 	u32 rb_select;
140 };
141 
142 struct nand_rw_cmd {
143 	u32 cmd0;
144 	u32 addrs[MAX_CYCLE_ADDRS];
145 	u32 cmd1;
146 };
147 
148 struct nand_timing {
149 	u32 twb;
150 	u32 tadl;
151 	u32 tbers_max;
152 };
153 
154 struct meson_nfc {
155 	struct nand_controller controller;
156 	struct clk *core_clk;
157 	struct clk *device_clk;
158 	struct clk *nand_clk;
159 	struct clk_divider nand_divider;
160 
161 	unsigned long clk_rate;
162 	u32 bus_timing;
163 
164 	struct device *dev;
165 	void __iomem *reg_base;
166 	void __iomem *reg_clk;
167 	struct completion completion;
168 	struct list_head chips;
169 	const struct meson_nfc_data *data;
170 	struct meson_nfc_param param;
171 	struct nand_timing timing;
172 	union {
173 		int cmd[32];
174 		struct nand_rw_cmd rw;
175 	} cmdfifo;
176 
177 	dma_addr_t daddr;
178 	dma_addr_t iaddr;
179 
180 	unsigned long assigned_cs;
181 };
182 
183 enum {
184 	NFC_ECC_BCH8_1K		= 2,
185 	NFC_ECC_BCH24_1K,
186 	NFC_ECC_BCH30_1K,
187 	NFC_ECC_BCH40_1K,
188 	NFC_ECC_BCH50_1K,
189 	NFC_ECC_BCH60_1K,
190 };
191 
192 #define MESON_ECC_DATA(b, s)	{ .bch = (b),	.strength = (s)}
193 
194 static struct meson_nand_ecc meson_ecc[] = {
195 	MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8),
196 	MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24),
197 	MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30),
198 	MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40),
199 	MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50),
200 	MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60),
201 };
202 
203 static int meson_nand_calc_ecc_bytes(int step_size, int strength)
204 {
205 	int ecc_bytes;
206 
207 	if (step_size == 512 && strength == 8)
208 		return ECC_PARITY_BCH8_512B;
209 
210 	ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
211 	ecc_bytes = ALIGN(ecc_bytes, 2);
212 
213 	return ecc_bytes;
214 }
215 
216 NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
217 		     meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
218 NAND_ECC_CAPS_SINGLE(meson_axg_ecc_caps,
219 		     meson_nand_calc_ecc_bytes, 1024, 8);
220 
221 static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
222 {
223 	return container_of(nand, struct meson_nfc_nand_chip, nand);
224 }
225 
226 static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
227 {
228 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
229 	struct meson_nfc *nfc = nand_get_controller_data(nand);
230 	int ret, value;
231 
232 	if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
233 		return;
234 
235 	nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
236 	nfc->param.rb_select = nfc->param.chip_select;
237 	nfc->timing.twb = meson_chip->twb;
238 	nfc->timing.tadl = meson_chip->tadl;
239 	nfc->timing.tbers_max = meson_chip->tbers_max;
240 
241 	if (nfc->clk_rate != meson_chip->clk_rate) {
242 		ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
243 		if (ret) {
244 			dev_err(nfc->dev, "failed to set clock rate\n");
245 			return;
246 		}
247 		nfc->clk_rate = meson_chip->clk_rate;
248 	}
249 	if (nfc->bus_timing != meson_chip->bus_timing) {
250 		value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
251 		writel(value, nfc->reg_base + NFC_REG_CFG);
252 		writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
253 		nfc->bus_timing =  meson_chip->bus_timing;
254 	}
255 }
256 
257 static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
258 {
259 	writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
260 	       nfc->reg_base + NFC_REG_CMD);
261 }
262 
263 static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
264 {
265 	writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
266 	       nfc->reg_base + NFC_REG_CMD);
267 }
268 
269 static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
270 				 int scrambler)
271 {
272 	struct mtd_info *mtd = nand_to_mtd(nand);
273 	struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
274 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
275 	u32 bch = meson_chip->bch_mode, cmd;
276 	int len = mtd->writesize, pagesize, pages;
277 
278 	pagesize = nand->ecc.size;
279 
280 	if (raw) {
281 		len = mtd->writesize + mtd->oobsize;
282 		cmd = (len & GENMASK(5, 0)) | scrambler | DMA_DIR(dir);
283 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
284 		return;
285 	}
286 
287 	pages = len / nand->ecc.size;
288 
289 	cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
290 		       NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
291 
292 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
293 }
294 
295 static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
296 {
297 	/*
298 	 * Insert two commands to make sure all valid commands are finished.
299 	 *
300 	 * The Nand flash controller is designed as two stages pipleline -
301 	 *  a) fetch and b) excute.
302 	 * There might be cases when the driver see command queue is empty,
303 	 * but the Nand flash controller still has two commands buffered,
304 	 * one is fetched into NFC request queue (ready to run), and another
305 	 * is actively executing. So pushing 2 "IDLE" commands guarantees that
306 	 * the pipeline is emptied.
307 	 */
308 	meson_nfc_cmd_idle(nfc, 0);
309 	meson_nfc_cmd_idle(nfc, 0);
310 }
311 
312 static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
313 				     unsigned int timeout_ms)
314 {
315 	u32 cmd_size = 0;
316 	int ret;
317 
318 	/* wait cmd fifo is empty */
319 	ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
320 					 !NFC_CMD_GET_SIZE(cmd_size),
321 					 10, timeout_ms * 1000);
322 	if (ret)
323 		dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
324 
325 	return ret;
326 }
327 
328 static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
329 {
330 	meson_nfc_drain_cmd(nfc);
331 
332 	return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
333 }
334 
335 static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
336 {
337 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
338 	int len;
339 
340 	len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
341 
342 	return meson_chip->data_buf + len;
343 }
344 
345 static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
346 {
347 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
348 	int len, temp;
349 
350 	temp = nand->ecc.size + nand->ecc.bytes;
351 	len = (temp + 2) * i;
352 
353 	return meson_chip->data_buf + len;
354 }
355 
356 static void meson_nfc_get_data_oob(struct nand_chip *nand,
357 				   u8 *buf, u8 *oobbuf)
358 {
359 	int i, oob_len = 0;
360 	u8 *dsrc, *osrc;
361 
362 	oob_len = nand->ecc.bytes + 2;
363 	for (i = 0; i < nand->ecc.steps; i++) {
364 		if (buf) {
365 			dsrc = meson_nfc_data_ptr(nand, i);
366 			memcpy(buf, dsrc, nand->ecc.size);
367 			buf += nand->ecc.size;
368 		}
369 		osrc = meson_nfc_oob_ptr(nand, i);
370 		memcpy(oobbuf, osrc, oob_len);
371 		oobbuf += oob_len;
372 	}
373 }
374 
375 static void meson_nfc_set_data_oob(struct nand_chip *nand,
376 				   const u8 *buf, u8 *oobbuf)
377 {
378 	int i, oob_len = 0;
379 	u8 *dsrc, *osrc;
380 
381 	oob_len = nand->ecc.bytes + 2;
382 	for (i = 0; i < nand->ecc.steps; i++) {
383 		if (buf) {
384 			dsrc = meson_nfc_data_ptr(nand, i);
385 			memcpy(dsrc, buf, nand->ecc.size);
386 			buf += nand->ecc.size;
387 		}
388 		osrc = meson_nfc_oob_ptr(nand, i);
389 		memcpy(osrc, oobbuf, oob_len);
390 		oobbuf += oob_len;
391 	}
392 }
393 
394 static int meson_nfc_queue_rb(struct meson_nfc *nfc, int timeout_ms)
395 {
396 	u32 cmd, cfg;
397 	int ret = 0;
398 
399 	meson_nfc_cmd_idle(nfc, nfc->timing.twb);
400 	meson_nfc_drain_cmd(nfc);
401 	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
402 
403 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
404 	cfg |= NFC_RB_IRQ_EN;
405 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
406 
407 	reinit_completion(&nfc->completion);
408 
409 	/* use the max erase time as the maximum clock for waiting R/B */
410 	cmd = NFC_CMD_RB | NFC_CMD_RB_INT
411 		| nfc->param.chip_select | nfc->timing.tbers_max;
412 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
413 
414 	ret = wait_for_completion_timeout(&nfc->completion,
415 					  msecs_to_jiffies(timeout_ms));
416 	if (ret == 0)
417 		ret = -1;
418 
419 	return ret;
420 }
421 
422 static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
423 {
424 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
425 	__le64 *info;
426 	int i, count;
427 
428 	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
429 		info = &meson_chip->info_buf[i];
430 		*info |= oob_buf[count];
431 		*info |= oob_buf[count + 1] << 8;
432 	}
433 }
434 
435 static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
436 {
437 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
438 	__le64 *info;
439 	int i, count;
440 
441 	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
442 		info = &meson_chip->info_buf[i];
443 		oob_buf[count] = *info;
444 		oob_buf[count + 1] = *info >> 8;
445 	}
446 }
447 
448 static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
449 				 u64 *correct_bitmap)
450 {
451 	struct mtd_info *mtd = nand_to_mtd(nand);
452 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
453 	__le64 *info;
454 	int ret = 0, i;
455 
456 	for (i = 0; i < nand->ecc.steps; i++) {
457 		info = &meson_chip->info_buf[i];
458 		if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
459 			mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
460 			*bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
461 			*correct_bitmap |= BIT_ULL(i);
462 			continue;
463 		}
464 		if ((nand->options & NAND_NEED_SCRAMBLING) &&
465 		    ECC_ZERO_CNT(*info) < nand->ecc.strength) {
466 			mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
467 			*bitflips = max_t(u32, *bitflips,
468 					  ECC_ZERO_CNT(*info));
469 			ret = ECC_CHECK_RETURN_FF;
470 		} else {
471 			ret = -EBADMSG;
472 		}
473 	}
474 	return ret;
475 }
476 
477 static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
478 				      int datalen, void *infobuf, int infolen,
479 				      enum dma_data_direction dir)
480 {
481 	struct meson_nfc *nfc = nand_get_controller_data(nand);
482 	u32 cmd;
483 	int ret = 0;
484 
485 	nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
486 	ret = dma_mapping_error(nfc->dev, nfc->daddr);
487 	if (ret) {
488 		dev_err(nfc->dev, "DMA mapping error\n");
489 		return ret;
490 	}
491 	cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
492 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
493 
494 	cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
495 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
496 
497 	if (infobuf) {
498 		nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
499 		ret = dma_mapping_error(nfc->dev, nfc->iaddr);
500 		if (ret) {
501 			dev_err(nfc->dev, "DMA mapping error\n");
502 			dma_unmap_single(nfc->dev,
503 					 nfc->daddr, datalen, dir);
504 			return ret;
505 		}
506 		cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
507 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
508 
509 		cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
510 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
511 	}
512 
513 	return ret;
514 }
515 
516 static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
517 					 int datalen, int infolen,
518 					 enum dma_data_direction dir)
519 {
520 	struct meson_nfc *nfc = nand_get_controller_data(nand);
521 
522 	dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
523 	if (infolen)
524 		dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
525 }
526 
527 static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
528 {
529 	struct meson_nfc *nfc = nand_get_controller_data(nand);
530 	int ret = 0;
531 	u32 cmd;
532 	u8 *info;
533 
534 	info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
535 	if (!info)
536 		return -ENOMEM;
537 
538 	ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
539 					 PER_INFO_BYTE, DMA_FROM_DEVICE);
540 	if (ret)
541 		goto out;
542 
543 	cmd = NFC_CMD_N2M | (len & GENMASK(5, 0));
544 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
545 
546 	meson_nfc_drain_cmd(nfc);
547 	meson_nfc_wait_cmd_finish(nfc, 1000);
548 	meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
549 
550 out:
551 	kfree(info);
552 
553 	return ret;
554 }
555 
556 static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
557 {
558 	struct meson_nfc *nfc = nand_get_controller_data(nand);
559 	int ret = 0;
560 	u32 cmd;
561 
562 	ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
563 					 0, DMA_TO_DEVICE);
564 	if (ret)
565 		return ret;
566 
567 	cmd = NFC_CMD_M2N | (len & GENMASK(5, 0));
568 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
569 
570 	meson_nfc_drain_cmd(nfc);
571 	meson_nfc_wait_cmd_finish(nfc, 1000);
572 	meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
573 
574 	return ret;
575 }
576 
577 static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
578 						int page, bool in)
579 {
580 	const struct nand_sdr_timings *sdr =
581 		nand_get_sdr_timings(nand_get_interface_config(nand));
582 	struct mtd_info *mtd = nand_to_mtd(nand);
583 	struct meson_nfc *nfc = nand_get_controller_data(nand);
584 	u32 *addrs = nfc->cmdfifo.rw.addrs;
585 	u32 cs = nfc->param.chip_select;
586 	u32 cmd0, cmd_num, row_start;
587 	int i;
588 
589 	cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
590 
591 	cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
592 	nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
593 
594 	addrs[0] = cs | NFC_CMD_ALE | 0;
595 	if (mtd->writesize <= 512) {
596 		cmd_num--;
597 		row_start = 1;
598 	} else {
599 		addrs[1] = cs | NFC_CMD_ALE | 0;
600 		row_start = 2;
601 	}
602 
603 	addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
604 	addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
605 
606 	if (nand->options & NAND_ROW_ADDR_3)
607 		addrs[row_start + 2] =
608 			cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
609 	else
610 		cmd_num--;
611 
612 	/* subtract cmd1 */
613 	cmd_num--;
614 
615 	for (i = 0; i < cmd_num; i++)
616 		writel_relaxed(nfc->cmdfifo.cmd[i],
617 			       nfc->reg_base + NFC_REG_CMD);
618 
619 	if (in) {
620 		nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
621 		writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
622 		meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tR_max));
623 	} else {
624 		meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
625 	}
626 
627 	return 0;
628 }
629 
630 static int meson_nfc_write_page_sub(struct nand_chip *nand,
631 				    int page, int raw)
632 {
633 	const struct nand_sdr_timings *sdr =
634 		nand_get_sdr_timings(nand_get_interface_config(nand));
635 	struct mtd_info *mtd = nand_to_mtd(nand);
636 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
637 	struct meson_nfc *nfc = nand_get_controller_data(nand);
638 	int data_len, info_len;
639 	u32 cmd;
640 	int ret;
641 
642 	meson_nfc_select_chip(nand, nand->cur_cs);
643 
644 	data_len =  mtd->writesize + mtd->oobsize;
645 	info_len = nand->ecc.steps * PER_INFO_BYTE;
646 
647 	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
648 	if (ret)
649 		return ret;
650 
651 	ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
652 					 data_len, meson_chip->info_buf,
653 					 info_len, DMA_TO_DEVICE);
654 	if (ret)
655 		return ret;
656 
657 	if (nand->options & NAND_NEED_SCRAMBLING) {
658 		meson_nfc_cmd_seed(nfc, page);
659 		meson_nfc_cmd_access(nand, raw, DIRWRITE,
660 				     NFC_CMD_SCRAMBLER_ENABLE);
661 	} else {
662 		meson_nfc_cmd_access(nand, raw, DIRWRITE,
663 				     NFC_CMD_SCRAMBLER_DISABLE);
664 	}
665 
666 	cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
667 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
668 	meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tPROG_max));
669 
670 	meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
671 
672 	return ret;
673 }
674 
675 static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
676 				    int oob_required, int page)
677 {
678 	u8 *oob_buf = nand->oob_poi;
679 
680 	meson_nfc_set_data_oob(nand, buf, oob_buf);
681 
682 	return meson_nfc_write_page_sub(nand, page, 1);
683 }
684 
685 static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
686 				      const u8 *buf, int oob_required, int page)
687 {
688 	struct mtd_info *mtd = nand_to_mtd(nand);
689 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
690 	u8 *oob_buf = nand->oob_poi;
691 
692 	memcpy(meson_chip->data_buf, buf, mtd->writesize);
693 	memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
694 	meson_nfc_set_user_byte(nand, oob_buf);
695 
696 	return meson_nfc_write_page_sub(nand, page, 0);
697 }
698 
699 static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
700 					    struct nand_chip *nand, int raw)
701 {
702 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
703 	__le64 *info;
704 	u32 neccpages;
705 	int ret;
706 
707 	neccpages = raw ? 1 : nand->ecc.steps;
708 	info = &meson_chip->info_buf[neccpages - 1];
709 	do {
710 		usleep_range(10, 15);
711 		/* info is updated by nfc dma engine*/
712 		smp_rmb();
713 		ret = *info & ECC_COMPLETE;
714 	} while (!ret);
715 }
716 
717 static int meson_nfc_read_page_sub(struct nand_chip *nand,
718 				   int page, int raw)
719 {
720 	struct mtd_info *mtd = nand_to_mtd(nand);
721 	struct meson_nfc *nfc = nand_get_controller_data(nand);
722 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
723 	int data_len, info_len;
724 	int ret;
725 
726 	meson_nfc_select_chip(nand, nand->cur_cs);
727 
728 	data_len =  mtd->writesize + mtd->oobsize;
729 	info_len = nand->ecc.steps * PER_INFO_BYTE;
730 
731 	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
732 	if (ret)
733 		return ret;
734 
735 	ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
736 					 data_len, meson_chip->info_buf,
737 					 info_len, DMA_FROM_DEVICE);
738 	if (ret)
739 		return ret;
740 
741 	if (nand->options & NAND_NEED_SCRAMBLING) {
742 		meson_nfc_cmd_seed(nfc, page);
743 		meson_nfc_cmd_access(nand, raw, DIRREAD,
744 				     NFC_CMD_SCRAMBLER_ENABLE);
745 	} else {
746 		meson_nfc_cmd_access(nand, raw, DIRREAD,
747 				     NFC_CMD_SCRAMBLER_DISABLE);
748 	}
749 
750 	ret = meson_nfc_wait_dma_finish(nfc);
751 	meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
752 
753 	meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
754 
755 	return ret;
756 }
757 
758 static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
759 				   int oob_required, int page)
760 {
761 	u8 *oob_buf = nand->oob_poi;
762 	int ret;
763 
764 	ret = meson_nfc_read_page_sub(nand, page, 1);
765 	if (ret)
766 		return ret;
767 
768 	meson_nfc_get_data_oob(nand, buf, oob_buf);
769 
770 	return 0;
771 }
772 
773 static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
774 				     int oob_required, int page)
775 {
776 	struct mtd_info *mtd = nand_to_mtd(nand);
777 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
778 	struct nand_ecc_ctrl *ecc = &nand->ecc;
779 	u64 correct_bitmap = 0;
780 	u32 bitflips = 0;
781 	u8 *oob_buf = nand->oob_poi;
782 	int ret, i;
783 
784 	ret = meson_nfc_read_page_sub(nand, page, 0);
785 	if (ret)
786 		return ret;
787 
788 	meson_nfc_get_user_byte(nand, oob_buf);
789 	ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
790 	if (ret == ECC_CHECK_RETURN_FF) {
791 		if (buf)
792 			memset(buf, 0xff, mtd->writesize);
793 		memset(oob_buf, 0xff, mtd->oobsize);
794 	} else if (ret < 0) {
795 		if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
796 			mtd->ecc_stats.failed++;
797 			return bitflips;
798 		}
799 		ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
800 		if (ret)
801 			return ret;
802 
803 		for (i = 0; i < nand->ecc.steps ; i++) {
804 			u8 *data = buf + i * ecc->size;
805 			u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
806 
807 			if (correct_bitmap & BIT_ULL(i))
808 				continue;
809 			ret = nand_check_erased_ecc_chunk(data,	ecc->size,
810 							  oob, ecc->bytes + 2,
811 							  NULL, 0,
812 							  ecc->strength);
813 			if (ret < 0) {
814 				mtd->ecc_stats.failed++;
815 			} else {
816 				mtd->ecc_stats.corrected += ret;
817 				bitflips =  max_t(u32, bitflips, ret);
818 			}
819 		}
820 	} else if (buf && buf != meson_chip->data_buf) {
821 		memcpy(buf, meson_chip->data_buf, mtd->writesize);
822 	}
823 
824 	return bitflips;
825 }
826 
827 static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
828 {
829 	return meson_nfc_read_page_raw(nand, NULL, 1, page);
830 }
831 
832 static int meson_nfc_read_oob(struct nand_chip *nand, int page)
833 {
834 	return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
835 }
836 
837 static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
838 {
839 	if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
840 		return true;
841 	return false;
842 }
843 
844 static void *
845 meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
846 {
847 	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
848 		return NULL;
849 
850 	if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
851 		return instr->ctx.data.buf.in;
852 
853 	return kzalloc(instr->ctx.data.len, GFP_KERNEL);
854 }
855 
856 static void
857 meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
858 				     void *buf)
859 {
860 	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
861 	    WARN_ON(!buf))
862 		return;
863 
864 	if (buf == instr->ctx.data.buf.in)
865 		return;
866 
867 	memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
868 	kfree(buf);
869 }
870 
871 static void *
872 meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
873 {
874 	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
875 		return NULL;
876 
877 	if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
878 		return (void *)instr->ctx.data.buf.out;
879 
880 	return kmemdup(instr->ctx.data.buf.out,
881 		       instr->ctx.data.len, GFP_KERNEL);
882 }
883 
884 static void
885 meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
886 				      const void *buf)
887 {
888 	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
889 	    WARN_ON(!buf))
890 		return;
891 
892 	if (buf != instr->ctx.data.buf.out)
893 		kfree(buf);
894 }
895 
896 static int meson_nfc_exec_op(struct nand_chip *nand,
897 			     const struct nand_operation *op, bool check_only)
898 {
899 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
900 	struct meson_nfc *nfc = nand_get_controller_data(nand);
901 	const struct nand_op_instr *instr = NULL;
902 	void *buf;
903 	u32 op_id, delay_idle, cmd;
904 	int i;
905 
906 	if (check_only)
907 		return 0;
908 
909 	meson_nfc_select_chip(nand, op->cs);
910 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
911 		instr = &op->instrs[op_id];
912 		delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
913 					  meson_chip->level1_divider *
914 					  NFC_CLK_CYCLE);
915 		switch (instr->type) {
916 		case NAND_OP_CMD_INSTR:
917 			cmd = nfc->param.chip_select | NFC_CMD_CLE;
918 			cmd |= instr->ctx.cmd.opcode & 0xff;
919 			writel(cmd, nfc->reg_base + NFC_REG_CMD);
920 			meson_nfc_cmd_idle(nfc, delay_idle);
921 			break;
922 
923 		case NAND_OP_ADDR_INSTR:
924 			for (i = 0; i < instr->ctx.addr.naddrs; i++) {
925 				cmd = nfc->param.chip_select | NFC_CMD_ALE;
926 				cmd |= instr->ctx.addr.addrs[i] & 0xff;
927 				writel(cmd, nfc->reg_base + NFC_REG_CMD);
928 			}
929 			meson_nfc_cmd_idle(nfc, delay_idle);
930 			break;
931 
932 		case NAND_OP_DATA_IN_INSTR:
933 			buf = meson_nand_op_get_dma_safe_input_buf(instr);
934 			if (!buf)
935 				return -ENOMEM;
936 			meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
937 			meson_nand_op_put_dma_safe_input_buf(instr, buf);
938 			break;
939 
940 		case NAND_OP_DATA_OUT_INSTR:
941 			buf = meson_nand_op_get_dma_safe_output_buf(instr);
942 			if (!buf)
943 				return -ENOMEM;
944 			meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
945 			meson_nand_op_put_dma_safe_output_buf(instr, buf);
946 			break;
947 
948 		case NAND_OP_WAITRDY_INSTR:
949 			meson_nfc_queue_rb(nfc, instr->ctx.waitrdy.timeout_ms);
950 			if (instr->delay_ns)
951 				meson_nfc_cmd_idle(nfc, delay_idle);
952 			break;
953 		}
954 	}
955 	meson_nfc_wait_cmd_finish(nfc, 1000);
956 	return 0;
957 }
958 
959 static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
960 			       struct mtd_oob_region *oobregion)
961 {
962 	struct nand_chip *nand = mtd_to_nand(mtd);
963 
964 	if (section >= nand->ecc.steps)
965 		return -ERANGE;
966 
967 	oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
968 	oobregion->length = nand->ecc.bytes;
969 
970 	return 0;
971 }
972 
973 static int meson_ooblayout_free(struct mtd_info *mtd, int section,
974 				struct mtd_oob_region *oobregion)
975 {
976 	struct nand_chip *nand = mtd_to_nand(mtd);
977 
978 	if (section >= nand->ecc.steps)
979 		return -ERANGE;
980 
981 	oobregion->offset = section * (2 + nand->ecc.bytes);
982 	oobregion->length = 2;
983 
984 	return 0;
985 }
986 
987 static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
988 	.ecc = meson_ooblayout_ecc,
989 	.free = meson_ooblayout_free,
990 };
991 
992 static int meson_nfc_clk_init(struct meson_nfc *nfc)
993 {
994 	struct clk_parent_data nfc_divider_parent_data[1];
995 	struct clk_init_data init = {0};
996 	int ret;
997 
998 	/* request core clock */
999 	nfc->core_clk = devm_clk_get(nfc->dev, "core");
1000 	if (IS_ERR(nfc->core_clk)) {
1001 		dev_err(nfc->dev, "failed to get core clock\n");
1002 		return PTR_ERR(nfc->core_clk);
1003 	}
1004 
1005 	nfc->device_clk = devm_clk_get(nfc->dev, "device");
1006 	if (IS_ERR(nfc->device_clk)) {
1007 		dev_err(nfc->dev, "failed to get device clock\n");
1008 		return PTR_ERR(nfc->device_clk);
1009 	}
1010 
1011 	init.name = devm_kasprintf(nfc->dev,
1012 				   GFP_KERNEL, "%s#div",
1013 				   dev_name(nfc->dev));
1014 	init.ops = &clk_divider_ops;
1015 	nfc_divider_parent_data[0].fw_name = "device";
1016 	init.parent_data = nfc_divider_parent_data;
1017 	init.num_parents = 1;
1018 	nfc->nand_divider.reg = nfc->reg_clk;
1019 	nfc->nand_divider.shift = CLK_DIV_SHIFT;
1020 	nfc->nand_divider.width = CLK_DIV_WIDTH;
1021 	nfc->nand_divider.hw.init = &init;
1022 	nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1023 				  CLK_DIVIDER_ROUND_CLOSEST |
1024 				  CLK_DIVIDER_ALLOW_ZERO;
1025 
1026 	nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1027 	if (IS_ERR(nfc->nand_clk))
1028 		return PTR_ERR(nfc->nand_clk);
1029 
1030 	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1031 	writel(CLK_SELECT_NAND | readl(nfc->reg_clk),
1032 	       nfc->reg_clk);
1033 
1034 	ret = clk_prepare_enable(nfc->core_clk);
1035 	if (ret) {
1036 		dev_err(nfc->dev, "failed to enable core clock\n");
1037 		return ret;
1038 	}
1039 
1040 	ret = clk_prepare_enable(nfc->device_clk);
1041 	if (ret) {
1042 		dev_err(nfc->dev, "failed to enable device clock\n");
1043 		goto err_device_clk;
1044 	}
1045 
1046 	ret = clk_prepare_enable(nfc->nand_clk);
1047 	if (ret) {
1048 		dev_err(nfc->dev, "pre enable NFC divider fail\n");
1049 		goto err_nand_clk;
1050 	}
1051 
1052 	ret = clk_set_rate(nfc->nand_clk, 24000000);
1053 	if (ret)
1054 		goto err_disable_clk;
1055 
1056 	return 0;
1057 
1058 err_disable_clk:
1059 	clk_disable_unprepare(nfc->nand_clk);
1060 err_nand_clk:
1061 	clk_disable_unprepare(nfc->device_clk);
1062 err_device_clk:
1063 	clk_disable_unprepare(nfc->core_clk);
1064 	return ret;
1065 }
1066 
1067 static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1068 {
1069 	clk_disable_unprepare(nfc->nand_clk);
1070 	clk_disable_unprepare(nfc->device_clk);
1071 	clk_disable_unprepare(nfc->core_clk);
1072 }
1073 
1074 static void meson_nfc_free_buffer(struct nand_chip *nand)
1075 {
1076 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1077 
1078 	kfree(meson_chip->info_buf);
1079 	kfree(meson_chip->data_buf);
1080 }
1081 
1082 static int meson_chip_buffer_init(struct nand_chip *nand)
1083 {
1084 	struct mtd_info *mtd = nand_to_mtd(nand);
1085 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1086 	u32 page_bytes, info_bytes, nsectors;
1087 
1088 	nsectors = mtd->writesize / nand->ecc.size;
1089 
1090 	page_bytes =  mtd->writesize + mtd->oobsize;
1091 	info_bytes = nsectors * PER_INFO_BYTE;
1092 
1093 	meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1094 	if (!meson_chip->data_buf)
1095 		return -ENOMEM;
1096 
1097 	meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1098 	if (!meson_chip->info_buf) {
1099 		kfree(meson_chip->data_buf);
1100 		return -ENOMEM;
1101 	}
1102 
1103 	return 0;
1104 }
1105 
1106 static
1107 int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1108 			      const struct nand_interface_config *conf)
1109 {
1110 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1111 	const struct nand_sdr_timings *timings;
1112 	u32 div, bt_min, bt_max, tbers_clocks;
1113 
1114 	timings = nand_get_sdr_timings(conf);
1115 	if (IS_ERR(timings))
1116 		return -ENOTSUPP;
1117 
1118 	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1119 		return 0;
1120 
1121 	div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1122 	bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1123 	bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1124 		  timings->tRC_min / 2) / div;
1125 
1126 	meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1127 				       div * NFC_CLK_CYCLE);
1128 	meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1129 					div * NFC_CLK_CYCLE);
1130 	tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1131 					div * NFC_CLK_CYCLE);
1132 	meson_chip->tbers_max = ilog2(tbers_clocks);
1133 	if (!is_power_of_2(tbers_clocks))
1134 		meson_chip->tbers_max++;
1135 
1136 	bt_min = DIV_ROUND_UP(bt_min, 1000);
1137 	bt_max = DIV_ROUND_UP(bt_max, 1000);
1138 
1139 	if (bt_max < bt_min)
1140 		return -EINVAL;
1141 
1142 	meson_chip->level1_divider = div;
1143 	meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1144 	meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1145 
1146 	return 0;
1147 }
1148 
1149 static int meson_nand_bch_mode(struct nand_chip *nand)
1150 {
1151 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1152 	int i;
1153 
1154 	if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1155 		return -EINVAL;
1156 
1157 	for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1158 		if (meson_ecc[i].strength == nand->ecc.strength) {
1159 			meson_chip->bch_mode = meson_ecc[i].bch;
1160 			return 0;
1161 		}
1162 	}
1163 
1164 	return -EINVAL;
1165 }
1166 
1167 static void meson_nand_detach_chip(struct nand_chip *nand)
1168 {
1169 	meson_nfc_free_buffer(nand);
1170 }
1171 
1172 static int meson_nand_attach_chip(struct nand_chip *nand)
1173 {
1174 	struct meson_nfc *nfc = nand_get_controller_data(nand);
1175 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1176 	struct mtd_info *mtd = nand_to_mtd(nand);
1177 	int nsectors = mtd->writesize / 1024;
1178 	int ret;
1179 
1180 	if (!mtd->name) {
1181 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1182 					   "%s:nand%d",
1183 					   dev_name(nfc->dev),
1184 					   meson_chip->sels[0]);
1185 		if (!mtd->name)
1186 			return -ENOMEM;
1187 	}
1188 
1189 	if (nand->bbt_options & NAND_BBT_USE_FLASH)
1190 		nand->bbt_options |= NAND_BBT_NO_OOB;
1191 
1192 	nand->options |= NAND_NO_SUBPAGE_WRITE;
1193 
1194 	ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1195 				   mtd->oobsize - 2 * nsectors);
1196 	if (ret) {
1197 		dev_err(nfc->dev, "failed to ECC init\n");
1198 		return -EINVAL;
1199 	}
1200 
1201 	mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1202 
1203 	ret = meson_nand_bch_mode(nand);
1204 	if (ret)
1205 		return -EINVAL;
1206 
1207 	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1208 	nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1209 	nand->ecc.write_page = meson_nfc_write_page_hwecc;
1210 	nand->ecc.write_oob_raw = nand_write_oob_std;
1211 	nand->ecc.write_oob = nand_write_oob_std;
1212 
1213 	nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1214 	nand->ecc.read_page = meson_nfc_read_page_hwecc;
1215 	nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1216 	nand->ecc.read_oob = meson_nfc_read_oob;
1217 
1218 	if (nand->options & NAND_BUSWIDTH_16) {
1219 		dev_err(nfc->dev, "16bits bus width not supported");
1220 		return -EINVAL;
1221 	}
1222 	ret = meson_chip_buffer_init(nand);
1223 	if (ret)
1224 		return -ENOMEM;
1225 
1226 	return ret;
1227 }
1228 
1229 static const struct nand_controller_ops meson_nand_controller_ops = {
1230 	.attach_chip = meson_nand_attach_chip,
1231 	.detach_chip = meson_nand_detach_chip,
1232 	.setup_interface = meson_nfc_setup_interface,
1233 	.exec_op = meson_nfc_exec_op,
1234 };
1235 
1236 static int
1237 meson_nfc_nand_chip_init(struct device *dev,
1238 			 struct meson_nfc *nfc, struct device_node *np)
1239 {
1240 	struct meson_nfc_nand_chip *meson_chip;
1241 	struct nand_chip *nand;
1242 	struct mtd_info *mtd;
1243 	int ret, i;
1244 	u32 tmp, nsels;
1245 
1246 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1247 	if (!nsels || nsels > MAX_CE_NUM) {
1248 		dev_err(dev, "invalid register property size\n");
1249 		return -EINVAL;
1250 	}
1251 
1252 	meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1253 				  GFP_KERNEL);
1254 	if (!meson_chip)
1255 		return -ENOMEM;
1256 
1257 	meson_chip->nsels = nsels;
1258 
1259 	for (i = 0; i < nsels; i++) {
1260 		ret = of_property_read_u32_index(np, "reg", i, &tmp);
1261 		if (ret) {
1262 			dev_err(dev, "could not retrieve register property: %d\n",
1263 				ret);
1264 			return ret;
1265 		}
1266 
1267 		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1268 			dev_err(dev, "CS %d already assigned\n", tmp);
1269 			return -EINVAL;
1270 		}
1271 	}
1272 
1273 	nand = &meson_chip->nand;
1274 	nand->controller = &nfc->controller;
1275 	nand->controller->ops = &meson_nand_controller_ops;
1276 	nand_set_flash_node(nand, np);
1277 	nand_set_controller_data(nand, nfc);
1278 
1279 	nand->options |= NAND_USES_DMA;
1280 	mtd = nand_to_mtd(nand);
1281 	mtd->owner = THIS_MODULE;
1282 	mtd->dev.parent = dev;
1283 
1284 	ret = nand_scan(nand, nsels);
1285 	if (ret)
1286 		return ret;
1287 
1288 	ret = mtd_device_register(mtd, NULL, 0);
1289 	if (ret) {
1290 		dev_err(dev, "failed to register MTD device: %d\n", ret);
1291 		nand_cleanup(nand);
1292 		return ret;
1293 	}
1294 
1295 	list_add_tail(&meson_chip->node, &nfc->chips);
1296 
1297 	return 0;
1298 }
1299 
1300 static void meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1301 {
1302 	struct meson_nfc_nand_chip *meson_chip;
1303 	struct mtd_info *mtd;
1304 
1305 	while (!list_empty(&nfc->chips)) {
1306 		meson_chip = list_first_entry(&nfc->chips,
1307 					      struct meson_nfc_nand_chip, node);
1308 		mtd = nand_to_mtd(&meson_chip->nand);
1309 		WARN_ON(mtd_device_unregister(mtd));
1310 
1311 		nand_cleanup(&meson_chip->nand);
1312 		list_del(&meson_chip->node);
1313 	}
1314 }
1315 
1316 static int meson_nfc_nand_chips_init(struct device *dev,
1317 				     struct meson_nfc *nfc)
1318 {
1319 	struct device_node *np = dev->of_node;
1320 	struct device_node *nand_np;
1321 	int ret;
1322 
1323 	for_each_child_of_node(np, nand_np) {
1324 		ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1325 		if (ret) {
1326 			meson_nfc_nand_chip_cleanup(nfc);
1327 			of_node_put(nand_np);
1328 			return ret;
1329 		}
1330 	}
1331 
1332 	return 0;
1333 }
1334 
1335 static irqreturn_t meson_nfc_irq(int irq, void *id)
1336 {
1337 	struct meson_nfc *nfc = id;
1338 	u32 cfg;
1339 
1340 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
1341 	if (!(cfg & NFC_RB_IRQ_EN))
1342 		return IRQ_NONE;
1343 
1344 	cfg &= ~(NFC_RB_IRQ_EN);
1345 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
1346 
1347 	complete(&nfc->completion);
1348 	return IRQ_HANDLED;
1349 }
1350 
1351 static const struct meson_nfc_data meson_gxl_data = {
1352 	.ecc_caps = &meson_gxl_ecc_caps,
1353 };
1354 
1355 static const struct meson_nfc_data meson_axg_data = {
1356 	.ecc_caps = &meson_axg_ecc_caps,
1357 };
1358 
1359 static const struct of_device_id meson_nfc_id_table[] = {
1360 	{
1361 		.compatible = "amlogic,meson-gxl-nfc",
1362 		.data = &meson_gxl_data,
1363 	}, {
1364 		.compatible = "amlogic,meson-axg-nfc",
1365 		.data = &meson_axg_data,
1366 	},
1367 	{}
1368 };
1369 MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1370 
1371 static int meson_nfc_probe(struct platform_device *pdev)
1372 {
1373 	struct device *dev = &pdev->dev;
1374 	struct meson_nfc *nfc;
1375 	int ret, irq;
1376 
1377 	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1378 	if (!nfc)
1379 		return -ENOMEM;
1380 
1381 	nfc->data = of_device_get_match_data(&pdev->dev);
1382 	if (!nfc->data)
1383 		return -ENODEV;
1384 
1385 	nand_controller_init(&nfc->controller);
1386 	INIT_LIST_HEAD(&nfc->chips);
1387 	init_completion(&nfc->completion);
1388 
1389 	nfc->dev = dev;
1390 
1391 	nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1392 	if (IS_ERR(nfc->reg_base))
1393 		return PTR_ERR(nfc->reg_base);
1394 
1395 	nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1396 	if (IS_ERR(nfc->reg_clk))
1397 		return PTR_ERR(nfc->reg_clk);
1398 
1399 	irq = platform_get_irq(pdev, 0);
1400 	if (irq < 0)
1401 		return -EINVAL;
1402 
1403 	ret = meson_nfc_clk_init(nfc);
1404 	if (ret) {
1405 		dev_err(dev, "failed to initialize NAND clock\n");
1406 		return ret;
1407 	}
1408 
1409 	writel(0, nfc->reg_base + NFC_REG_CFG);
1410 	ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1411 	if (ret) {
1412 		dev_err(dev, "failed to request NFC IRQ\n");
1413 		ret = -EINVAL;
1414 		goto err_clk;
1415 	}
1416 
1417 	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1418 	if (ret) {
1419 		dev_err(dev, "failed to set DMA mask\n");
1420 		goto err_clk;
1421 	}
1422 
1423 	platform_set_drvdata(pdev, nfc);
1424 
1425 	ret = meson_nfc_nand_chips_init(dev, nfc);
1426 	if (ret) {
1427 		dev_err(dev, "failed to init NAND chips\n");
1428 		goto err_clk;
1429 	}
1430 
1431 	return 0;
1432 err_clk:
1433 	meson_nfc_disable_clk(nfc);
1434 	return ret;
1435 }
1436 
1437 static int meson_nfc_remove(struct platform_device *pdev)
1438 {
1439 	struct meson_nfc *nfc = platform_get_drvdata(pdev);
1440 
1441 	meson_nfc_nand_chip_cleanup(nfc);
1442 
1443 	meson_nfc_disable_clk(nfc);
1444 
1445 	return 0;
1446 }
1447 
1448 static struct platform_driver meson_nfc_driver = {
1449 	.probe  = meson_nfc_probe,
1450 	.remove = meson_nfc_remove,
1451 	.driver = {
1452 		.name  = "meson-nand",
1453 		.of_match_table = meson_nfc_id_table,
1454 	},
1455 };
1456 module_platform_driver(meson_nfc_driver);
1457 
1458 MODULE_LICENSE("Dual MIT/GPL");
1459 MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1460 MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");
1461