xref: /linux/drivers/mtd/nand/raw/meson_nand.c (revision d0c9a21c8e0b2d7c55a2174f47bd0ea1d7302de6)
1 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2 /*
3  * Amlogic Meson Nand Flash Controller Driver
4  *
5  * Copyright (c) 2018 Amlogic, inc.
6  * Author: Liang Yang <liang.yang@amlogic.com>
7  */
8 
9 #include <linux/platform_device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/clk.h>
13 #include <linux/clk-provider.h>
14 #include <linux/mtd/rawnand.h>
15 #include <linux/mtd/mtd.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/regmap.h>
18 #include <linux/slab.h>
19 #include <linux/module.h>
20 #include <linux/iopoll.h>
21 #include <linux/of.h>
22 #include <linux/sched/task_stack.h>
23 
24 #define NFC_REG_CMD		0x00
25 #define NFC_CMD_IDLE		(0xc << 14)
26 #define NFC_CMD_CLE		(0x5 << 14)
27 #define NFC_CMD_ALE		(0x6 << 14)
28 #define NFC_CMD_ADL		((0 << 16) | (3 << 20))
29 #define NFC_CMD_ADH		((1 << 16) | (3 << 20))
30 #define NFC_CMD_AIL		((2 << 16) | (3 << 20))
31 #define NFC_CMD_AIH		((3 << 16) | (3 << 20))
32 #define NFC_CMD_SEED		((8 << 16) | (3 << 20))
33 #define NFC_CMD_M2N		((0 << 17) | (2 << 20))
34 #define NFC_CMD_N2M		((1 << 17) | (2 << 20))
35 #define NFC_CMD_RB		BIT(20)
36 #define NFC_CMD_SCRAMBLER_ENABLE	BIT(19)
37 #define NFC_CMD_SCRAMBLER_DISABLE	0
38 #define NFC_CMD_SHORTMODE_ENABLE	1
39 #define NFC_CMD_SHORTMODE_DISABLE	0
40 #define NFC_CMD_RB_INT		BIT(14)
41 #define NFC_CMD_RB_INT_NO_PIN	((0xb << 10) | BIT(18) | BIT(16))
42 
43 #define NFC_CMD_GET_SIZE(x)	(((x) >> 22) & GENMASK(4, 0))
44 
45 #define NFC_REG_CFG		0x04
46 #define NFC_REG_DADR		0x08
47 #define NFC_REG_IADR		0x0c
48 #define NFC_REG_BUF		0x10
49 #define NFC_REG_INFO		0x14
50 #define NFC_REG_DC		0x18
51 #define NFC_REG_ADR		0x1c
52 #define NFC_REG_DL		0x20
53 #define NFC_REG_DH		0x24
54 #define NFC_REG_CADR		0x28
55 #define NFC_REG_SADR		0x2c
56 #define NFC_REG_PINS		0x30
57 #define NFC_REG_VER		0x38
58 
59 #define NFC_RB_IRQ_EN		BIT(21)
60 
61 #define CLK_DIV_SHIFT		0
62 #define CLK_DIV_WIDTH		6
63 
64 #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)	\
65 	(								\
66 		(cmd_dir)			|			\
67 		(ran)				|			\
68 		((bch) << 14)			|			\
69 		((short_mode) << 13)		|			\
70 		(((page_size) & 0x7f) << 6)	|			\
71 		((pages) & 0x3f)					\
72 	)
73 
74 #define GENCMDDADDRL(adl, addr)		((adl) | ((addr) & 0xffff))
75 #define GENCMDDADDRH(adh, addr)		((adh) | (((addr) >> 16) & 0xffff))
76 #define GENCMDIADDRL(ail, addr)		((ail) | ((addr) & 0xffff))
77 #define GENCMDIADDRH(aih, addr)		((aih) | (((addr) >> 16) & 0xffff))
78 
79 #define DMA_DIR(dir)		((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
80 #define DMA_ADDR_ALIGN		8
81 
82 #define NFC_SHORT_MODE_ECC_SZ	384
83 
84 #define ECC_CHECK_RETURN_FF	(-1)
85 
86 #define NAND_CE0		(0xe << 10)
87 #define NAND_CE1		(0xd << 10)
88 
89 #define DMA_BUSY_TIMEOUT	0x100000
90 #define CMD_FIFO_EMPTY_TIMEOUT	1000
91 
92 #define MAX_CE_NUM		2
93 
94 /* eMMC clock register, misc control */
95 #define CLK_SELECT_NAND		BIT(31)
96 #define CLK_ALWAYS_ON_NAND	BIT(24)
97 #define CLK_SELECT_FIX_PLL2	BIT(6)
98 
99 #define NFC_CLK_CYCLE		6
100 
101 /* nand flash controller delay 3 ns */
102 #define NFC_DEFAULT_DELAY	3000
103 
104 #define ROW_ADDER(page, index)	(((page) >> (8 * (index))) & 0xff)
105 #define MAX_CYCLE_ADDRS		5
106 #define DIRREAD			1
107 #define DIRWRITE		0
108 
109 #define ECC_PARITY_BCH8_512B	14
110 #define ECC_COMPLETE            BIT(31)
111 #define ECC_ERR_CNT(x)		(((x) >> 24) & GENMASK(5, 0))
112 #define ECC_ZERO_CNT(x)		(((x) >> 16) & GENMASK(5, 0))
113 #define ECC_UNCORRECTABLE	0x3f
114 
115 #define PER_INFO_BYTE		8
116 
117 #define NFC_CMD_RAW_LEN	GENMASK(13, 0)
118 
119 #define NFC_COLUMN_ADDR_0	0
120 #define NFC_COLUMN_ADDR_1	0
121 
122 struct meson_nfc_nand_chip {
123 	struct list_head node;
124 	struct nand_chip nand;
125 	unsigned long clk_rate;
126 	unsigned long level1_divider;
127 	u32 bus_timing;
128 	u32 twb;
129 	u32 tadl;
130 	u32 tbers_max;
131 	u32 boot_pages;
132 	u32 boot_page_step;
133 
134 	u32 bch_mode;
135 	u8 *data_buf;
136 	__le64 *info_buf;
137 	u32 nsels;
138 	u8 sels[] __counted_by(nsels);
139 };
140 
141 struct meson_nand_ecc {
142 	u32 bch;
143 	u32 strength;
144 	u32 size;
145 };
146 
147 struct meson_nfc_data {
148 	const struct nand_ecc_caps *ecc_caps;
149 };
150 
151 struct meson_nfc_param {
152 	u32 chip_select;
153 	u32 rb_select;
154 };
155 
156 struct nand_rw_cmd {
157 	u32 cmd0;
158 	u32 addrs[MAX_CYCLE_ADDRS];
159 	u32 cmd1;
160 };
161 
162 struct nand_timing {
163 	u32 twb;
164 	u32 tadl;
165 	u32 tbers_max;
166 };
167 
168 struct meson_nfc {
169 	struct nand_controller controller;
170 	struct clk *core_clk;
171 	struct clk *device_clk;
172 	struct clk *nand_clk;
173 	struct clk_divider nand_divider;
174 
175 	unsigned long clk_rate;
176 	u32 bus_timing;
177 
178 	struct device *dev;
179 	void __iomem *reg_base;
180 	void __iomem *reg_clk;
181 	struct completion completion;
182 	struct list_head chips;
183 	const struct meson_nfc_data *data;
184 	struct meson_nfc_param param;
185 	struct nand_timing timing;
186 	union {
187 		int cmd[32];
188 		struct nand_rw_cmd rw;
189 	} cmdfifo;
190 
191 	dma_addr_t daddr;
192 	dma_addr_t iaddr;
193 	u32 info_bytes;
194 
195 	unsigned long assigned_cs;
196 	bool no_rb_pin;
197 };
198 
199 enum {
200 	NFC_ECC_BCH8_512	= 1,
201 	NFC_ECC_BCH8_1K,
202 	NFC_ECC_BCH24_1K,
203 	NFC_ECC_BCH30_1K,
204 	NFC_ECC_BCH40_1K,
205 	NFC_ECC_BCH50_1K,
206 	NFC_ECC_BCH60_1K,
207 };
208 
209 #define MESON_ECC_DATA(b, s, sz)	{ .bch = (b), .strength = (s), .size = (sz) }
210 
211 static struct meson_nand_ecc meson_ecc[] = {
212 	MESON_ECC_DATA(NFC_ECC_BCH8_512, 8,  512),
213 	MESON_ECC_DATA(NFC_ECC_BCH8_1K,  8,  1024),
214 	MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24, 1024),
215 	MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30, 1024),
216 	MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40, 1024),
217 	MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50, 1024),
218 	MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60, 1024),
219 };
220 
meson_nand_calc_ecc_bytes(int step_size,int strength)221 static int meson_nand_calc_ecc_bytes(int step_size, int strength)
222 {
223 	int ecc_bytes;
224 
225 	if (step_size == 512 && strength == 8)
226 		return ECC_PARITY_BCH8_512B;
227 
228 	ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
229 	ecc_bytes = ALIGN(ecc_bytes, 2);
230 
231 	return ecc_bytes;
232 }
233 
234 NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
235 		     meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
236 
237 static const int axg_stepinfo_strengths[] = { 8 };
238 
239 static const struct nand_ecc_step_info axg_stepinfo[] = {
240 	{
241 		.stepsize = 1024,
242 		.strengths = axg_stepinfo_strengths,
243 		.nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
244 	},
245 	{
246 		.stepsize = 512,
247 		.strengths = axg_stepinfo_strengths,
248 		.nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
249 	},
250 };
251 
252 static const struct nand_ecc_caps meson_axg_ecc_caps = {
253 	.stepinfos = axg_stepinfo,
254 	.nstepinfos = ARRAY_SIZE(axg_stepinfo),
255 	.calc_ecc_bytes = meson_nand_calc_ecc_bytes,
256 };
257 
to_meson_nand(struct nand_chip * nand)258 static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
259 {
260 	return container_of(nand, struct meson_nfc_nand_chip, nand);
261 }
262 
meson_nfc_select_chip(struct nand_chip * nand,int chip)263 static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
264 {
265 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
266 	struct meson_nfc *nfc = nand_get_controller_data(nand);
267 	int ret, value;
268 
269 	if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
270 		return;
271 
272 	nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
273 	nfc->param.rb_select = nfc->param.chip_select;
274 	nfc->timing.twb = meson_chip->twb;
275 	nfc->timing.tadl = meson_chip->tadl;
276 	nfc->timing.tbers_max = meson_chip->tbers_max;
277 
278 	if (nfc->clk_rate != meson_chip->clk_rate) {
279 		ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
280 		if (ret) {
281 			dev_err(nfc->dev, "failed to set clock rate\n");
282 			return;
283 		}
284 		nfc->clk_rate = meson_chip->clk_rate;
285 	}
286 	if (nfc->bus_timing != meson_chip->bus_timing) {
287 		value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
288 		writel(value, nfc->reg_base + NFC_REG_CFG);
289 		writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
290 		nfc->bus_timing =  meson_chip->bus_timing;
291 	}
292 }
293 
meson_nfc_cmd_idle(struct meson_nfc * nfc,u32 time)294 static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
295 {
296 	writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
297 	       nfc->reg_base + NFC_REG_CMD);
298 }
299 
meson_nfc_cmd_seed(struct meson_nfc * nfc,u32 seed)300 static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
301 {
302 	writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
303 	       nfc->reg_base + NFC_REG_CMD);
304 }
305 
meson_nfc_is_boot_page(struct nand_chip * nand,int page)306 static int meson_nfc_is_boot_page(struct nand_chip *nand, int page)
307 {
308 	const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
309 
310 	return (nand->options & NAND_IS_BOOT_MEDIUM) &&
311 	       !(page % meson_chip->boot_page_step) &&
312 	       (page < meson_chip->boot_pages);
313 }
314 
meson_nfc_cmd_access(struct nand_chip * nand,int raw,bool dir,int page)315 static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir, int page)
316 {
317 	const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
318 	struct mtd_info *mtd = nand_to_mtd(nand);
319 	struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
320 	int len = mtd->writesize, pagesize, pages;
321 	int scrambler;
322 	u32 cmd;
323 
324 	if (nand->options & NAND_NEED_SCRAMBLING)
325 		scrambler = NFC_CMD_SCRAMBLER_ENABLE;
326 	else
327 		scrambler = NFC_CMD_SCRAMBLER_DISABLE;
328 
329 	if (raw) {
330 		len = mtd->writesize + mtd->oobsize;
331 		cmd = len | scrambler | DMA_DIR(dir);
332 	} else if (meson_nfc_is_boot_page(nand, page)) {
333 		pagesize = NFC_SHORT_MODE_ECC_SZ >> 3;
334 		pages = mtd->writesize / 512;
335 
336 		scrambler = NFC_CMD_SCRAMBLER_ENABLE;
337 		cmd = CMDRWGEN(DMA_DIR(dir), scrambler, NFC_ECC_BCH8_1K,
338 			       NFC_CMD_SHORTMODE_ENABLE, pagesize, pages);
339 	} else {
340 		pagesize = nand->ecc.size >> 3;
341 		pages = len / nand->ecc.size;
342 
343 		cmd = CMDRWGEN(DMA_DIR(dir), scrambler, meson_chip->bch_mode,
344 			       NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
345 	}
346 
347 	if (scrambler == NFC_CMD_SCRAMBLER_ENABLE)
348 		meson_nfc_cmd_seed(nfc, page);
349 
350 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
351 }
352 
meson_nfc_drain_cmd(struct meson_nfc * nfc)353 static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
354 {
355 	/*
356 	 * Insert two commands to make sure all valid commands are finished.
357 	 *
358 	 * The Nand flash controller is designed as two stages pipleline -
359 	 *  a) fetch and b) excute.
360 	 * There might be cases when the driver see command queue is empty,
361 	 * but the Nand flash controller still has two commands buffered,
362 	 * one is fetched into NFC request queue (ready to run), and another
363 	 * is actively executing. So pushing 2 "IDLE" commands guarantees that
364 	 * the pipeline is emptied.
365 	 */
366 	meson_nfc_cmd_idle(nfc, 0);
367 	meson_nfc_cmd_idle(nfc, 0);
368 }
369 
meson_nfc_wait_cmd_finish(struct meson_nfc * nfc,unsigned int timeout_ms)370 static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
371 				     unsigned int timeout_ms)
372 {
373 	u32 cmd_size = 0;
374 	int ret;
375 
376 	/* wait cmd fifo is empty */
377 	ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
378 					 !NFC_CMD_GET_SIZE(cmd_size),
379 					 10, timeout_ms * 1000);
380 	if (ret)
381 		dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
382 
383 	return ret;
384 }
385 
meson_nfc_wait_dma_finish(struct meson_nfc * nfc)386 static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
387 {
388 	meson_nfc_drain_cmd(nfc);
389 
390 	return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
391 }
392 
meson_nfc_oob_ptr(struct nand_chip * nand,int i)393 static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
394 {
395 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
396 	int len;
397 
398 	len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
399 
400 	return meson_chip->data_buf + len;
401 }
402 
meson_nfc_data_ptr(struct nand_chip * nand,int i)403 static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
404 {
405 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
406 	int len, temp;
407 
408 	temp = nand->ecc.size + nand->ecc.bytes;
409 	len = (temp + 2) * i;
410 
411 	return meson_chip->data_buf + len;
412 }
413 
meson_nfc_get_data_oob(struct nand_chip * nand,u8 * buf,u8 * oobbuf)414 static void meson_nfc_get_data_oob(struct nand_chip *nand,
415 				   u8 *buf, u8 *oobbuf)
416 {
417 	int i, oob_len = 0;
418 	u8 *dsrc, *osrc;
419 
420 	oob_len = nand->ecc.bytes + 2;
421 	for (i = 0; i < nand->ecc.steps; i++) {
422 		if (buf) {
423 			dsrc = meson_nfc_data_ptr(nand, i);
424 			memcpy(buf, dsrc, nand->ecc.size);
425 			buf += nand->ecc.size;
426 		}
427 		osrc = meson_nfc_oob_ptr(nand, i);
428 		memcpy(oobbuf, osrc, oob_len);
429 		oobbuf += oob_len;
430 	}
431 }
432 
meson_nfc_set_data_oob(struct nand_chip * nand,const u8 * buf,u8 * oobbuf)433 static void meson_nfc_set_data_oob(struct nand_chip *nand,
434 				   const u8 *buf, u8 *oobbuf)
435 {
436 	int i, oob_len = 0;
437 	u8 *dsrc, *osrc;
438 
439 	oob_len = nand->ecc.bytes + 2;
440 	for (i = 0; i < nand->ecc.steps; i++) {
441 		if (buf) {
442 			dsrc = meson_nfc_data_ptr(nand, i);
443 			memcpy(dsrc, buf, nand->ecc.size);
444 			buf += nand->ecc.size;
445 		}
446 		osrc = meson_nfc_oob_ptr(nand, i);
447 		memcpy(osrc, oobbuf, oob_len);
448 		oobbuf += oob_len;
449 	}
450 }
451 
meson_nfc_wait_no_rb_pin(struct nand_chip * nand,int timeout_ms,bool need_cmd_read0)452 static int meson_nfc_wait_no_rb_pin(struct nand_chip *nand, int timeout_ms,
453 				    bool need_cmd_read0)
454 {
455 	struct meson_nfc *nfc = nand_get_controller_data(nand);
456 	u32 cmd, cfg;
457 
458 	meson_nfc_cmd_idle(nfc, nfc->timing.twb);
459 	meson_nfc_drain_cmd(nfc);
460 	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
461 
462 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
463 	cfg |= NFC_RB_IRQ_EN;
464 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
465 
466 	reinit_completion(&nfc->completion);
467 	nand_status_op(nand, NULL);
468 
469 	/* use the max erase time as the maximum clock for waiting R/B */
470 	cmd = NFC_CMD_RB | NFC_CMD_RB_INT_NO_PIN | nfc->timing.tbers_max;
471 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
472 
473 	if (!wait_for_completion_timeout(&nfc->completion,
474 					 msecs_to_jiffies(timeout_ms)))
475 		return -ETIMEDOUT;
476 
477 	if (need_cmd_read0)
478 		nand_exit_status_op(nand);
479 
480 	return 0;
481 }
482 
meson_nfc_wait_rb_pin(struct meson_nfc * nfc,int timeout_ms)483 static int meson_nfc_wait_rb_pin(struct meson_nfc *nfc, int timeout_ms)
484 {
485 	u32 cmd, cfg;
486 	int ret = 0;
487 
488 	meson_nfc_cmd_idle(nfc, nfc->timing.twb);
489 	meson_nfc_drain_cmd(nfc);
490 	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
491 
492 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
493 	cfg |= NFC_RB_IRQ_EN;
494 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
495 
496 	reinit_completion(&nfc->completion);
497 
498 	/* use the max erase time as the maximum clock for waiting R/B */
499 	cmd = NFC_CMD_RB | NFC_CMD_RB_INT
500 		| nfc->param.chip_select | nfc->timing.tbers_max;
501 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
502 
503 	ret = wait_for_completion_timeout(&nfc->completion,
504 					  msecs_to_jiffies(timeout_ms));
505 	if (ret == 0)
506 		ret = -1;
507 
508 	return ret;
509 }
510 
meson_nfc_queue_rb(struct nand_chip * nand,int timeout_ms,bool need_cmd_read0)511 static int meson_nfc_queue_rb(struct nand_chip *nand, int timeout_ms,
512 			      bool need_cmd_read0)
513 {
514 	struct meson_nfc *nfc = nand_get_controller_data(nand);
515 
516 	if (nfc->no_rb_pin) {
517 		/* This mode is used when there is no wired R/B pin.
518 		 * It works like 'nand_soft_waitrdy()', but instead of
519 		 * polling NAND_CMD_STATUS bit in the software loop,
520 		 * it will wait for interrupt - controllers checks IO
521 		 * bus and when it detects NAND_CMD_STATUS on it, it
522 		 * raises interrupt. After interrupt, NAND_CMD_READ0 is
523 		 * sent as terminator of the ready waiting procedure if
524 		 * needed (for all cases except page programming - this
525 		 * is reason of 'need_cmd_read0' flag).
526 		 */
527 		return meson_nfc_wait_no_rb_pin(nand, timeout_ms,
528 						need_cmd_read0);
529 	} else {
530 		return meson_nfc_wait_rb_pin(nfc, timeout_ms);
531 	}
532 }
533 
meson_nfc_set_user_byte(struct nand_chip * nand,u8 * oob_buf)534 static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
535 {
536 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
537 	__le64 *info;
538 	int i, count;
539 
540 	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) {
541 		info = &meson_chip->info_buf[i];
542 		*info |= oob_buf[count];
543 		*info |= oob_buf[count + 1] << 8;
544 	}
545 }
546 
meson_nfc_get_user_byte(struct nand_chip * nand,u8 * oob_buf)547 static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
548 {
549 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
550 	__le64 *info;
551 	int i, count;
552 
553 	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) {
554 		info = &meson_chip->info_buf[i];
555 		oob_buf[count] = *info;
556 		oob_buf[count + 1] = *info >> 8;
557 	}
558 }
559 
meson_nfc_ecc_correct(struct nand_chip * nand,u32 * bitflips,u64 * correct_bitmap)560 static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
561 				 u64 *correct_bitmap)
562 {
563 	struct mtd_info *mtd = nand_to_mtd(nand);
564 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
565 	__le64 *info;
566 	int ret = 0, i;
567 
568 	for (i = 0; i < nand->ecc.steps; i++) {
569 		info = &meson_chip->info_buf[i];
570 		if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
571 			mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
572 			*bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
573 			*correct_bitmap |= BIT_ULL(i);
574 			continue;
575 		}
576 		if ((nand->options & NAND_NEED_SCRAMBLING) &&
577 		    ECC_ZERO_CNT(*info) < nand->ecc.strength) {
578 			mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
579 			*bitflips = max_t(u32, *bitflips,
580 					  ECC_ZERO_CNT(*info));
581 			ret = ECC_CHECK_RETURN_FF;
582 		} else {
583 			ret = -EBADMSG;
584 		}
585 	}
586 	return ret;
587 }
588 
meson_nfc_dma_buffer_setup(struct nand_chip * nand,void * databuf,int datalen,void * infobuf,int infolen,enum dma_data_direction dir)589 static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
590 				      int datalen, void *infobuf, int infolen,
591 				      enum dma_data_direction dir)
592 {
593 	struct meson_nfc *nfc = nand_get_controller_data(nand);
594 	u32 cmd;
595 	int ret = 0;
596 
597 	nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
598 	ret = dma_mapping_error(nfc->dev, nfc->daddr);
599 	if (ret) {
600 		dev_err(nfc->dev, "DMA mapping error\n");
601 		return ret;
602 	}
603 	cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
604 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
605 
606 	cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
607 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
608 
609 	if (infobuf) {
610 		nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
611 		ret = dma_mapping_error(nfc->dev, nfc->iaddr);
612 		if (ret) {
613 			dev_err(nfc->dev, "DMA mapping error\n");
614 			dma_unmap_single(nfc->dev,
615 					 nfc->daddr, datalen, dir);
616 			return ret;
617 		}
618 		nfc->info_bytes = infolen;
619 		cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
620 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
621 
622 		cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
623 		writel(cmd, nfc->reg_base + NFC_REG_CMD);
624 	}
625 
626 	return ret;
627 }
628 
meson_nfc_dma_buffer_release(struct nand_chip * nand,int datalen,int infolen,enum dma_data_direction dir)629 static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
630 					 int datalen, int infolen,
631 					 enum dma_data_direction dir)
632 {
633 	struct meson_nfc *nfc = nand_get_controller_data(nand);
634 
635 	dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
636 	if (infolen) {
637 		dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
638 		nfc->info_bytes = 0;
639 	}
640 }
641 
meson_nfc_read_buf(struct nand_chip * nand,u8 * buf,int len)642 static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
643 {
644 	struct meson_nfc *nfc = nand_get_controller_data(nand);
645 	int ret = 0;
646 	u32 cmd;
647 	u8 *info;
648 
649 	info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
650 	if (!info)
651 		return -ENOMEM;
652 
653 	ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
654 					 PER_INFO_BYTE, DMA_FROM_DEVICE);
655 	if (ret)
656 		goto out;
657 
658 	cmd = NFC_CMD_N2M | len;
659 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
660 
661 	meson_nfc_drain_cmd(nfc);
662 	meson_nfc_wait_cmd_finish(nfc, 1000);
663 	meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
664 
665 out:
666 	kfree(info);
667 
668 	return ret;
669 }
670 
meson_nfc_write_buf(struct nand_chip * nand,u8 * buf,int len)671 static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
672 {
673 	struct meson_nfc *nfc = nand_get_controller_data(nand);
674 	int ret = 0;
675 	u32 cmd;
676 
677 	ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
678 					 0, DMA_TO_DEVICE);
679 	if (ret)
680 		return ret;
681 
682 	cmd = NFC_CMD_M2N | len;
683 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
684 
685 	meson_nfc_drain_cmd(nfc);
686 	meson_nfc_wait_cmd_finish(nfc, 1000);
687 	meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
688 
689 	return ret;
690 }
691 
meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip * nand,int page,bool in)692 static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
693 						int page, bool in)
694 {
695 	const struct nand_sdr_timings *sdr =
696 		nand_get_sdr_timings(nand_get_interface_config(nand));
697 	struct mtd_info *mtd = nand_to_mtd(nand);
698 	struct meson_nfc *nfc = nand_get_controller_data(nand);
699 	u32 *addrs = nfc->cmdfifo.rw.addrs;
700 	u32 cs = nfc->param.chip_select;
701 	u32 cmd0, cmd_num, row_start;
702 	int i;
703 
704 	cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
705 
706 	cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
707 	nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
708 
709 	addrs[0] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_0;
710 	if (mtd->writesize <= 512) {
711 		cmd_num--;
712 		row_start = 1;
713 	} else {
714 		addrs[1] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_1;
715 		row_start = 2;
716 	}
717 
718 	addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
719 	addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
720 
721 	if (nand->options & NAND_ROW_ADDR_3)
722 		addrs[row_start + 2] =
723 			cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
724 	else
725 		cmd_num--;
726 
727 	/* subtract cmd1 */
728 	cmd_num--;
729 
730 	for (i = 0; i < cmd_num; i++)
731 		writel_relaxed(nfc->cmdfifo.cmd[i],
732 			       nfc->reg_base + NFC_REG_CMD);
733 
734 	if (in) {
735 		nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
736 		writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
737 		meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tR_max), true);
738 	} else {
739 		meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
740 	}
741 
742 	return 0;
743 }
744 
meson_nfc_write_page_sub(struct nand_chip * nand,int page,int raw)745 static int meson_nfc_write_page_sub(struct nand_chip *nand,
746 				    int page, int raw)
747 {
748 	const struct nand_sdr_timings *sdr =
749 		nand_get_sdr_timings(nand_get_interface_config(nand));
750 	struct mtd_info *mtd = nand_to_mtd(nand);
751 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
752 	struct meson_nfc *nfc = nand_get_controller_data(nand);
753 	int data_len, info_len;
754 	u32 cmd;
755 	int ret;
756 
757 	meson_nfc_select_chip(nand, nand->cur_cs);
758 
759 	data_len =  mtd->writesize + mtd->oobsize;
760 	info_len = nand->ecc.steps * PER_INFO_BYTE;
761 
762 	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
763 	if (ret)
764 		return ret;
765 
766 	ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
767 					 data_len, meson_chip->info_buf,
768 					 info_len, DMA_TO_DEVICE);
769 	if (ret)
770 		return ret;
771 
772 	meson_nfc_cmd_access(nand, raw, DIRWRITE, page);
773 
774 	cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
775 	writel(cmd, nfc->reg_base + NFC_REG_CMD);
776 	meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tPROG_max), false);
777 
778 	meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
779 
780 	return ret;
781 }
782 
meson_nfc_write_page_raw(struct nand_chip * nand,const u8 * buf,int oob_required,int page)783 static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
784 				    int oob_required, int page)
785 {
786 	u8 *oob_buf = nand->oob_poi;
787 
788 	meson_nfc_set_data_oob(nand, buf, oob_buf);
789 
790 	return meson_nfc_write_page_sub(nand, page, 1);
791 }
792 
meson_nfc_write_page_hwecc(struct nand_chip * nand,const u8 * buf,int oob_required,int page)793 static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
794 				      const u8 *buf, int oob_required, int page)
795 {
796 	struct mtd_info *mtd = nand_to_mtd(nand);
797 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
798 	u8 *oob_buf = nand->oob_poi;
799 
800 	memcpy(meson_chip->data_buf, buf, mtd->writesize);
801 	memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
802 	meson_nfc_set_user_byte(nand, oob_buf);
803 
804 	return meson_nfc_write_page_sub(nand, page, 0);
805 }
806 
meson_nfc_check_ecc_pages_valid(struct meson_nfc * nfc,struct nand_chip * nand,int raw)807 static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
808 					    struct nand_chip *nand, int raw)
809 {
810 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
811 	__le64 *info;
812 	u32 neccpages;
813 	int ret;
814 
815 	neccpages = raw ? 1 : nand->ecc.steps;
816 	info = &meson_chip->info_buf[neccpages - 1];
817 	do {
818 		usleep_range(10, 15);
819 		/* info is updated by nfc dma engine*/
820 		smp_rmb();
821 		dma_sync_single_for_cpu(nfc->dev, nfc->iaddr, nfc->info_bytes,
822 					DMA_FROM_DEVICE);
823 		ret = *info & ECC_COMPLETE;
824 	} while (!ret);
825 }
826 
meson_nfc_read_page_sub(struct nand_chip * nand,int page,int raw)827 static int meson_nfc_read_page_sub(struct nand_chip *nand,
828 				   int page, int raw)
829 {
830 	struct mtd_info *mtd = nand_to_mtd(nand);
831 	struct meson_nfc *nfc = nand_get_controller_data(nand);
832 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
833 	int data_len, info_len;
834 	int ret;
835 
836 	meson_nfc_select_chip(nand, nand->cur_cs);
837 
838 	data_len =  mtd->writesize + mtd->oobsize;
839 	info_len = nand->ecc.steps * PER_INFO_BYTE;
840 
841 	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
842 	if (ret)
843 		return ret;
844 
845 	ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
846 					 data_len, meson_chip->info_buf,
847 					 info_len, DMA_FROM_DEVICE);
848 	if (ret)
849 		return ret;
850 
851 	meson_nfc_cmd_access(nand, raw, DIRREAD, page);
852 
853 	ret = meson_nfc_wait_dma_finish(nfc);
854 	meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
855 
856 	meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
857 
858 	return ret;
859 }
860 
meson_nfc_read_page_raw(struct nand_chip * nand,u8 * buf,int oob_required,int page)861 static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
862 				   int oob_required, int page)
863 {
864 	u8 *oob_buf = nand->oob_poi;
865 	int ret;
866 
867 	ret = meson_nfc_read_page_sub(nand, page, 1);
868 	if (ret)
869 		return ret;
870 
871 	meson_nfc_get_data_oob(nand, buf, oob_buf);
872 
873 	return 0;
874 }
875 
meson_nfc_read_page_hwecc(struct nand_chip * nand,u8 * buf,int oob_required,int page)876 static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
877 				     int oob_required, int page)
878 {
879 	struct mtd_info *mtd = nand_to_mtd(nand);
880 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
881 	struct nand_ecc_ctrl *ecc = &nand->ecc;
882 	u64 correct_bitmap = 0;
883 	u32 bitflips = 0;
884 	u8 *oob_buf = nand->oob_poi;
885 	int ret, i;
886 
887 	ret = meson_nfc_read_page_sub(nand, page, 0);
888 	if (ret)
889 		return ret;
890 
891 	meson_nfc_get_user_byte(nand, oob_buf);
892 	ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
893 	if (ret == ECC_CHECK_RETURN_FF) {
894 		if (buf)
895 			memset(buf, 0xff, mtd->writesize);
896 		memset(oob_buf, 0xff, mtd->oobsize);
897 	} else if (ret < 0) {
898 		if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
899 			mtd->ecc_stats.failed++;
900 			return bitflips;
901 		}
902 		ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
903 		if (ret)
904 			return ret;
905 
906 		for (i = 0; i < nand->ecc.steps ; i++) {
907 			u8 *data = buf + i * ecc->size;
908 			u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
909 
910 			if (correct_bitmap & BIT_ULL(i))
911 				continue;
912 			ret = nand_check_erased_ecc_chunk(data,	ecc->size,
913 							  oob, ecc->bytes + 2,
914 							  NULL, 0,
915 							  ecc->strength);
916 			if (ret < 0) {
917 				mtd->ecc_stats.failed++;
918 			} else {
919 				mtd->ecc_stats.corrected += ret;
920 				bitflips =  max_t(u32, bitflips, ret);
921 			}
922 		}
923 	} else if (buf && buf != meson_chip->data_buf) {
924 		memcpy(buf, meson_chip->data_buf, mtd->writesize);
925 	}
926 
927 	return bitflips;
928 }
929 
meson_nfc_read_oob_raw(struct nand_chip * nand,int page)930 static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
931 {
932 	return meson_nfc_read_page_raw(nand, NULL, 1, page);
933 }
934 
meson_nfc_read_oob(struct nand_chip * nand,int page)935 static int meson_nfc_read_oob(struct nand_chip *nand, int page)
936 {
937 	return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
938 }
939 
meson_nfc_is_buffer_dma_safe(const void * buffer)940 static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
941 {
942 	if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
943 		return false;
944 
945 	if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
946 		return true;
947 	return false;
948 }
949 
950 static void *
meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr * instr)951 meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
952 {
953 	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
954 		return NULL;
955 
956 	if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
957 		return instr->ctx.data.buf.in;
958 
959 	return kzalloc(instr->ctx.data.len, GFP_KERNEL);
960 }
961 
962 static void
meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr * instr,void * buf)963 meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
964 				     void *buf)
965 {
966 	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
967 	    WARN_ON(!buf))
968 		return;
969 
970 	if (buf == instr->ctx.data.buf.in)
971 		return;
972 
973 	memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
974 	kfree(buf);
975 }
976 
977 static void *
meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr * instr)978 meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
979 {
980 	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
981 		return NULL;
982 
983 	if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
984 		return (void *)instr->ctx.data.buf.out;
985 
986 	return kmemdup(instr->ctx.data.buf.out,
987 		       instr->ctx.data.len, GFP_KERNEL);
988 }
989 
990 static void
meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr * instr,const void * buf)991 meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
992 				      const void *buf)
993 {
994 	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
995 	    WARN_ON(!buf))
996 		return;
997 
998 	if (buf != instr->ctx.data.buf.out)
999 		kfree(buf);
1000 }
1001 
meson_nfc_check_op(struct nand_chip * chip,const struct nand_operation * op)1002 static int meson_nfc_check_op(struct nand_chip *chip,
1003 			      const struct nand_operation *op)
1004 {
1005 	int op_id;
1006 
1007 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
1008 		const struct nand_op_instr *instr;
1009 
1010 		instr = &op->instrs[op_id];
1011 
1012 		switch (instr->type) {
1013 		case NAND_OP_DATA_IN_INSTR:
1014 		case NAND_OP_DATA_OUT_INSTR:
1015 			if (instr->ctx.data.len > NFC_CMD_RAW_LEN)
1016 				return -ENOTSUPP;
1017 
1018 			break;
1019 		default:
1020 			break;
1021 		}
1022 	}
1023 
1024 	return 0;
1025 }
1026 
meson_nfc_exec_op(struct nand_chip * nand,const struct nand_operation * op,bool check_only)1027 static int meson_nfc_exec_op(struct nand_chip *nand,
1028 			     const struct nand_operation *op, bool check_only)
1029 {
1030 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1031 	struct meson_nfc *nfc = nand_get_controller_data(nand);
1032 	const struct nand_op_instr *instr = NULL;
1033 	void *buf;
1034 	u32 op_id, delay_idle, cmd;
1035 	int err;
1036 	int i;
1037 
1038 	err = meson_nfc_check_op(nand, op);
1039 	if (err)
1040 		return err;
1041 
1042 	if (check_only)
1043 		return 0;
1044 
1045 	meson_nfc_select_chip(nand, op->cs);
1046 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
1047 		instr = &op->instrs[op_id];
1048 		delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
1049 					  meson_chip->level1_divider *
1050 					  NFC_CLK_CYCLE);
1051 		switch (instr->type) {
1052 		case NAND_OP_CMD_INSTR:
1053 			cmd = nfc->param.chip_select | NFC_CMD_CLE;
1054 			cmd |= instr->ctx.cmd.opcode & 0xff;
1055 			writel(cmd, nfc->reg_base + NFC_REG_CMD);
1056 			meson_nfc_cmd_idle(nfc, delay_idle);
1057 			break;
1058 
1059 		case NAND_OP_ADDR_INSTR:
1060 			for (i = 0; i < instr->ctx.addr.naddrs; i++) {
1061 				cmd = nfc->param.chip_select | NFC_CMD_ALE;
1062 				cmd |= instr->ctx.addr.addrs[i] & 0xff;
1063 				writel(cmd, nfc->reg_base + NFC_REG_CMD);
1064 			}
1065 			meson_nfc_cmd_idle(nfc, delay_idle);
1066 			break;
1067 
1068 		case NAND_OP_DATA_IN_INSTR:
1069 			buf = meson_nand_op_get_dma_safe_input_buf(instr);
1070 			if (!buf)
1071 				return -ENOMEM;
1072 			meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
1073 			meson_nand_op_put_dma_safe_input_buf(instr, buf);
1074 			break;
1075 
1076 		case NAND_OP_DATA_OUT_INSTR:
1077 			buf = meson_nand_op_get_dma_safe_output_buf(instr);
1078 			if (!buf)
1079 				return -ENOMEM;
1080 			meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
1081 			meson_nand_op_put_dma_safe_output_buf(instr, buf);
1082 			break;
1083 
1084 		case NAND_OP_WAITRDY_INSTR:
1085 			meson_nfc_queue_rb(nand, instr->ctx.waitrdy.timeout_ms,
1086 					   true);
1087 			if (instr->delay_ns)
1088 				meson_nfc_cmd_idle(nfc, delay_idle);
1089 			break;
1090 		}
1091 	}
1092 	meson_nfc_wait_cmd_finish(nfc, 1000);
1093 	return 0;
1094 }
1095 
meson_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)1096 static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
1097 			       struct mtd_oob_region *oobregion)
1098 {
1099 	struct nand_chip *nand = mtd_to_nand(mtd);
1100 
1101 	if (section >= nand->ecc.steps)
1102 		return -ERANGE;
1103 
1104 	oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
1105 	oobregion->length = nand->ecc.bytes;
1106 
1107 	return 0;
1108 }
1109 
meson_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)1110 static int meson_ooblayout_free(struct mtd_info *mtd, int section,
1111 				struct mtd_oob_region *oobregion)
1112 {
1113 	struct nand_chip *nand = mtd_to_nand(mtd);
1114 
1115 	if (section >= nand->ecc.steps)
1116 		return -ERANGE;
1117 
1118 	oobregion->offset = section * (2 + nand->ecc.bytes);
1119 	oobregion->length = 2;
1120 
1121 	return 0;
1122 }
1123 
1124 static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
1125 	.ecc = meson_ooblayout_ecc,
1126 	.free = meson_ooblayout_free,
1127 };
1128 
meson_nfc_clk_init(struct meson_nfc * nfc)1129 static int meson_nfc_clk_init(struct meson_nfc *nfc)
1130 {
1131 	struct clk_parent_data nfc_divider_parent_data[1] = {0};
1132 	struct clk_init_data init = {0};
1133 	int ret;
1134 
1135 	/* request core clock */
1136 	nfc->core_clk = devm_clk_get(nfc->dev, "core");
1137 	if (IS_ERR(nfc->core_clk)) {
1138 		dev_err(nfc->dev, "failed to get core clock\n");
1139 		return PTR_ERR(nfc->core_clk);
1140 	}
1141 
1142 	nfc->device_clk = devm_clk_get(nfc->dev, "device");
1143 	if (IS_ERR(nfc->device_clk)) {
1144 		dev_err(nfc->dev, "failed to get device clock\n");
1145 		return PTR_ERR(nfc->device_clk);
1146 	}
1147 
1148 	init.name = devm_kasprintf(nfc->dev,
1149 				   GFP_KERNEL, "%s#div",
1150 				   dev_name(nfc->dev));
1151 	if (!init.name)
1152 		return -ENOMEM;
1153 
1154 	init.ops = &clk_divider_ops;
1155 	nfc_divider_parent_data[0].fw_name = "device";
1156 	init.parent_data = nfc_divider_parent_data;
1157 	init.num_parents = 1;
1158 	nfc->nand_divider.reg = nfc->reg_clk;
1159 	nfc->nand_divider.shift = CLK_DIV_SHIFT;
1160 	nfc->nand_divider.width = CLK_DIV_WIDTH;
1161 	nfc->nand_divider.hw.init = &init;
1162 	nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1163 				  CLK_DIVIDER_ROUND_CLOSEST |
1164 				  CLK_DIVIDER_ALLOW_ZERO;
1165 
1166 	nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1167 	if (IS_ERR(nfc->nand_clk))
1168 		return PTR_ERR(nfc->nand_clk);
1169 
1170 	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1171 	writel(CLK_ALWAYS_ON_NAND | CLK_SELECT_NAND | CLK_SELECT_FIX_PLL2,
1172 	       nfc->reg_clk);
1173 
1174 	ret = clk_prepare_enable(nfc->core_clk);
1175 	if (ret) {
1176 		dev_err(nfc->dev, "failed to enable core clock\n");
1177 		return ret;
1178 	}
1179 
1180 	ret = clk_prepare_enable(nfc->device_clk);
1181 	if (ret) {
1182 		dev_err(nfc->dev, "failed to enable device clock\n");
1183 		goto err_device_clk;
1184 	}
1185 
1186 	ret = clk_prepare_enable(nfc->nand_clk);
1187 	if (ret) {
1188 		dev_err(nfc->dev, "pre enable NFC divider fail\n");
1189 		goto err_nand_clk;
1190 	}
1191 
1192 	ret = clk_set_rate(nfc->nand_clk, 24000000);
1193 	if (ret)
1194 		goto err_disable_clk;
1195 
1196 	return 0;
1197 
1198 err_disable_clk:
1199 	clk_disable_unprepare(nfc->nand_clk);
1200 err_nand_clk:
1201 	clk_disable_unprepare(nfc->device_clk);
1202 err_device_clk:
1203 	clk_disable_unprepare(nfc->core_clk);
1204 	return ret;
1205 }
1206 
meson_nfc_disable_clk(struct meson_nfc * nfc)1207 static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1208 {
1209 	clk_disable_unprepare(nfc->nand_clk);
1210 	clk_disable_unprepare(nfc->device_clk);
1211 	clk_disable_unprepare(nfc->core_clk);
1212 }
1213 
meson_nfc_free_buffer(struct nand_chip * nand)1214 static void meson_nfc_free_buffer(struct nand_chip *nand)
1215 {
1216 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1217 
1218 	kfree(meson_chip->info_buf);
1219 	kfree(meson_chip->data_buf);
1220 }
1221 
meson_chip_buffer_init(struct nand_chip * nand)1222 static int meson_chip_buffer_init(struct nand_chip *nand)
1223 {
1224 	struct mtd_info *mtd = nand_to_mtd(nand);
1225 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1226 	u32 page_bytes, info_bytes, nsectors;
1227 
1228 	nsectors = mtd->writesize / nand->ecc.size;
1229 
1230 	page_bytes =  mtd->writesize + mtd->oobsize;
1231 	info_bytes = nsectors * PER_INFO_BYTE;
1232 
1233 	meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1234 	if (!meson_chip->data_buf)
1235 		return -ENOMEM;
1236 
1237 	meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1238 	if (!meson_chip->info_buf) {
1239 		kfree(meson_chip->data_buf);
1240 		return -ENOMEM;
1241 	}
1242 
1243 	return 0;
1244 }
1245 
1246 static
meson_nfc_setup_interface(struct nand_chip * nand,int csline,const struct nand_interface_config * conf)1247 int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1248 			      const struct nand_interface_config *conf)
1249 {
1250 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1251 	const struct nand_sdr_timings *timings;
1252 	u32 div, bt_min, bt_max, tbers_clocks;
1253 
1254 	timings = nand_get_sdr_timings(conf);
1255 	if (IS_ERR(timings))
1256 		return -ENOTSUPP;
1257 
1258 	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1259 		return 0;
1260 
1261 	div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1262 	bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1263 	bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1264 		  timings->tRC_min / 2) / div;
1265 
1266 	meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1267 				       div * NFC_CLK_CYCLE);
1268 	meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1269 					div * NFC_CLK_CYCLE);
1270 	tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1271 					div * NFC_CLK_CYCLE);
1272 	meson_chip->tbers_max = ilog2(tbers_clocks);
1273 	if (!is_power_of_2(tbers_clocks))
1274 		meson_chip->tbers_max++;
1275 
1276 	bt_min = DIV_ROUND_UP(bt_min, 1000);
1277 	bt_max = DIV_ROUND_UP(bt_max, 1000);
1278 
1279 	if (bt_max < bt_min)
1280 		return -EINVAL;
1281 
1282 	meson_chip->level1_divider = div;
1283 	meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1284 	meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1285 
1286 	return 0;
1287 }
1288 
meson_nand_bch_mode(struct nand_chip * nand)1289 static int meson_nand_bch_mode(struct nand_chip *nand)
1290 {
1291 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1292 	int i;
1293 
1294 	if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1295 		return -EINVAL;
1296 
1297 	for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1298 		if (meson_ecc[i].strength == nand->ecc.strength &&
1299 		    meson_ecc[i].size == nand->ecc.size) {
1300 			meson_chip->bch_mode = meson_ecc[i].bch;
1301 			return 0;
1302 		}
1303 	}
1304 
1305 	return -EINVAL;
1306 }
1307 
meson_nand_detach_chip(struct nand_chip * nand)1308 static void meson_nand_detach_chip(struct nand_chip *nand)
1309 {
1310 	meson_nfc_free_buffer(nand);
1311 }
1312 
meson_nand_attach_chip(struct nand_chip * nand)1313 static int meson_nand_attach_chip(struct nand_chip *nand)
1314 {
1315 	struct meson_nfc *nfc = nand_get_controller_data(nand);
1316 	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1317 	struct mtd_info *mtd = nand_to_mtd(nand);
1318 	int raw_writesize;
1319 	int ret;
1320 
1321 	if (!mtd->name) {
1322 		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1323 					   "%s:nand%d",
1324 					   dev_name(nfc->dev),
1325 					   meson_chip->sels[0]);
1326 		if (!mtd->name)
1327 			return -ENOMEM;
1328 	}
1329 
1330 	raw_writesize = mtd->writesize + mtd->oobsize;
1331 	if (raw_writesize > NFC_CMD_RAW_LEN) {
1332 		dev_err(nfc->dev, "too big write size in raw mode: %d > %ld\n",
1333 			raw_writesize, NFC_CMD_RAW_LEN);
1334 		return -EINVAL;
1335 	}
1336 
1337 	if (nand->bbt_options & NAND_BBT_USE_FLASH)
1338 		nand->bbt_options |= NAND_BBT_NO_OOB;
1339 
1340 	nand->options |= NAND_NO_SUBPAGE_WRITE;
1341 
1342 	ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1343 				   mtd->oobsize - 2);
1344 	if (ret) {
1345 		dev_err(nfc->dev, "failed to ECC init\n");
1346 		return -EINVAL;
1347 	}
1348 
1349 	mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1350 
1351 	ret = meson_nand_bch_mode(nand);
1352 	if (ret)
1353 		return -EINVAL;
1354 
1355 	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1356 	nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1357 	nand->ecc.write_page = meson_nfc_write_page_hwecc;
1358 	nand->ecc.write_oob_raw = nand_write_oob_std;
1359 	nand->ecc.write_oob = nand_write_oob_std;
1360 
1361 	nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1362 	nand->ecc.read_page = meson_nfc_read_page_hwecc;
1363 	nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1364 	nand->ecc.read_oob = meson_nfc_read_oob;
1365 
1366 	if (nand->options & NAND_BUSWIDTH_16) {
1367 		dev_err(nfc->dev, "16bits bus width not supported");
1368 		return -EINVAL;
1369 	}
1370 	ret = meson_chip_buffer_init(nand);
1371 	if (ret)
1372 		return -ENOMEM;
1373 
1374 	return ret;
1375 }
1376 
1377 static const struct nand_controller_ops meson_nand_controller_ops = {
1378 	.attach_chip = meson_nand_attach_chip,
1379 	.detach_chip = meson_nand_detach_chip,
1380 	.setup_interface = meson_nfc_setup_interface,
1381 	.exec_op = meson_nfc_exec_op,
1382 };
1383 
1384 static int
meson_nfc_nand_chip_init(struct device * dev,struct meson_nfc * nfc,struct device_node * np)1385 meson_nfc_nand_chip_init(struct device *dev,
1386 			 struct meson_nfc *nfc, struct device_node *np)
1387 {
1388 	struct meson_nfc_nand_chip *meson_chip;
1389 	struct nand_chip *nand;
1390 	struct mtd_info *mtd;
1391 	int ret, i;
1392 	u32 tmp, nsels;
1393 	u32 nand_rb_val = 0;
1394 
1395 	nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1396 	if (!nsels || nsels > MAX_CE_NUM) {
1397 		dev_err(dev, "invalid register property size\n");
1398 		return -EINVAL;
1399 	}
1400 
1401 	meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1402 				  GFP_KERNEL);
1403 	if (!meson_chip)
1404 		return -ENOMEM;
1405 
1406 	meson_chip->nsels = nsels;
1407 
1408 	for (i = 0; i < nsels; i++) {
1409 		ret = of_property_read_u32_index(np, "reg", i, &tmp);
1410 		if (ret) {
1411 			dev_err(dev, "could not retrieve register property: %d\n",
1412 				ret);
1413 			return ret;
1414 		}
1415 
1416 		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1417 			dev_err(dev, "CS %d already assigned\n", tmp);
1418 			return -EINVAL;
1419 		}
1420 	}
1421 
1422 	nand = &meson_chip->nand;
1423 	nand->controller = &nfc->controller;
1424 	nand->controller->ops = &meson_nand_controller_ops;
1425 	nand_set_flash_node(nand, np);
1426 	nand_set_controller_data(nand, nfc);
1427 
1428 	nand->options |= NAND_USES_DMA;
1429 	mtd = nand_to_mtd(nand);
1430 	mtd->owner = THIS_MODULE;
1431 	mtd->dev.parent = dev;
1432 
1433 	ret = of_property_read_u32(np, "nand-rb", &nand_rb_val);
1434 	if (ret == -EINVAL)
1435 		nfc->no_rb_pin = true;
1436 	else if (ret)
1437 		return ret;
1438 
1439 	if (nand_rb_val)
1440 		return -EINVAL;
1441 
1442 	ret = nand_scan(nand, nsels);
1443 	if (ret)
1444 		return ret;
1445 
1446 	if (nand->options & NAND_IS_BOOT_MEDIUM) {
1447 		ret = of_property_read_u32(np, "amlogic,boot-pages",
1448 					   &meson_chip->boot_pages);
1449 		if (ret) {
1450 			dev_err(dev, "could not retrieve 'amlogic,boot-pages' property: %d",
1451 				ret);
1452 			nand_cleanup(nand);
1453 			return ret;
1454 		}
1455 
1456 		ret = of_property_read_u32(np, "amlogic,boot-page-step",
1457 					   &meson_chip->boot_page_step);
1458 		if (ret) {
1459 			dev_err(dev, "could not retrieve 'amlogic,boot-page-step' property: %d",
1460 				ret);
1461 			nand_cleanup(nand);
1462 			return ret;
1463 		}
1464 	}
1465 
1466 	ret = mtd_device_register(mtd, NULL, 0);
1467 	if (ret) {
1468 		dev_err(dev, "failed to register MTD device: %d\n", ret);
1469 		nand_cleanup(nand);
1470 		return ret;
1471 	}
1472 
1473 	list_add_tail(&meson_chip->node, &nfc->chips);
1474 
1475 	return 0;
1476 }
1477 
meson_nfc_nand_chips_cleanup(struct meson_nfc * nfc)1478 static void meson_nfc_nand_chips_cleanup(struct meson_nfc *nfc)
1479 {
1480 	struct meson_nfc_nand_chip *meson_chip;
1481 	struct mtd_info *mtd;
1482 
1483 	while (!list_empty(&nfc->chips)) {
1484 		meson_chip = list_first_entry(&nfc->chips,
1485 					      struct meson_nfc_nand_chip, node);
1486 		mtd = nand_to_mtd(&meson_chip->nand);
1487 		WARN_ON(mtd_device_unregister(mtd));
1488 
1489 		nand_cleanup(&meson_chip->nand);
1490 		list_del(&meson_chip->node);
1491 	}
1492 }
1493 
meson_nfc_nand_chips_init(struct device * dev,struct meson_nfc * nfc)1494 static int meson_nfc_nand_chips_init(struct device *dev,
1495 				     struct meson_nfc *nfc)
1496 {
1497 	struct device_node *np = dev->of_node;
1498 	int ret;
1499 
1500 	for_each_child_of_node_scoped(np, nand_np) {
1501 		ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1502 		if (ret) {
1503 			meson_nfc_nand_chips_cleanup(nfc);
1504 			return ret;
1505 		}
1506 	}
1507 
1508 	return 0;
1509 }
1510 
meson_nfc_irq(int irq,void * id)1511 static irqreturn_t meson_nfc_irq(int irq, void *id)
1512 {
1513 	struct meson_nfc *nfc = id;
1514 	u32 cfg;
1515 
1516 	cfg = readl(nfc->reg_base + NFC_REG_CFG);
1517 	if (!(cfg & NFC_RB_IRQ_EN))
1518 		return IRQ_NONE;
1519 
1520 	cfg &= ~(NFC_RB_IRQ_EN);
1521 	writel(cfg, nfc->reg_base + NFC_REG_CFG);
1522 
1523 	complete(&nfc->completion);
1524 	return IRQ_HANDLED;
1525 }
1526 
1527 static const struct meson_nfc_data meson_gxl_data = {
1528 	.ecc_caps = &meson_gxl_ecc_caps,
1529 };
1530 
1531 static const struct meson_nfc_data meson_axg_data = {
1532 	.ecc_caps = &meson_axg_ecc_caps,
1533 };
1534 
1535 static const struct of_device_id meson_nfc_id_table[] = {
1536 	{
1537 		.compatible = "amlogic,meson-gxl-nfc",
1538 		.data = &meson_gxl_data,
1539 	}, {
1540 		.compatible = "amlogic,meson-axg-nfc",
1541 		.data = &meson_axg_data,
1542 	},
1543 	{}
1544 };
1545 MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1546 
meson_nfc_probe(struct platform_device * pdev)1547 static int meson_nfc_probe(struct platform_device *pdev)
1548 {
1549 	struct device *dev = &pdev->dev;
1550 	struct meson_nfc *nfc;
1551 	int ret, irq;
1552 
1553 	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1554 	if (!nfc)
1555 		return -ENOMEM;
1556 
1557 	nfc->data = of_device_get_match_data(&pdev->dev);
1558 	if (!nfc->data)
1559 		return -ENODEV;
1560 
1561 	nand_controller_init(&nfc->controller);
1562 	INIT_LIST_HEAD(&nfc->chips);
1563 	init_completion(&nfc->completion);
1564 
1565 	nfc->dev = dev;
1566 
1567 	nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1568 	if (IS_ERR(nfc->reg_base))
1569 		return PTR_ERR(nfc->reg_base);
1570 
1571 	nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1572 	if (IS_ERR(nfc->reg_clk))
1573 		return PTR_ERR(nfc->reg_clk);
1574 
1575 	irq = platform_get_irq(pdev, 0);
1576 	if (irq < 0)
1577 		return -EINVAL;
1578 
1579 	ret = meson_nfc_clk_init(nfc);
1580 	if (ret) {
1581 		dev_err(dev, "failed to initialize NAND clock\n");
1582 		return ret;
1583 	}
1584 
1585 	writel(0, nfc->reg_base + NFC_REG_CFG);
1586 	ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1587 	if (ret) {
1588 		dev_err(dev, "failed to request NFC IRQ\n");
1589 		ret = -EINVAL;
1590 		goto err_clk;
1591 	}
1592 
1593 	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1594 	if (ret) {
1595 		dev_err(dev, "failed to set DMA mask\n");
1596 		goto err_clk;
1597 	}
1598 
1599 	platform_set_drvdata(pdev, nfc);
1600 
1601 	ret = meson_nfc_nand_chips_init(dev, nfc);
1602 	if (ret) {
1603 		dev_err(dev, "failed to init NAND chips\n");
1604 		goto err_clk;
1605 	}
1606 
1607 	return 0;
1608 err_clk:
1609 	meson_nfc_disable_clk(nfc);
1610 	return ret;
1611 }
1612 
meson_nfc_remove(struct platform_device * pdev)1613 static void meson_nfc_remove(struct platform_device *pdev)
1614 {
1615 	struct meson_nfc *nfc = platform_get_drvdata(pdev);
1616 
1617 	meson_nfc_nand_chips_cleanup(nfc);
1618 
1619 	meson_nfc_disable_clk(nfc);
1620 }
1621 
1622 static struct platform_driver meson_nfc_driver = {
1623 	.probe  = meson_nfc_probe,
1624 	.remove = meson_nfc_remove,
1625 	.driver = {
1626 		.name  = "meson-nand",
1627 		.of_match_table = meson_nfc_id_table,
1628 	},
1629 };
1630 module_platform_driver(meson_nfc_driver);
1631 
1632 MODULE_LICENSE("Dual MIT/GPL");
1633 MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1634 MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");
1635