1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Cadence NAND flash controller driver
4 *
5 * Copyright (C) 2019 Cadence
6 *
7 * Author: Piotr Sroka <piotrs@cadence.com>
8 */
9
10 #include <linux/bitfield.h>
11 #include <linux/clk.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/interrupt.h>
15 #include <linux/module.h>
16 #include <linux/mtd/mtd.h>
17 #include <linux/mtd/rawnand.h>
18 #include <linux/iopoll.h>
19 #include <linux/of.h>
20 #include <linux/platform_device.h>
21 #include <linux/property.h>
22 #include <linux/slab.h>
23
24 /*
25 * HPNFC can work in 3 modes:
26 * - PIO - can work in master or slave DMA
27 * - CDMA - needs Master DMA for accessing command descriptors.
28 * - Generic mode - can use only slave DMA.
29 * CDMA and PIO modes can be used to execute only base commands.
30 * Generic mode can be used to execute any command
31 * on NAND flash memory. Driver uses CDMA mode for
32 * block erasing, page reading, page programing.
33 * Generic mode is used for executing rest of commands.
34 */
35
36 #define MAX_ADDRESS_CYC 6
37 #define MAX_ERASE_ADDRESS_CYC 3
38 #define MAX_DATA_SIZE 0xFFFC
39 #define DMA_DATA_SIZE_ALIGN 8
40
41 /* Register definition. */
42 /*
43 * Command register 0.
44 * Writing data to this register will initiate a new transaction
45 * of the NF controller.
46 */
47 #define CMD_REG0 0x0000
48 /* Command type field mask. */
49 #define CMD_REG0_CT GENMASK(31, 30)
50 /* Command type CDMA. */
51 #define CMD_REG0_CT_CDMA 0uL
52 /* Command type generic. */
53 #define CMD_REG0_CT_GEN 3uL
54 /* Command thread number field mask. */
55 #define CMD_REG0_TN GENMASK(27, 24)
56
57 /* Command register 2. */
58 #define CMD_REG2 0x0008
59 /* Command register 3. */
60 #define CMD_REG3 0x000C
61 /* Pointer register to select which thread status will be selected. */
62 #define CMD_STATUS_PTR 0x0010
63 /* Command status register for selected thread. */
64 #define CMD_STATUS 0x0014
65
66 /* Interrupt status register. */
67 #define INTR_STATUS 0x0110
68 #define INTR_STATUS_SDMA_ERR BIT(22)
69 #define INTR_STATUS_SDMA_TRIGG BIT(21)
70 #define INTR_STATUS_UNSUPP_CMD BIT(19)
71 #define INTR_STATUS_DDMA_TERR BIT(18)
72 #define INTR_STATUS_CDMA_TERR BIT(17)
73 #define INTR_STATUS_CDMA_IDL BIT(16)
74
75 /* Interrupt enable register. */
76 #define INTR_ENABLE 0x0114
77 #define INTR_ENABLE_INTR_EN BIT(31)
78 #define INTR_ENABLE_SDMA_ERR_EN BIT(22)
79 #define INTR_ENABLE_SDMA_TRIGG_EN BIT(21)
80 #define INTR_ENABLE_UNSUPP_CMD_EN BIT(19)
81 #define INTR_ENABLE_DDMA_TERR_EN BIT(18)
82 #define INTR_ENABLE_CDMA_TERR_EN BIT(17)
83 #define INTR_ENABLE_CDMA_IDLE_EN BIT(16)
84
85 /* Controller internal state. */
86 #define CTRL_STATUS 0x0118
87 #define CTRL_STATUS_INIT_COMP BIT(9)
88 #define CTRL_STATUS_CTRL_BUSY BIT(8)
89
90 /* Command Engine threads state. */
91 #define TRD_STATUS 0x0120
92
93 /* Command Engine interrupt thread error status. */
94 #define TRD_ERR_INT_STATUS 0x0128
95 /* Command Engine interrupt thread error enable. */
96 #define TRD_ERR_INT_STATUS_EN 0x0130
97 /* Command Engine interrupt thread complete status. */
98 #define TRD_COMP_INT_STATUS 0x0138
99
100 /*
101 * Transfer config 0 register.
102 * Configures data transfer parameters.
103 */
104 #define TRAN_CFG_0 0x0400
105 /* Offset value from the beginning of the page. */
106 #define TRAN_CFG_0_OFFSET GENMASK(31, 16)
107 /* Numbers of sectors to transfer within singlNF device's page. */
108 #define TRAN_CFG_0_SEC_CNT GENMASK(7, 0)
109
110 /*
111 * Transfer config 1 register.
112 * Configures data transfer parameters.
113 */
114 #define TRAN_CFG_1 0x0404
115 /* Size of last data sector. */
116 #define TRAN_CFG_1_LAST_SEC_SIZE GENMASK(31, 16)
117 /* Size of not-last data sector. */
118 #define TRAN_CFG_1_SECTOR_SIZE GENMASK(15, 0)
119
120 /* ECC engine configuration register 0. */
121 #define ECC_CONFIG_0 0x0428
122 /* Correction strength. */
123 #define ECC_CONFIG_0_CORR_STR GENMASK(10, 8)
124 /* Enable erased pages detection mechanism. */
125 #define ECC_CONFIG_0_ERASE_DET_EN BIT(1)
126 /* Enable controller ECC check bits generation and correction. */
127 #define ECC_CONFIG_0_ECC_EN BIT(0)
128
129 /* ECC engine configuration register 1. */
130 #define ECC_CONFIG_1 0x042C
131
132 /* Multiplane settings register. */
133 #define MULTIPLANE_CFG 0x0434
134 /* Cache operation settings. */
135 #define CACHE_CFG 0x0438
136
137 /* DMA settings register. */
138 #define DMA_SETINGS 0x043C
139 /* Enable SDMA error report on access unprepared slave DMA interface. */
140 #define DMA_SETINGS_SDMA_ERR_RSP BIT(17)
141
142 /* Transferred data block size for the slave DMA module. */
143 #define SDMA_SIZE 0x0440
144
145 /* Thread number associated with transferred data block
146 * for the slave DMA module.
147 */
148 #define SDMA_TRD_NUM 0x0444
149 /* Thread number mask. */
150 #define SDMA_TRD_NUM_SDMA_TRD GENMASK(2, 0)
151
152 #define CONTROL_DATA_CTRL 0x0494
153 /* Thread number mask. */
154 #define CONTROL_DATA_CTRL_SIZE GENMASK(15, 0)
155
156 #define CTRL_VERSION 0x800
157 #define CTRL_VERSION_REV GENMASK(7, 0)
158
159 /* Available hardware features of the controller. */
160 #define CTRL_FEATURES 0x804
161 /* Support for NV-DDR2/3 work mode. */
162 #define CTRL_FEATURES_NVDDR_2_3 BIT(28)
163 /* Support for NV-DDR work mode. */
164 #define CTRL_FEATURES_NVDDR BIT(27)
165 /* Support for asynchronous work mode. */
166 #define CTRL_FEATURES_ASYNC BIT(26)
167 /* Support for asynchronous work mode. */
168 #define CTRL_FEATURES_N_BANKS GENMASK(25, 24)
169 /* Slave and Master DMA data width. */
170 #define CTRL_FEATURES_DMA_DWITH64 BIT(21)
171 /* Availability of Control Data feature.*/
172 #define CTRL_FEATURES_CONTROL_DATA BIT(10)
173
174 /* BCH Engine identification register 0 - correction strengths. */
175 #define BCH_CFG_0 0x838
176 #define BCH_CFG_0_CORR_CAP_0 GENMASK(7, 0)
177 #define BCH_CFG_0_CORR_CAP_1 GENMASK(15, 8)
178 #define BCH_CFG_0_CORR_CAP_2 GENMASK(23, 16)
179 #define BCH_CFG_0_CORR_CAP_3 GENMASK(31, 24)
180
181 /* BCH Engine identification register 1 - correction strengths. */
182 #define BCH_CFG_1 0x83C
183 #define BCH_CFG_1_CORR_CAP_4 GENMASK(7, 0)
184 #define BCH_CFG_1_CORR_CAP_5 GENMASK(15, 8)
185 #define BCH_CFG_1_CORR_CAP_6 GENMASK(23, 16)
186 #define BCH_CFG_1_CORR_CAP_7 GENMASK(31, 24)
187
188 /* BCH Engine identification register 2 - sector sizes. */
189 #define BCH_CFG_2 0x840
190 #define BCH_CFG_2_SECT_0 GENMASK(15, 0)
191 #define BCH_CFG_2_SECT_1 GENMASK(31, 16)
192
193 /* BCH Engine identification register 3. */
194 #define BCH_CFG_3 0x844
195 #define BCH_CFG_3_METADATA_SIZE GENMASK(23, 16)
196
197 /* Ready/Busy# line status. */
198 #define RBN_SETINGS 0x1004
199
200 /* Common settings. */
201 #define COMMON_SET 0x1008
202 /* 16 bit device connected to the NAND Flash interface. */
203 #define COMMON_SET_DEVICE_16BIT BIT(8)
204
205 /* Skip_bytes registers. */
206 #define SKIP_BYTES_CONF 0x100C
207 #define SKIP_BYTES_MARKER_VALUE GENMASK(31, 16)
208 #define SKIP_BYTES_NUM_OF_BYTES GENMASK(7, 0)
209
210 #define SKIP_BYTES_OFFSET 0x1010
211 #define SKIP_BYTES_OFFSET_VALUE GENMASK(23, 0)
212
213 /* Timings configuration. */
214 #define ASYNC_TOGGLE_TIMINGS 0x101c
215 #define ASYNC_TOGGLE_TIMINGS_TRH GENMASK(28, 24)
216 #define ASYNC_TOGGLE_TIMINGS_TRP GENMASK(20, 16)
217 #define ASYNC_TOGGLE_TIMINGS_TWH GENMASK(12, 8)
218 #define ASYNC_TOGGLE_TIMINGS_TWP GENMASK(4, 0)
219
220 #define TIMINGS0 0x1024
221 #define TIMINGS0_TADL GENMASK(31, 24)
222 #define TIMINGS0_TCCS GENMASK(23, 16)
223 #define TIMINGS0_TWHR GENMASK(15, 8)
224 #define TIMINGS0_TRHW GENMASK(7, 0)
225
226 #define TIMINGS1 0x1028
227 #define TIMINGS1_TRHZ GENMASK(31, 24)
228 #define TIMINGS1_TWB GENMASK(23, 16)
229 #define TIMINGS1_TVDLY GENMASK(7, 0)
230
231 #define TIMINGS2 0x102c
232 #define TIMINGS2_TFEAT GENMASK(25, 16)
233 #define TIMINGS2_CS_HOLD_TIME GENMASK(13, 8)
234 #define TIMINGS2_CS_SETUP_TIME GENMASK(5, 0)
235
236 /* Configuration of the resynchronization of slave DLL of PHY. */
237 #define DLL_PHY_CTRL 0x1034
238 #define DLL_PHY_CTRL_DLL_RST_N BIT(24)
239 #define DLL_PHY_CTRL_EXTENDED_WR_MODE BIT(17)
240 #define DLL_PHY_CTRL_EXTENDED_RD_MODE BIT(16)
241 #define DLL_PHY_CTRL_RS_HIGH_WAIT_CNT GENMASK(11, 8)
242 #define DLL_PHY_CTRL_RS_IDLE_CNT GENMASK(7, 0)
243
244 /* Register controlling DQ related timing. */
245 #define PHY_DQ_TIMING 0x2000
246 /* Register controlling DSQ related timing. */
247 #define PHY_DQS_TIMING 0x2004
248 #define PHY_DQS_TIMING_DQS_SEL_OE_END GENMASK(3, 0)
249 #define PHY_DQS_TIMING_PHONY_DQS_SEL BIT(16)
250 #define PHY_DQS_TIMING_USE_PHONY_DQS BIT(20)
251
252 /* Register controlling the gate and loopback control related timing. */
253 #define PHY_GATE_LPBK_CTRL 0x2008
254 #define PHY_GATE_LPBK_CTRL_RDS GENMASK(24, 19)
255
256 /* Register holds the control for the master DLL logic. */
257 #define PHY_DLL_MASTER_CTRL 0x200C
258 #define PHY_DLL_MASTER_CTRL_BYPASS_MODE BIT(23)
259
260 /* Register holds the control for the slave DLL logic. */
261 #define PHY_DLL_SLAVE_CTRL 0x2010
262
263 /* This register handles the global control settings for the PHY. */
264 #define PHY_CTRL 0x2080
265 #define PHY_CTRL_SDR_DQS BIT(14)
266 #define PHY_CTRL_PHONY_DQS GENMASK(9, 4)
267
268 /*
269 * This register handles the global control settings
270 * for the termination selects for reads.
271 */
272 #define PHY_TSEL 0x2084
273
274 /* Generic command layout. */
275 #define GCMD_LAY_CS GENMASK_ULL(11, 8)
276 /*
277 * This bit informs the minicotroller if it has to wait for tWB
278 * after sending the last CMD/ADDR/DATA in the sequence.
279 */
280 #define GCMD_LAY_TWB BIT_ULL(6)
281 /* Type of generic instruction. */
282 #define GCMD_LAY_INSTR GENMASK_ULL(5, 0)
283
284 /* Generic CMD sequence type. */
285 #define GCMD_LAY_INSTR_CMD 0
286 /* Generic ADDR sequence type. */
287 #define GCMD_LAY_INSTR_ADDR 1
288 /* Generic data transfer sequence type. */
289 #define GCMD_LAY_INSTR_DATA 2
290
291 /* Input part of generic command type of input is command. */
292 #define GCMD_LAY_INPUT_CMD GENMASK_ULL(23, 16)
293
294 /* Generic command address sequence - address fields. */
295 #define GCMD_LAY_INPUT_ADDR GENMASK_ULL(63, 16)
296 /* Generic command address sequence - address size. */
297 #define GCMD_LAY_INPUT_ADDR_SIZE GENMASK_ULL(13, 11)
298
299 /* Transfer direction field of generic command data sequence. */
300 #define GCMD_DIR BIT_ULL(11)
301 /* Read transfer direction of generic command data sequence. */
302 #define GCMD_DIR_READ 0
303 /* Write transfer direction of generic command data sequence. */
304 #define GCMD_DIR_WRITE 1
305
306 /* ECC enabled flag of generic command data sequence - ECC enabled. */
307 #define GCMD_ECC_EN BIT_ULL(12)
308 /* Generic command data sequence - sector size. */
309 #define GCMD_SECT_SIZE GENMASK_ULL(31, 16)
310 /* Generic command data sequence - sector count. */
311 #define GCMD_SECT_CNT GENMASK_ULL(39, 32)
312 /* Generic command data sequence - last sector size. */
313 #define GCMD_LAST_SIZE GENMASK_ULL(55, 40)
314
315 /* CDMA descriptor fields. */
316 /* Erase command type of CDMA descriptor. */
317 #define CDMA_CT_ERASE 0x1000
318 /* Program page command type of CDMA descriptor. */
319 #define CDMA_CT_WR 0x2100
320 /* Read page command type of CDMA descriptor. */
321 #define CDMA_CT_RD 0x2200
322
323 /* Flash pointer memory shift. */
324 #define CDMA_CFPTR_MEM_SHIFT 24
325 /* Flash pointer memory mask. */
326 #define CDMA_CFPTR_MEM GENMASK(26, 24)
327
328 /*
329 * Command DMA descriptor flags. If set causes issue interrupt after
330 * the completion of descriptor processing.
331 */
332 #define CDMA_CF_INT BIT(8)
333 /*
334 * Command DMA descriptor flags - the next descriptor
335 * address field is valid and descriptor processing should continue.
336 */
337 #define CDMA_CF_CONT BIT(9)
338 /* DMA master flag of command DMA descriptor. */
339 #define CDMA_CF_DMA_MASTER BIT(10)
340
341 /* Operation complete status of command descriptor. */
342 #define CDMA_CS_COMP BIT(15)
343 /* Operation complete status of command descriptor. */
344 /* Command descriptor status - operation fail. */
345 #define CDMA_CS_FAIL BIT(14)
346 /* Command descriptor status - page erased. */
347 #define CDMA_CS_ERP BIT(11)
348 /* Command descriptor status - timeout occurred. */
349 #define CDMA_CS_TOUT BIT(10)
350 /*
351 * Maximum amount of correction applied to one ECC sector.
352 * It is part of command descriptor status.
353 */
354 #define CDMA_CS_MAXERR GENMASK(9, 2)
355 /* Command descriptor status - uncorrectable ECC error. */
356 #define CDMA_CS_UNCE BIT(1)
357 /* Command descriptor status - descriptor error. */
358 #define CDMA_CS_ERR BIT(0)
359
360 /* Status of operation - OK. */
361 #define STAT_OK 0
362 /* Status of operation - FAIL. */
363 #define STAT_FAIL 2
364 /* Status of operation - uncorrectable ECC error. */
365 #define STAT_ECC_UNCORR 3
366 /* Status of operation - page erased. */
367 #define STAT_ERASED 5
368 /* Status of operation - correctable ECC error. */
369 #define STAT_ECC_CORR 6
370 /* Status of operation - unsuspected state. */
371 #define STAT_UNKNOWN 7
372 /* Status of operation - operation is not completed yet. */
373 #define STAT_BUSY 0xFF
374
375 #define BCH_MAX_NUM_CORR_CAPS 8
376 #define BCH_MAX_NUM_SECTOR_SIZES 2
377
378 struct cadence_nand_timings {
379 u32 async_toggle_timings;
380 u32 timings0;
381 u32 timings1;
382 u32 timings2;
383 u32 dll_phy_ctrl;
384 u32 phy_ctrl;
385 u32 phy_dqs_timing;
386 u32 phy_gate_lpbk_ctrl;
387 };
388
389 /* Command DMA descriptor. */
390 struct cadence_nand_cdma_desc {
391 /* Next descriptor address. */
392 u64 next_pointer;
393
394 /* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
395 u32 flash_pointer;
396 /*field appears in HPNFC version 13*/
397 u16 bank;
398 u16 rsvd0;
399
400 /* Operation the controller needs to perform. */
401 u16 command_type;
402 u16 rsvd1;
403 /* Flags for operation of this command. */
404 u16 command_flags;
405 u16 rsvd2;
406
407 /* System/host memory address required for data DMA commands. */
408 u64 memory_pointer;
409
410 /* Status of operation. */
411 u32 status;
412 u32 rsvd3;
413
414 /* Address pointer to sync buffer location. */
415 u64 sync_flag_pointer;
416
417 /* Controls the buffer sync mechanism. */
418 u32 sync_arguments;
419 u32 rsvd4;
420
421 /* Control data pointer. */
422 u64 ctrl_data_ptr;
423 };
424
425 /* Interrupt status. */
426 struct cadence_nand_irq_status {
427 /* Thread operation complete status. */
428 u32 trd_status;
429 /* Thread operation error. */
430 u32 trd_error;
431 /* Controller status. */
432 u32 status;
433 };
434
435 /* Cadence NAND flash controller capabilities get from driver data. */
436 struct cadence_nand_dt_devdata {
437 /* Skew value of the output signals of the NAND Flash interface. */
438 u32 if_skew;
439 /* It informs if slave DMA interface is connected to DMA engine. */
440 unsigned int has_dma:1;
441 };
442
443 /* Cadence NAND flash controller capabilities read from registers. */
444 struct cdns_nand_caps {
445 /* Maximum number of banks supported by hardware. */
446 u8 max_banks;
447 /* Slave and Master DMA data width in bytes (4 or 8). */
448 u8 data_dma_width;
449 /* Control Data feature supported. */
450 bool data_control_supp;
451 /* Is PHY type DLL. */
452 bool is_phy_type_dll;
453 };
454
455 struct cdns_nand_ctrl {
456 struct device *dev;
457 struct nand_controller controller;
458 struct cadence_nand_cdma_desc *cdma_desc;
459 /* IP capability. */
460 const struct cadence_nand_dt_devdata *caps1;
461 struct cdns_nand_caps caps2;
462 u8 ctrl_rev;
463 dma_addr_t dma_cdma_desc;
464 u8 *buf;
465 u32 buf_size;
466 u8 curr_corr_str_idx;
467
468 /* Register interface. */
469 void __iomem *reg;
470
471 struct {
472 void __iomem *virt;
473 dma_addr_t dma;
474 dma_addr_t iova_dma;
475 u32 size;
476 } io;
477
478 int irq;
479 /* Interrupts that have happened. */
480 struct cadence_nand_irq_status irq_status;
481 /* Interrupts we are waiting for. */
482 struct cadence_nand_irq_status irq_mask;
483 struct completion complete;
484 /* Protect irq_mask and irq_status. */
485 spinlock_t irq_lock;
486
487 int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
488 struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
489 struct nand_ecc_caps ecc_caps;
490
491 int curr_trans_type;
492
493 struct dma_chan *dmac;
494
495 u32 nf_clk_rate;
496 /*
497 * Estimated Board delay. The value includes the total
498 * round trip delay for the signals and is used for deciding on values
499 * associated with data read capture.
500 */
501 u32 board_delay;
502
503 struct nand_chip *selected_chip;
504
505 unsigned long assigned_cs;
506 struct list_head chips;
507 u8 bch_metadata_size;
508 };
509
510 struct cdns_nand_chip {
511 struct cadence_nand_timings timings;
512 struct nand_chip chip;
513 u8 nsels;
514 struct list_head node;
515
516 /*
517 * part of oob area of NAND flash memory page.
518 * This part is available for user to read or write.
519 */
520 u32 avail_oob_size;
521
522 /* Sector size. There are few sectors per mtd->writesize */
523 u32 sector_size;
524 u32 sector_count;
525
526 /* Offset of BBM. */
527 u8 bbm_offs;
528 /* Number of bytes reserved for BBM. */
529 u8 bbm_len;
530 /* ECC strength index. */
531 u8 corr_str_idx;
532
533 u8 cs[] __counted_by(nsels);
534 };
535
536 static inline struct
to_cdns_nand_chip(struct nand_chip * chip)537 cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
538 {
539 return container_of(chip, struct cdns_nand_chip, chip);
540 }
541
542 static inline struct
to_cdns_nand_ctrl(struct nand_controller * controller)543 cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
544 {
545 return container_of(controller, struct cdns_nand_ctrl, controller);
546 }
547
548 static bool
cadence_nand_dma_buf_ok(struct cdns_nand_ctrl * cdns_ctrl,const void * buf,u32 buf_len)549 cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
550 u32 buf_len)
551 {
552 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
553
554 return buf && virt_addr_valid(buf) &&
555 likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
556 likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
557 }
558
cadence_nand_wait_for_value(struct cdns_nand_ctrl * cdns_ctrl,u32 reg_offset,u32 timeout_us,u32 mask,bool is_clear)559 static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
560 u32 reg_offset, u32 timeout_us,
561 u32 mask, bool is_clear)
562 {
563 u32 val;
564 int ret;
565
566 ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
567 val, !(val & mask) == is_clear,
568 10, timeout_us);
569
570 if (ret < 0) {
571 dev_err(cdns_ctrl->dev,
572 "Timeout while waiting for reg %x with mask %x is clear %d\n",
573 reg_offset, mask, is_clear);
574 }
575
576 return ret;
577 }
578
cadence_nand_set_ecc_enable(struct cdns_nand_ctrl * cdns_ctrl,bool enable)579 static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
580 bool enable)
581 {
582 u32 reg;
583
584 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
585 1000000,
586 CTRL_STATUS_CTRL_BUSY, true))
587 return -ETIMEDOUT;
588
589 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
590
591 if (enable)
592 reg |= ECC_CONFIG_0_ECC_EN;
593 else
594 reg &= ~ECC_CONFIG_0_ECC_EN;
595
596 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
597
598 return 0;
599 }
600
cadence_nand_set_ecc_strength(struct cdns_nand_ctrl * cdns_ctrl,u8 corr_str_idx)601 static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
602 u8 corr_str_idx)
603 {
604 u32 reg;
605
606 if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
607 return;
608
609 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
610 reg &= ~ECC_CONFIG_0_CORR_STR;
611 reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
612 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
613
614 cdns_ctrl->curr_corr_str_idx = corr_str_idx;
615 }
616
cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl * cdns_ctrl,u8 strength)617 static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
618 u8 strength)
619 {
620 int i, corr_str_idx = -1;
621
622 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
623 if (cdns_ctrl->ecc_strengths[i] == strength) {
624 corr_str_idx = i;
625 break;
626 }
627 }
628
629 return corr_str_idx;
630 }
631
cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl * cdns_ctrl,u16 marker_value)632 static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
633 u16 marker_value)
634 {
635 u32 reg;
636
637 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
638 1000000,
639 CTRL_STATUS_CTRL_BUSY, true))
640 return -ETIMEDOUT;
641
642 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
643 reg &= ~SKIP_BYTES_MARKER_VALUE;
644 reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
645 marker_value);
646
647 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
648
649 return 0;
650 }
651
cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl * cdns_ctrl,u8 num_of_bytes,u32 offset_value,int enable)652 static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
653 u8 num_of_bytes,
654 u32 offset_value,
655 int enable)
656 {
657 u32 reg, skip_bytes_offset;
658
659 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
660 1000000,
661 CTRL_STATUS_CTRL_BUSY, true))
662 return -ETIMEDOUT;
663
664 if (!enable) {
665 num_of_bytes = 0;
666 offset_value = 0;
667 }
668
669 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
670 reg &= ~SKIP_BYTES_NUM_OF_BYTES;
671 reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
672 num_of_bytes);
673 skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
674 offset_value);
675
676 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
677 writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
678
679 return 0;
680 }
681
682 /* Functions enables/disables hardware detection of erased data */
cadence_nand_set_erase_detection(struct cdns_nand_ctrl * cdns_ctrl,bool enable,u8 bitflips_threshold)683 static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
684 bool enable,
685 u8 bitflips_threshold)
686 {
687 u32 reg;
688
689 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
690
691 if (enable)
692 reg |= ECC_CONFIG_0_ERASE_DET_EN;
693 else
694 reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
695
696 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
697 writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
698 }
699
cadence_nand_set_access_width16(struct cdns_nand_ctrl * cdns_ctrl,bool bit_bus16)700 static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
701 bool bit_bus16)
702 {
703 u32 reg;
704
705 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
706 1000000,
707 CTRL_STATUS_CTRL_BUSY, true))
708 return -ETIMEDOUT;
709
710 reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
711
712 if (!bit_bus16)
713 reg &= ~COMMON_SET_DEVICE_16BIT;
714 else
715 reg |= COMMON_SET_DEVICE_16BIT;
716 writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
717
718 return 0;
719 }
720
721 static void
cadence_nand_clear_interrupt(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_status)722 cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
723 struct cadence_nand_irq_status *irq_status)
724 {
725 writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
726 writel_relaxed(irq_status->trd_status,
727 cdns_ctrl->reg + TRD_COMP_INT_STATUS);
728 writel_relaxed(irq_status->trd_error,
729 cdns_ctrl->reg + TRD_ERR_INT_STATUS);
730 }
731
732 static void
cadence_nand_read_int_status(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_status)733 cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
734 struct cadence_nand_irq_status *irq_status)
735 {
736 irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
737 irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
738 + TRD_COMP_INT_STATUS);
739 irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
740 + TRD_ERR_INT_STATUS);
741 }
742
irq_detected(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_status)743 static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
744 struct cadence_nand_irq_status *irq_status)
745 {
746 cadence_nand_read_int_status(cdns_ctrl, irq_status);
747
748 return irq_status->status || irq_status->trd_status ||
749 irq_status->trd_error;
750 }
751
cadence_nand_reset_irq(struct cdns_nand_ctrl * cdns_ctrl)752 static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
753 {
754 unsigned long flags;
755
756 spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
757 memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
758 memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
759 spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
760 }
761
762 /*
763 * This is the interrupt service routine. It handles all interrupts
764 * sent to this device.
765 */
cadence_nand_isr(int irq,void * dev_id)766 static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
767 {
768 struct cdns_nand_ctrl *cdns_ctrl = dev_id;
769 struct cadence_nand_irq_status irq_status;
770 irqreturn_t result = IRQ_NONE;
771
772 spin_lock(&cdns_ctrl->irq_lock);
773
774 if (irq_detected(cdns_ctrl, &irq_status)) {
775 /* Handle interrupt. */
776 /* First acknowledge it. */
777 cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
778 /* Status in the device context for someone to read. */
779 cdns_ctrl->irq_status.status |= irq_status.status;
780 cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
781 cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
782 /* Notify anyone who cares that it happened. */
783 complete(&cdns_ctrl->complete);
784 /* Tell the OS that we've handled this. */
785 result = IRQ_HANDLED;
786 }
787 spin_unlock(&cdns_ctrl->irq_lock);
788
789 return result;
790 }
791
cadence_nand_set_irq_mask(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_mask)792 static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
793 struct cadence_nand_irq_status *irq_mask)
794 {
795 writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
796 cdns_ctrl->reg + INTR_ENABLE);
797
798 writel_relaxed(irq_mask->trd_error,
799 cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
800 }
801
802 static void
cadence_nand_wait_for_irq(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_irq_status * irq_mask,struct cadence_nand_irq_status * irq_status)803 cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
804 struct cadence_nand_irq_status *irq_mask,
805 struct cadence_nand_irq_status *irq_status)
806 {
807 unsigned long timeout = msecs_to_jiffies(10000);
808 unsigned long time_left;
809
810 time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
811 timeout);
812
813 *irq_status = cdns_ctrl->irq_status;
814 if (time_left == 0) {
815 /* Timeout error. */
816 dev_err(cdns_ctrl->dev, "timeout occurred:\n");
817 dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
818 irq_status->status, irq_mask->status);
819 dev_err(cdns_ctrl->dev,
820 "\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
821 irq_status->trd_status, irq_mask->trd_status);
822 dev_err(cdns_ctrl->dev,
823 "\t trd_error = 0x%x, trd_error mask = 0x%x\n",
824 irq_status->trd_error, irq_mask->trd_error);
825 }
826 }
827
828 /* Execute generic command on NAND controller. */
cadence_nand_generic_cmd_send(struct cdns_nand_ctrl * cdns_ctrl,u8 chip_nr,u64 mini_ctrl_cmd)829 static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
830 u8 chip_nr,
831 u64 mini_ctrl_cmd)
832 {
833 u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
834
835 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
836 mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
837 mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
838
839 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
840 1000000,
841 CTRL_STATUS_CTRL_BUSY, true))
842 return -ETIMEDOUT;
843
844 cadence_nand_reset_irq(cdns_ctrl);
845
846 writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
847 writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
848
849 /* Select generic command. */
850 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
851 /* Thread number. */
852 reg |= FIELD_PREP(CMD_REG0_TN, 0);
853
854 /* Issue command. */
855 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
856
857 return 0;
858 }
859
860 /* Wait for data on slave DMA interface. */
cadence_nand_wait_on_sdma(struct cdns_nand_ctrl * cdns_ctrl,u8 * out_sdma_trd,u32 * out_sdma_size)861 static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
862 u8 *out_sdma_trd,
863 u32 *out_sdma_size)
864 {
865 struct cadence_nand_irq_status irq_mask, irq_status;
866
867 irq_mask.trd_status = 0;
868 irq_mask.trd_error = 0;
869 irq_mask.status = INTR_STATUS_SDMA_TRIGG
870 | INTR_STATUS_SDMA_ERR
871 | INTR_STATUS_UNSUPP_CMD;
872
873 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
874 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
875 if (irq_status.status == 0) {
876 dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
877 return -ETIMEDOUT;
878 }
879
880 if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
881 *out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
882 *out_sdma_trd = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
883 *out_sdma_trd =
884 FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
885 } else {
886 dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
887 irq_status.status);
888 return -EIO;
889 }
890
891 return 0;
892 }
893
cadence_nand_get_caps(struct cdns_nand_ctrl * cdns_ctrl)894 static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
895 {
896 u32 reg;
897
898 reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
899
900 cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
901
902 if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
903 cdns_ctrl->caps2.data_dma_width = 8;
904 else
905 cdns_ctrl->caps2.data_dma_width = 4;
906
907 if (reg & CTRL_FEATURES_CONTROL_DATA)
908 cdns_ctrl->caps2.data_control_supp = true;
909
910 if (reg & (CTRL_FEATURES_NVDDR_2_3
911 | CTRL_FEATURES_NVDDR))
912 cdns_ctrl->caps2.is_phy_type_dll = true;
913 }
914
915 /* Prepare CDMA descriptor. */
916 static void
cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl * cdns_ctrl,char nf_mem,u32 flash_ptr,dma_addr_t mem_ptr,dma_addr_t ctrl_data_ptr,u16 ctype)917 cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
918 char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
919 dma_addr_t ctrl_data_ptr, u16 ctype)
920 {
921 struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
922
923 memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
924
925 /* Set fields for one descriptor. */
926 cdma_desc->flash_pointer = flash_ptr;
927 if (cdns_ctrl->ctrl_rev >= 13)
928 cdma_desc->bank = nf_mem;
929 else
930 cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
931
932 cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
933 cdma_desc->command_flags |= CDMA_CF_INT;
934
935 cdma_desc->memory_pointer = mem_ptr;
936 cdma_desc->status = 0;
937 cdma_desc->sync_flag_pointer = 0;
938 cdma_desc->sync_arguments = 0;
939
940 cdma_desc->command_type = ctype;
941 cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
942 }
943
cadence_nand_check_desc_error(struct cdns_nand_ctrl * cdns_ctrl,u32 desc_status)944 static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
945 u32 desc_status)
946 {
947 if (desc_status & CDMA_CS_ERP)
948 return STAT_ERASED;
949
950 if (desc_status & CDMA_CS_UNCE)
951 return STAT_ECC_UNCORR;
952
953 if (desc_status & CDMA_CS_ERR) {
954 dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
955 return STAT_FAIL;
956 }
957
958 if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
959 return STAT_ECC_CORR;
960
961 return STAT_FAIL;
962 }
963
cadence_nand_cdma_finish(struct cdns_nand_ctrl * cdns_ctrl)964 static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
965 {
966 struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
967 u8 status = STAT_BUSY;
968
969 if (desc_ptr->status & CDMA_CS_FAIL) {
970 status = cadence_nand_check_desc_error(cdns_ctrl,
971 desc_ptr->status);
972 dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
973 } else if (desc_ptr->status & CDMA_CS_COMP) {
974 /* Descriptor finished with no errors. */
975 if (desc_ptr->command_flags & CDMA_CF_CONT) {
976 dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
977 status = STAT_UNKNOWN;
978 } else {
979 /* Last descriptor. */
980 status = STAT_OK;
981 }
982 }
983
984 return status;
985 }
986
cadence_nand_cdma_send(struct cdns_nand_ctrl * cdns_ctrl,u8 thread)987 static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
988 u8 thread)
989 {
990 u32 reg;
991 int status;
992
993 /* Wait for thread ready. */
994 status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
995 1000000,
996 BIT(thread), true);
997 if (status)
998 return status;
999
1000 cadence_nand_reset_irq(cdns_ctrl);
1001 reinit_completion(&cdns_ctrl->complete);
1002
1003 writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1004 cdns_ctrl->reg + CMD_REG2);
1005 writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1006
1007 /* Select CDMA mode. */
1008 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1009 /* Thread number. */
1010 reg |= FIELD_PREP(CMD_REG0_TN, thread);
1011 /* Issue command. */
1012 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1013
1014 return 0;
1015 }
1016
1017 /* Send SDMA command and wait for finish. */
1018 static u32
cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl * cdns_ctrl,u8 thread)1019 cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1020 u8 thread)
1021 {
1022 struct cadence_nand_irq_status irq_mask, irq_status = {0};
1023 int status;
1024
1025 irq_mask.trd_status = BIT(thread);
1026 irq_mask.trd_error = BIT(thread);
1027 irq_mask.status = INTR_STATUS_CDMA_TERR;
1028
1029 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1030
1031 status = cadence_nand_cdma_send(cdns_ctrl, thread);
1032 if (status)
1033 return status;
1034
1035 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1036
1037 if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1038 irq_status.trd_error == 0) {
1039 dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1040 return -ETIMEDOUT;
1041 }
1042 if (irq_status.status & irq_mask.status) {
1043 dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1044 return -EIO;
1045 }
1046
1047 return 0;
1048 }
1049
1050 /*
1051 * ECC size depends on configured ECC strength and on maximum supported
1052 * ECC step size.
1053 */
cadence_nand_calc_ecc_bytes(int max_step_size,int strength)1054 static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1055 {
1056 int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1057
1058 return ALIGN(nbytes, 2);
1059 }
1060
1061 #define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1062 static int \
1063 cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1064 int strength)\
1065 {\
1066 return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1067 }
1068
1069 CADENCE_NAND_CALC_ECC_BYTES(256)
1070 CADENCE_NAND_CALC_ECC_BYTES(512)
1071 CADENCE_NAND_CALC_ECC_BYTES(1024)
1072 CADENCE_NAND_CALC_ECC_BYTES(2048)
1073 CADENCE_NAND_CALC_ECC_BYTES(4096)
1074
1075 /* Function reads BCH capabilities. */
cadence_nand_read_bch_caps(struct cdns_nand_ctrl * cdns_ctrl)1076 static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1077 {
1078 struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1079 int max_step_size = 0, nstrengths, i;
1080 u32 reg;
1081
1082 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1083 cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1084 if (cdns_ctrl->bch_metadata_size < 4) {
1085 dev_err(cdns_ctrl->dev,
1086 "Driver needs at least 4 bytes of BCH meta data\n");
1087 return -EIO;
1088 }
1089
1090 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1091 cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1092 cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1093 cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1094 cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1095
1096 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1097 cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1098 cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1099 cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1100 cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1101
1102 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1103 cdns_ctrl->ecc_stepinfos[0].stepsize =
1104 FIELD_GET(BCH_CFG_2_SECT_0, reg);
1105
1106 cdns_ctrl->ecc_stepinfos[1].stepsize =
1107 FIELD_GET(BCH_CFG_2_SECT_1, reg);
1108
1109 nstrengths = 0;
1110 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1111 if (cdns_ctrl->ecc_strengths[i] != 0)
1112 nstrengths++;
1113 }
1114
1115 ecc_caps->nstepinfos = 0;
1116 for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1117 /* ECC strengths are common for all step infos. */
1118 cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1119 cdns_ctrl->ecc_stepinfos[i].strengths =
1120 cdns_ctrl->ecc_strengths;
1121
1122 if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1123 ecc_caps->nstepinfos++;
1124
1125 if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1126 max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1127 }
1128 ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1129
1130 switch (max_step_size) {
1131 case 256:
1132 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1133 break;
1134 case 512:
1135 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1136 break;
1137 case 1024:
1138 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1139 break;
1140 case 2048:
1141 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1142 break;
1143 case 4096:
1144 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1145 break;
1146 default:
1147 dev_err(cdns_ctrl->dev,
1148 "Unsupported sector size(ecc step size) %d\n",
1149 max_step_size);
1150 return -EIO;
1151 }
1152
1153 return 0;
1154 }
1155
1156 /* Hardware initialization. */
cadence_nand_hw_init(struct cdns_nand_ctrl * cdns_ctrl)1157 static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1158 {
1159 int status;
1160 u32 reg;
1161
1162 status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1163 1000000,
1164 CTRL_STATUS_INIT_COMP, false);
1165 if (status)
1166 return status;
1167
1168 reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1169 cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1170
1171 dev_info(cdns_ctrl->dev,
1172 "%s: cadence nand controller version reg %x\n",
1173 __func__, reg);
1174
1175 /* Disable cache and multiplane. */
1176 writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1177 writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1178
1179 /* Clear all interrupts. */
1180 writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1181
1182 cadence_nand_get_caps(cdns_ctrl);
1183 if (cadence_nand_read_bch_caps(cdns_ctrl))
1184 return -EIO;
1185
1186 #ifndef CONFIG_64BIT
1187 if (cdns_ctrl->caps2.data_dma_width == 8) {
1188 dev_err(cdns_ctrl->dev,
1189 "cannot access 64-bit dma on !64-bit architectures");
1190 return -EIO;
1191 }
1192 #endif
1193
1194 /*
1195 * Set IO width access to 8.
1196 * It is because during SW device discovering width access
1197 * is expected to be 8.
1198 */
1199 status = cadence_nand_set_access_width16(cdns_ctrl, false);
1200
1201 return status;
1202 }
1203
1204 #define TT_MAIN_OOB_AREAS 2
1205 #define TT_RAW_PAGE 3
1206 #define TT_BBM 4
1207 #define TT_MAIN_OOB_AREA_EXT 5
1208
1209 /* Prepare size of data to transfer. */
1210 static void
cadence_nand_prepare_data_size(struct nand_chip * chip,int transfer_type)1211 cadence_nand_prepare_data_size(struct nand_chip *chip,
1212 int transfer_type)
1213 {
1214 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1215 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1216 struct mtd_info *mtd = nand_to_mtd(chip);
1217 u32 sec_size = 0, offset = 0, sec_cnt = 1;
1218 u32 last_sec_size = cdns_chip->sector_size;
1219 u32 data_ctrl_size = 0;
1220 u32 reg = 0;
1221
1222 if (cdns_ctrl->curr_trans_type == transfer_type)
1223 return;
1224
1225 switch (transfer_type) {
1226 case TT_MAIN_OOB_AREA_EXT:
1227 sec_cnt = cdns_chip->sector_count;
1228 sec_size = cdns_chip->sector_size;
1229 data_ctrl_size = cdns_chip->avail_oob_size;
1230 break;
1231 case TT_MAIN_OOB_AREAS:
1232 sec_cnt = cdns_chip->sector_count;
1233 last_sec_size = cdns_chip->sector_size
1234 + cdns_chip->avail_oob_size;
1235 sec_size = cdns_chip->sector_size;
1236 break;
1237 case TT_RAW_PAGE:
1238 last_sec_size = mtd->writesize + mtd->oobsize;
1239 break;
1240 case TT_BBM:
1241 offset = mtd->writesize + cdns_chip->bbm_offs;
1242 last_sec_size = 8;
1243 break;
1244 }
1245
1246 reg = 0;
1247 reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1248 reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1249 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1250
1251 reg = 0;
1252 reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1253 reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1254 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1255
1256 if (cdns_ctrl->caps2.data_control_supp) {
1257 reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1258 reg &= ~CONTROL_DATA_CTRL_SIZE;
1259 reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1260 writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1261 }
1262
1263 cdns_ctrl->curr_trans_type = transfer_type;
1264 }
1265
1266 static int
cadence_nand_cdma_transfer(struct cdns_nand_ctrl * cdns_ctrl,u8 chip_nr,int page,void * buf,void * ctrl_dat,u32 buf_size,u32 ctrl_dat_size,enum dma_data_direction dir,bool with_ecc)1267 cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1268 int page, void *buf, void *ctrl_dat, u32 buf_size,
1269 u32 ctrl_dat_size, enum dma_data_direction dir,
1270 bool with_ecc)
1271 {
1272 dma_addr_t dma_buf, dma_ctrl_dat = 0;
1273 u8 thread_nr = chip_nr;
1274 int status;
1275 u16 ctype;
1276
1277 if (dir == DMA_FROM_DEVICE)
1278 ctype = CDMA_CT_RD;
1279 else
1280 ctype = CDMA_CT_WR;
1281
1282 cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1283
1284 dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1285 if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1286 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1287 return -EIO;
1288 }
1289
1290 if (ctrl_dat && ctrl_dat_size) {
1291 dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1292 ctrl_dat_size, dir);
1293 if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1294 dma_unmap_single(cdns_ctrl->dev, dma_buf,
1295 buf_size, dir);
1296 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1297 return -EIO;
1298 }
1299 }
1300
1301 cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1302 dma_buf, dma_ctrl_dat, ctype);
1303
1304 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1305
1306 dma_unmap_single(cdns_ctrl->dev, dma_buf,
1307 buf_size, dir);
1308
1309 if (ctrl_dat && ctrl_dat_size)
1310 dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1311 ctrl_dat_size, dir);
1312 if (status)
1313 return status;
1314
1315 return cadence_nand_cdma_finish(cdns_ctrl);
1316 }
1317
cadence_nand_set_timings(struct cdns_nand_ctrl * cdns_ctrl,struct cadence_nand_timings * t)1318 static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1319 struct cadence_nand_timings *t)
1320 {
1321 writel_relaxed(t->async_toggle_timings,
1322 cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1323 writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1324 writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1325 writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1326
1327 if (cdns_ctrl->caps2.is_phy_type_dll)
1328 writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1329
1330 writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1331
1332 if (cdns_ctrl->caps2.is_phy_type_dll) {
1333 writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1334 writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1335 writel_relaxed(t->phy_dqs_timing,
1336 cdns_ctrl->reg + PHY_DQS_TIMING);
1337 writel_relaxed(t->phy_gate_lpbk_ctrl,
1338 cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1339 writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1340 cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1341 writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1342 }
1343 }
1344
cadence_nand_select_target(struct nand_chip * chip)1345 static int cadence_nand_select_target(struct nand_chip *chip)
1346 {
1347 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1348 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1349
1350 if (chip == cdns_ctrl->selected_chip)
1351 return 0;
1352
1353 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1354 1000000,
1355 CTRL_STATUS_CTRL_BUSY, true))
1356 return -ETIMEDOUT;
1357
1358 cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1359
1360 cadence_nand_set_ecc_strength(cdns_ctrl,
1361 cdns_chip->corr_str_idx);
1362
1363 cadence_nand_set_erase_detection(cdns_ctrl, true,
1364 chip->ecc.strength);
1365
1366 cdns_ctrl->curr_trans_type = -1;
1367 cdns_ctrl->selected_chip = chip;
1368
1369 return 0;
1370 }
1371
cadence_nand_erase(struct nand_chip * chip,u32 page)1372 static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1373 {
1374 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1375 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1376 int status;
1377 u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1378
1379 cadence_nand_cdma_desc_prepare(cdns_ctrl,
1380 cdns_chip->cs[chip->cur_cs],
1381 page, 0, 0,
1382 CDMA_CT_ERASE);
1383 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1384 if (status) {
1385 dev_err(cdns_ctrl->dev, "erase operation failed\n");
1386 return -EIO;
1387 }
1388
1389 status = cadence_nand_cdma_finish(cdns_ctrl);
1390 if (status)
1391 return status;
1392
1393 return 0;
1394 }
1395
cadence_nand_read_bbm(struct nand_chip * chip,int page,u8 * buf)1396 static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1397 {
1398 int status;
1399 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1400 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1401 struct mtd_info *mtd = nand_to_mtd(chip);
1402
1403 cadence_nand_prepare_data_size(chip, TT_BBM);
1404
1405 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1406
1407 /*
1408 * Read only bad block marker from offset
1409 * defined by a memory manufacturer.
1410 */
1411 status = cadence_nand_cdma_transfer(cdns_ctrl,
1412 cdns_chip->cs[chip->cur_cs],
1413 page, cdns_ctrl->buf, NULL,
1414 mtd->oobsize,
1415 0, DMA_FROM_DEVICE, false);
1416 if (status) {
1417 dev_err(cdns_ctrl->dev, "read BBM failed\n");
1418 return -EIO;
1419 }
1420
1421 memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1422
1423 return 0;
1424 }
1425
cadence_nand_write_page(struct nand_chip * chip,const u8 * buf,int oob_required,int page)1426 static int cadence_nand_write_page(struct nand_chip *chip,
1427 const u8 *buf, int oob_required,
1428 int page)
1429 {
1430 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1431 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1432 struct mtd_info *mtd = nand_to_mtd(chip);
1433 int status;
1434 u16 marker_val = 0xFFFF;
1435
1436 status = cadence_nand_select_target(chip);
1437 if (status)
1438 return status;
1439
1440 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1441 mtd->writesize
1442 + cdns_chip->bbm_offs,
1443 1);
1444
1445 if (oob_required) {
1446 marker_val = *(u16 *)(chip->oob_poi
1447 + cdns_chip->bbm_offs);
1448 } else {
1449 /* Set oob data to 0xFF. */
1450 memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1451 cdns_chip->avail_oob_size);
1452 }
1453
1454 cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1455
1456 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1457
1458 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1459 cdns_ctrl->caps2.data_control_supp) {
1460 u8 *oob;
1461
1462 if (oob_required)
1463 oob = chip->oob_poi;
1464 else
1465 oob = cdns_ctrl->buf + mtd->writesize;
1466
1467 status = cadence_nand_cdma_transfer(cdns_ctrl,
1468 cdns_chip->cs[chip->cur_cs],
1469 page, (void *)buf, oob,
1470 mtd->writesize,
1471 cdns_chip->avail_oob_size,
1472 DMA_TO_DEVICE, true);
1473 if (status) {
1474 dev_err(cdns_ctrl->dev, "write page failed\n");
1475 return -EIO;
1476 }
1477
1478 return 0;
1479 }
1480
1481 if (oob_required) {
1482 /* Transfer the data to the oob area. */
1483 memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1484 cdns_chip->avail_oob_size);
1485 }
1486
1487 memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1488
1489 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1490
1491 return cadence_nand_cdma_transfer(cdns_ctrl,
1492 cdns_chip->cs[chip->cur_cs],
1493 page, cdns_ctrl->buf, NULL,
1494 mtd->writesize
1495 + cdns_chip->avail_oob_size,
1496 0, DMA_TO_DEVICE, true);
1497 }
1498
cadence_nand_write_oob(struct nand_chip * chip,int page)1499 static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1500 {
1501 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1502 struct mtd_info *mtd = nand_to_mtd(chip);
1503
1504 memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1505
1506 return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1507 }
1508
cadence_nand_write_page_raw(struct nand_chip * chip,const u8 * buf,int oob_required,int page)1509 static int cadence_nand_write_page_raw(struct nand_chip *chip,
1510 const u8 *buf, int oob_required,
1511 int page)
1512 {
1513 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1514 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1515 struct mtd_info *mtd = nand_to_mtd(chip);
1516 int writesize = mtd->writesize;
1517 int oobsize = mtd->oobsize;
1518 int ecc_steps = chip->ecc.steps;
1519 int ecc_size = chip->ecc.size;
1520 int ecc_bytes = chip->ecc.bytes;
1521 void *tmp_buf = cdns_ctrl->buf;
1522 int oob_skip = cdns_chip->bbm_len;
1523 size_t size = writesize + oobsize;
1524 int i, pos, len;
1525 int status = 0;
1526
1527 status = cadence_nand_select_target(chip);
1528 if (status)
1529 return status;
1530
1531 /*
1532 * Fill the buffer with 0xff first except the full page transfer.
1533 * This simplifies the logic.
1534 */
1535 if (!buf || !oob_required)
1536 memset(tmp_buf, 0xff, size);
1537
1538 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1539
1540 /* Arrange the buffer for syndrome payload/ecc layout. */
1541 if (buf) {
1542 for (i = 0; i < ecc_steps; i++) {
1543 pos = i * (ecc_size + ecc_bytes);
1544 len = ecc_size;
1545
1546 if (pos >= writesize)
1547 pos += oob_skip;
1548 else if (pos + len > writesize)
1549 len = writesize - pos;
1550
1551 memcpy(tmp_buf + pos, buf, len);
1552 buf += len;
1553 if (len < ecc_size) {
1554 len = ecc_size - len;
1555 memcpy(tmp_buf + writesize + oob_skip, buf,
1556 len);
1557 buf += len;
1558 }
1559 }
1560 }
1561
1562 if (oob_required) {
1563 const u8 *oob = chip->oob_poi;
1564 u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1565 (cdns_chip->sector_size + chip->ecc.bytes)
1566 + cdns_chip->sector_size + oob_skip;
1567
1568 /* BBM at the beginning of the OOB area. */
1569 memcpy(tmp_buf + writesize, oob, oob_skip);
1570
1571 /* OOB free. */
1572 memcpy(tmp_buf + oob_data_offset, oob,
1573 cdns_chip->avail_oob_size);
1574 oob += cdns_chip->avail_oob_size;
1575
1576 /* OOB ECC. */
1577 for (i = 0; i < ecc_steps; i++) {
1578 pos = ecc_size + i * (ecc_size + ecc_bytes);
1579 if (i == (ecc_steps - 1))
1580 pos += cdns_chip->avail_oob_size;
1581
1582 len = ecc_bytes;
1583
1584 if (pos >= writesize)
1585 pos += oob_skip;
1586 else if (pos + len > writesize)
1587 len = writesize - pos;
1588
1589 memcpy(tmp_buf + pos, oob, len);
1590 oob += len;
1591 if (len < ecc_bytes) {
1592 len = ecc_bytes - len;
1593 memcpy(tmp_buf + writesize + oob_skip, oob,
1594 len);
1595 oob += len;
1596 }
1597 }
1598 }
1599
1600 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1601
1602 return cadence_nand_cdma_transfer(cdns_ctrl,
1603 cdns_chip->cs[chip->cur_cs],
1604 page, cdns_ctrl->buf, NULL,
1605 mtd->writesize +
1606 mtd->oobsize,
1607 0, DMA_TO_DEVICE, false);
1608 }
1609
cadence_nand_write_oob_raw(struct nand_chip * chip,int page)1610 static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1611 int page)
1612 {
1613 return cadence_nand_write_page_raw(chip, NULL, true, page);
1614 }
1615
cadence_nand_read_page(struct nand_chip * chip,u8 * buf,int oob_required,int page)1616 static int cadence_nand_read_page(struct nand_chip *chip,
1617 u8 *buf, int oob_required, int page)
1618 {
1619 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1620 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1621 struct mtd_info *mtd = nand_to_mtd(chip);
1622 int status = 0;
1623 int ecc_err_count = 0;
1624
1625 status = cadence_nand_select_target(chip);
1626 if (status)
1627 return status;
1628
1629 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1630 mtd->writesize
1631 + cdns_chip->bbm_offs, 1);
1632
1633 /*
1634 * If data buffer can be accessed by DMA and data_control feature
1635 * is supported then transfer data and oob directly.
1636 */
1637 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1638 cdns_ctrl->caps2.data_control_supp) {
1639 u8 *oob;
1640
1641 if (oob_required)
1642 oob = chip->oob_poi;
1643 else
1644 oob = cdns_ctrl->buf + mtd->writesize;
1645
1646 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1647 status = cadence_nand_cdma_transfer(cdns_ctrl,
1648 cdns_chip->cs[chip->cur_cs],
1649 page, buf, oob,
1650 mtd->writesize,
1651 cdns_chip->avail_oob_size,
1652 DMA_FROM_DEVICE, true);
1653 /* Otherwise use bounce buffer. */
1654 } else {
1655 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1656 status = cadence_nand_cdma_transfer(cdns_ctrl,
1657 cdns_chip->cs[chip->cur_cs],
1658 page, cdns_ctrl->buf,
1659 NULL, mtd->writesize
1660 + cdns_chip->avail_oob_size,
1661 0, DMA_FROM_DEVICE, true);
1662
1663 memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1664 if (oob_required)
1665 memcpy(chip->oob_poi,
1666 cdns_ctrl->buf + mtd->writesize,
1667 mtd->oobsize);
1668 }
1669
1670 switch (status) {
1671 case STAT_ECC_UNCORR:
1672 mtd->ecc_stats.failed++;
1673 ecc_err_count++;
1674 break;
1675 case STAT_ECC_CORR:
1676 ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1677 cdns_ctrl->cdma_desc->status);
1678 mtd->ecc_stats.corrected += ecc_err_count;
1679 break;
1680 case STAT_ERASED:
1681 case STAT_OK:
1682 break;
1683 default:
1684 dev_err(cdns_ctrl->dev, "read page failed\n");
1685 return -EIO;
1686 }
1687
1688 if (oob_required)
1689 if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1690 return -EIO;
1691
1692 return ecc_err_count;
1693 }
1694
1695 /* Reads OOB data from the device. */
cadence_nand_read_oob(struct nand_chip * chip,int page)1696 static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1697 {
1698 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1699
1700 return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1701 }
1702
cadence_nand_read_page_raw(struct nand_chip * chip,u8 * buf,int oob_required,int page)1703 static int cadence_nand_read_page_raw(struct nand_chip *chip,
1704 u8 *buf, int oob_required, int page)
1705 {
1706 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1707 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1708 struct mtd_info *mtd = nand_to_mtd(chip);
1709 int oob_skip = cdns_chip->bbm_len;
1710 int writesize = mtd->writesize;
1711 int ecc_steps = chip->ecc.steps;
1712 int ecc_size = chip->ecc.size;
1713 int ecc_bytes = chip->ecc.bytes;
1714 void *tmp_buf = cdns_ctrl->buf;
1715 int i, pos, len;
1716 int status = 0;
1717
1718 status = cadence_nand_select_target(chip);
1719 if (status)
1720 return status;
1721
1722 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1723
1724 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1725 status = cadence_nand_cdma_transfer(cdns_ctrl,
1726 cdns_chip->cs[chip->cur_cs],
1727 page, cdns_ctrl->buf, NULL,
1728 mtd->writesize
1729 + mtd->oobsize,
1730 0, DMA_FROM_DEVICE, false);
1731
1732 switch (status) {
1733 case STAT_ERASED:
1734 case STAT_OK:
1735 break;
1736 default:
1737 dev_err(cdns_ctrl->dev, "read raw page failed\n");
1738 return -EIO;
1739 }
1740
1741 /* Arrange the buffer for syndrome payload/ecc layout. */
1742 if (buf) {
1743 for (i = 0; i < ecc_steps; i++) {
1744 pos = i * (ecc_size + ecc_bytes);
1745 len = ecc_size;
1746
1747 if (pos >= writesize)
1748 pos += oob_skip;
1749 else if (pos + len > writesize)
1750 len = writesize - pos;
1751
1752 memcpy(buf, tmp_buf + pos, len);
1753 buf += len;
1754 if (len < ecc_size) {
1755 len = ecc_size - len;
1756 memcpy(buf, tmp_buf + writesize + oob_skip,
1757 len);
1758 buf += len;
1759 }
1760 }
1761 }
1762
1763 if (oob_required) {
1764 u8 *oob = chip->oob_poi;
1765 u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1766 (cdns_chip->sector_size + chip->ecc.bytes)
1767 + cdns_chip->sector_size + oob_skip;
1768
1769 /* OOB free. */
1770 memcpy(oob, tmp_buf + oob_data_offset,
1771 cdns_chip->avail_oob_size);
1772
1773 /* BBM at the beginning of the OOB area. */
1774 memcpy(oob, tmp_buf + writesize, oob_skip);
1775
1776 oob += cdns_chip->avail_oob_size;
1777
1778 /* OOB ECC */
1779 for (i = 0; i < ecc_steps; i++) {
1780 pos = ecc_size + i * (ecc_size + ecc_bytes);
1781 len = ecc_bytes;
1782
1783 if (i == (ecc_steps - 1))
1784 pos += cdns_chip->avail_oob_size;
1785
1786 if (pos >= writesize)
1787 pos += oob_skip;
1788 else if (pos + len > writesize)
1789 len = writesize - pos;
1790
1791 memcpy(oob, tmp_buf + pos, len);
1792 oob += len;
1793 if (len < ecc_bytes) {
1794 len = ecc_bytes - len;
1795 memcpy(oob, tmp_buf + writesize + oob_skip,
1796 len);
1797 oob += len;
1798 }
1799 }
1800 }
1801
1802 return 0;
1803 }
1804
cadence_nand_read_oob_raw(struct nand_chip * chip,int page)1805 static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1806 int page)
1807 {
1808 return cadence_nand_read_page_raw(chip, NULL, true, page);
1809 }
1810
cadence_nand_slave_dma_transfer_finished(void * data)1811 static void cadence_nand_slave_dma_transfer_finished(void *data)
1812 {
1813 struct completion *finished = data;
1814
1815 complete(finished);
1816 }
1817
cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl * cdns_ctrl,void * buf,dma_addr_t dev_dma,size_t len,enum dma_data_direction dir)1818 static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1819 void *buf,
1820 dma_addr_t dev_dma, size_t len,
1821 enum dma_data_direction dir)
1822 {
1823 DECLARE_COMPLETION_ONSTACK(finished);
1824 struct dma_chan *chan;
1825 struct dma_device *dma_dev;
1826 dma_addr_t src_dma, dst_dma, buf_dma;
1827 struct dma_async_tx_descriptor *tx;
1828 dma_cookie_t cookie;
1829
1830 chan = cdns_ctrl->dmac;
1831 dma_dev = chan->device;
1832
1833 buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1834 if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1835 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1836 goto err;
1837 }
1838
1839 if (dir == DMA_FROM_DEVICE) {
1840 src_dma = cdns_ctrl->io.iova_dma;
1841 dst_dma = buf_dma;
1842 } else {
1843 src_dma = buf_dma;
1844 dst_dma = cdns_ctrl->io.iova_dma;
1845 }
1846
1847 tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1848 DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1849 if (!tx) {
1850 dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1851 goto err_unmap;
1852 }
1853
1854 tx->callback = cadence_nand_slave_dma_transfer_finished;
1855 tx->callback_param = &finished;
1856
1857 cookie = dmaengine_submit(tx);
1858 if (dma_submit_error(cookie)) {
1859 dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1860 goto err_unmap;
1861 }
1862
1863 dma_async_issue_pending(cdns_ctrl->dmac);
1864 wait_for_completion(&finished);
1865
1866 dma_unmap_single(dma_dev->dev, buf_dma, len, dir);
1867
1868 return 0;
1869
1870 err_unmap:
1871 dma_unmap_single(dma_dev->dev, buf_dma, len, dir);
1872
1873 err:
1874 dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1875
1876 return -EIO;
1877 }
1878
cadence_nand_read_buf(struct cdns_nand_ctrl * cdns_ctrl,u8 * buf,int len)1879 static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1880 u8 *buf, int len)
1881 {
1882 u8 thread_nr = 0;
1883 u32 sdma_size;
1884 int status;
1885
1886 /* Wait until slave DMA interface is ready to data transfer. */
1887 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1888 if (status)
1889 return status;
1890
1891 if (!cdns_ctrl->caps1->has_dma) {
1892 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1893
1894 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1895
1896 /* read alignment data */
1897 if (data_dma_width == 4)
1898 ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1899 #ifdef CONFIG_64BIT
1900 else
1901 readsq(cdns_ctrl->io.virt, buf, len_in_words);
1902 #endif
1903
1904 if (sdma_size > len) {
1905 int read_bytes = (data_dma_width == 4) ?
1906 len_in_words << 2 : len_in_words << 3;
1907
1908 /* read rest data from slave DMA interface if any */
1909 if (data_dma_width == 4)
1910 ioread32_rep(cdns_ctrl->io.virt,
1911 cdns_ctrl->buf,
1912 sdma_size / 4 - len_in_words);
1913 #ifdef CONFIG_64BIT
1914 else
1915 readsq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1916 sdma_size / 8 - len_in_words);
1917 #endif
1918
1919 /* copy rest of data */
1920 memcpy(buf + read_bytes, cdns_ctrl->buf,
1921 len - read_bytes);
1922 }
1923 return 0;
1924 }
1925
1926 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1927 status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1928 cdns_ctrl->io.dma,
1929 len, DMA_FROM_DEVICE);
1930 if (status == 0)
1931 return 0;
1932
1933 dev_warn(cdns_ctrl->dev,
1934 "Slave DMA transfer failed. Try again using bounce buffer.");
1935 }
1936
1937 /* If DMA transfer is not possible or failed then use bounce buffer. */
1938 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1939 cdns_ctrl->io.dma,
1940 sdma_size, DMA_FROM_DEVICE);
1941
1942 if (status) {
1943 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1944 return status;
1945 }
1946
1947 memcpy(buf, cdns_ctrl->buf, len);
1948
1949 return 0;
1950 }
1951
cadence_nand_write_buf(struct cdns_nand_ctrl * cdns_ctrl,const u8 * buf,int len)1952 static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1953 const u8 *buf, int len)
1954 {
1955 u8 thread_nr = 0;
1956 u32 sdma_size;
1957 int status;
1958
1959 /* Wait until slave DMA interface is ready to data transfer. */
1960 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1961 if (status)
1962 return status;
1963
1964 if (!cdns_ctrl->caps1->has_dma) {
1965 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1966
1967 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1968
1969 if (data_dma_width == 4)
1970 iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1971 #ifdef CONFIG_64BIT
1972 else
1973 writesq(cdns_ctrl->io.virt, buf, len_in_words);
1974 #endif
1975
1976 if (sdma_size > len) {
1977 int written_bytes = (data_dma_width == 4) ?
1978 len_in_words << 2 : len_in_words << 3;
1979
1980 /* copy rest of data */
1981 memcpy(cdns_ctrl->buf, buf + written_bytes,
1982 len - written_bytes);
1983
1984 /* write all expected by nand controller data */
1985 if (data_dma_width == 4)
1986 iowrite32_rep(cdns_ctrl->io.virt,
1987 cdns_ctrl->buf,
1988 sdma_size / 4 - len_in_words);
1989 #ifdef CONFIG_64BIT
1990 else
1991 writesq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1992 sdma_size / 8 - len_in_words);
1993 #endif
1994 }
1995
1996 return 0;
1997 }
1998
1999 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
2000 status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
2001 cdns_ctrl->io.dma,
2002 len, DMA_TO_DEVICE);
2003 if (status == 0)
2004 return 0;
2005
2006 dev_warn(cdns_ctrl->dev,
2007 "Slave DMA transfer failed. Try again using bounce buffer.");
2008 }
2009
2010 /* If DMA transfer is not possible or failed then use bounce buffer. */
2011 memcpy(cdns_ctrl->buf, buf, len);
2012
2013 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
2014 cdns_ctrl->io.dma,
2015 sdma_size, DMA_TO_DEVICE);
2016
2017 if (status)
2018 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
2019
2020 return status;
2021 }
2022
cadence_nand_force_byte_access(struct nand_chip * chip,bool force_8bit)2023 static int cadence_nand_force_byte_access(struct nand_chip *chip,
2024 bool force_8bit)
2025 {
2026 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2027
2028 /*
2029 * Callers of this function do not verify if the NAND is using a 16-bit
2030 * an 8-bit bus for normal operations, so we need to take care of that
2031 * here by leaving the configuration unchanged if the NAND does not have
2032 * the NAND_BUSWIDTH_16 flag set.
2033 */
2034 if (!(chip->options & NAND_BUSWIDTH_16))
2035 return 0;
2036
2037 return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
2038 }
2039
cadence_nand_cmd_opcode(struct nand_chip * chip,const struct nand_subop * subop)2040 static int cadence_nand_cmd_opcode(struct nand_chip *chip,
2041 const struct nand_subop *subop)
2042 {
2043 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2044 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2045 const struct nand_op_instr *instr;
2046 unsigned int op_id = 0;
2047 u64 mini_ctrl_cmd = 0;
2048 int ret;
2049
2050 instr = &subop->instrs[op_id];
2051
2052 if (instr->delay_ns > 0)
2053 mini_ctrl_cmd |= GCMD_LAY_TWB;
2054
2055 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2056 GCMD_LAY_INSTR_CMD);
2057 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2058 instr->ctx.cmd.opcode);
2059
2060 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2061 cdns_chip->cs[chip->cur_cs],
2062 mini_ctrl_cmd);
2063 if (ret)
2064 dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2065 instr->ctx.cmd.opcode);
2066
2067 return ret;
2068 }
2069
cadence_nand_cmd_address(struct nand_chip * chip,const struct nand_subop * subop)2070 static int cadence_nand_cmd_address(struct nand_chip *chip,
2071 const struct nand_subop *subop)
2072 {
2073 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2074 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2075 const struct nand_op_instr *instr;
2076 unsigned int op_id = 0;
2077 u64 mini_ctrl_cmd = 0;
2078 unsigned int offset, naddrs;
2079 u64 address = 0;
2080 const u8 *addrs;
2081 int ret;
2082 int i;
2083
2084 instr = &subop->instrs[op_id];
2085
2086 if (instr->delay_ns > 0)
2087 mini_ctrl_cmd |= GCMD_LAY_TWB;
2088
2089 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2090 GCMD_LAY_INSTR_ADDR);
2091
2092 offset = nand_subop_get_addr_start_off(subop, op_id);
2093 naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2094 addrs = &instr->ctx.addr.addrs[offset];
2095
2096 for (i = 0; i < naddrs; i++)
2097 address |= (u64)addrs[i] << (8 * i);
2098
2099 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2100 address);
2101 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2102 naddrs - 1);
2103
2104 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2105 cdns_chip->cs[chip->cur_cs],
2106 mini_ctrl_cmd);
2107 if (ret)
2108 dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2109
2110 return ret;
2111 }
2112
cadence_nand_cmd_erase(struct nand_chip * chip,const struct nand_subop * subop)2113 static int cadence_nand_cmd_erase(struct nand_chip *chip,
2114 const struct nand_subop *subop)
2115 {
2116 unsigned int op_id;
2117
2118 if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2119 int i;
2120 const struct nand_op_instr *instr = NULL;
2121 unsigned int offset, naddrs;
2122 const u8 *addrs;
2123 u32 page = 0;
2124
2125 instr = &subop->instrs[1];
2126 offset = nand_subop_get_addr_start_off(subop, 1);
2127 naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2128 addrs = &instr->ctx.addr.addrs[offset];
2129
2130 for (i = 0; i < naddrs; i++)
2131 page |= (u32)addrs[i] << (8 * i);
2132
2133 return cadence_nand_erase(chip, page);
2134 }
2135
2136 /*
2137 * If it is not an erase operation then handle operation
2138 * by calling exec_op function.
2139 */
2140 for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2141 int ret;
2142 const struct nand_operation nand_op = {
2143 .cs = chip->cur_cs,
2144 .instrs = &subop->instrs[op_id],
2145 .ninstrs = 1};
2146 ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2147 if (ret)
2148 return ret;
2149 }
2150
2151 return 0;
2152 }
2153
cadence_nand_cmd_data(struct nand_chip * chip,const struct nand_subop * subop)2154 static int cadence_nand_cmd_data(struct nand_chip *chip,
2155 const struct nand_subop *subop)
2156 {
2157 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2158 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2159 const struct nand_op_instr *instr;
2160 unsigned int offset, op_id = 0;
2161 u64 mini_ctrl_cmd = 0;
2162 int len = 0;
2163 int ret;
2164
2165 instr = &subop->instrs[op_id];
2166
2167 if (instr->delay_ns > 0)
2168 mini_ctrl_cmd |= GCMD_LAY_TWB;
2169
2170 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2171 GCMD_LAY_INSTR_DATA);
2172
2173 if (instr->type == NAND_OP_DATA_OUT_INSTR)
2174 mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2175 GCMD_DIR_WRITE);
2176
2177 len = nand_subop_get_data_len(subop, op_id);
2178 offset = nand_subop_get_data_start_off(subop, op_id);
2179 mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2180 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2181 if (instr->ctx.data.force_8bit) {
2182 ret = cadence_nand_force_byte_access(chip, true);
2183 if (ret) {
2184 dev_err(cdns_ctrl->dev,
2185 "cannot change byte access generic data cmd failed\n");
2186 return ret;
2187 }
2188 }
2189
2190 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2191 cdns_chip->cs[chip->cur_cs],
2192 mini_ctrl_cmd);
2193 if (ret) {
2194 dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2195 return ret;
2196 }
2197
2198 if (instr->type == NAND_OP_DATA_IN_INSTR) {
2199 void *buf = instr->ctx.data.buf.in + offset;
2200
2201 ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2202 } else {
2203 const void *buf = instr->ctx.data.buf.out + offset;
2204
2205 ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2206 }
2207
2208 if (ret) {
2209 dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2210 return ret;
2211 }
2212
2213 if (instr->ctx.data.force_8bit) {
2214 ret = cadence_nand_force_byte_access(chip, false);
2215 if (ret) {
2216 dev_err(cdns_ctrl->dev,
2217 "cannot change byte access generic data cmd failed\n");
2218 }
2219 }
2220
2221 return ret;
2222 }
2223
cadence_nand_cmd_waitrdy(struct nand_chip * chip,const struct nand_subop * subop)2224 static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2225 const struct nand_subop *subop)
2226 {
2227 int status;
2228 unsigned int op_id = 0;
2229 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2230 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2231 const struct nand_op_instr *instr = &subop->instrs[op_id];
2232 u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2233
2234 status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2235 timeout_us,
2236 BIT(cdns_chip->cs[chip->cur_cs]),
2237 false);
2238 return status;
2239 }
2240
2241 static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2242 NAND_OP_PARSER_PATTERN(
2243 cadence_nand_cmd_erase,
2244 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2245 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2246 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2247 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2248 NAND_OP_PARSER_PATTERN(
2249 cadence_nand_cmd_opcode,
2250 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2251 NAND_OP_PARSER_PATTERN(
2252 cadence_nand_cmd_address,
2253 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2254 NAND_OP_PARSER_PATTERN(
2255 cadence_nand_cmd_data,
2256 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2257 NAND_OP_PARSER_PATTERN(
2258 cadence_nand_cmd_data,
2259 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2260 NAND_OP_PARSER_PATTERN(
2261 cadence_nand_cmd_waitrdy,
2262 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2263 );
2264
cadence_nand_exec_op(struct nand_chip * chip,const struct nand_operation * op,bool check_only)2265 static int cadence_nand_exec_op(struct nand_chip *chip,
2266 const struct nand_operation *op,
2267 bool check_only)
2268 {
2269 if (!check_only) {
2270 int status = cadence_nand_select_target(chip);
2271
2272 if (status)
2273 return status;
2274 }
2275
2276 return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2277 check_only);
2278 }
2279
cadence_nand_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)2280 static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2281 struct mtd_oob_region *oobregion)
2282 {
2283 struct nand_chip *chip = mtd_to_nand(mtd);
2284 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2285
2286 if (section)
2287 return -ERANGE;
2288
2289 oobregion->offset = cdns_chip->bbm_len;
2290 oobregion->length = cdns_chip->avail_oob_size
2291 - cdns_chip->bbm_len;
2292
2293 return 0;
2294 }
2295
cadence_nand_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)2296 static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2297 struct mtd_oob_region *oobregion)
2298 {
2299 struct nand_chip *chip = mtd_to_nand(mtd);
2300 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2301
2302 if (section)
2303 return -ERANGE;
2304
2305 oobregion->offset = cdns_chip->avail_oob_size;
2306 oobregion->length = chip->ecc.total;
2307
2308 return 0;
2309 }
2310
2311 static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2312 .free = cadence_nand_ooblayout_free,
2313 .ecc = cadence_nand_ooblayout_ecc,
2314 };
2315
calc_cycl(u32 timing,u32 clock)2316 static int calc_cycl(u32 timing, u32 clock)
2317 {
2318 if (timing == 0 || clock == 0)
2319 return 0;
2320
2321 if ((timing % clock) > 0)
2322 return timing / clock;
2323 else
2324 return timing / clock - 1;
2325 }
2326
2327 /* Calculate max data valid window. */
calc_tdvw_max(u32 trp_cnt,u32 clk_period,u32 trhoh_min,u32 board_delay_skew_min,u32 ext_mode)2328 static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2329 u32 board_delay_skew_min, u32 ext_mode)
2330 {
2331 if (ext_mode == 0)
2332 clk_period /= 2;
2333
2334 return (trp_cnt + 1) * clk_period + trhoh_min +
2335 board_delay_skew_min;
2336 }
2337
2338 /* Calculate data valid window. */
calc_tdvw(u32 trp_cnt,u32 clk_period,u32 trhoh_min,u32 trea_max,u32 ext_mode)2339 static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2340 u32 trea_max, u32 ext_mode)
2341 {
2342 if (ext_mode == 0)
2343 clk_period /= 2;
2344
2345 return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2346 }
2347
2348 static int
cadence_nand_setup_interface(struct nand_chip * chip,int chipnr,const struct nand_interface_config * conf)2349 cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2350 const struct nand_interface_config *conf)
2351 {
2352 const struct nand_sdr_timings *sdr;
2353 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2354 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2355 struct cadence_nand_timings *t = &cdns_chip->timings;
2356 u32 reg;
2357 u32 board_delay = cdns_ctrl->board_delay;
2358 u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2359 cdns_ctrl->nf_clk_rate);
2360 u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2361 u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2362 u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2363 u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2364 u32 if_skew = cdns_ctrl->caps1->if_skew;
2365 u32 board_delay_skew_min = board_delay - if_skew;
2366 u32 board_delay_skew_max = board_delay + if_skew;
2367 u32 dqs_sampl_res, phony_dqs_mod;
2368 u32 tdvw, tdvw_min, tdvw_max;
2369 u32 ext_rd_mode, ext_wr_mode;
2370 u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2371 u32 sampling_point;
2372
2373 sdr = nand_get_sdr_timings(conf);
2374 if (IS_ERR(sdr))
2375 return PTR_ERR(sdr);
2376
2377 memset(t, 0, sizeof(*t));
2378 /* Sampling point calculation. */
2379
2380 if (cdns_ctrl->caps2.is_phy_type_dll)
2381 phony_dqs_mod = 2;
2382 else
2383 phony_dqs_mod = 1;
2384
2385 dqs_sampl_res = clk_period / phony_dqs_mod;
2386
2387 tdvw_min = sdr->tREA_max + board_delay_skew_max;
2388 /*
2389 * The idea of those calculation is to get the optimum value
2390 * for tRP and tRH timings. If it is NOT possible to sample data
2391 * with optimal tRP/tRH settings, the parameters will be extended.
2392 * If clk_period is 50ns (the lowest value) this condition is met
2393 * for SDR timing modes 1, 2, 3, 4 and 5.
2394 * If clk_period is 20ns the condition is met only for SDR timing
2395 * mode 5.
2396 */
2397 if (sdr->tRC_min <= clk_period &&
2398 sdr->tRP_min <= (clk_period / 2) &&
2399 sdr->tREH_min <= (clk_period / 2)) {
2400 /* Performance mode. */
2401 ext_rd_mode = 0;
2402 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2403 sdr->tREA_max, ext_rd_mode);
2404 tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2405 board_delay_skew_min,
2406 ext_rd_mode);
2407 /*
2408 * Check if data valid window and sampling point can be found
2409 * and is not on the edge (ie. we have hold margin).
2410 * If not extend the tRP timings.
2411 */
2412 if (tdvw > 0) {
2413 if (tdvw_max <= tdvw_min ||
2414 (tdvw_max % dqs_sampl_res) == 0) {
2415 /*
2416 * No valid sampling point so the RE pulse need
2417 * to be widen widening by half clock cycle.
2418 */
2419 ext_rd_mode = 1;
2420 }
2421 } else {
2422 /*
2423 * There is no valid window
2424 * to be able to sample data the tRP need to be widen.
2425 * Very safe calculations are performed here.
2426 */
2427 trp_cnt = (sdr->tREA_max + board_delay_skew_max
2428 + dqs_sampl_res) / clk_period;
2429 ext_rd_mode = 1;
2430 }
2431
2432 } else {
2433 /* Extended read mode. */
2434 u32 trh;
2435
2436 ext_rd_mode = 1;
2437 trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2438 trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2439 if (sdr->tREH_min >= trh)
2440 trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2441 else
2442 trh_cnt = calc_cycl(trh, clk_period);
2443
2444 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2445 sdr->tREA_max, ext_rd_mode);
2446 /*
2447 * Check if data valid window and sampling point can be found
2448 * or if it is at the edge check if previous is valid
2449 * - if not extend the tRP timings.
2450 */
2451 if (tdvw > 0) {
2452 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2453 sdr->tRHOH_min,
2454 board_delay_skew_min,
2455 ext_rd_mode);
2456
2457 if ((((tdvw_max / dqs_sampl_res)
2458 * dqs_sampl_res) <= tdvw_min) ||
2459 (((tdvw_max % dqs_sampl_res) == 0) &&
2460 (((tdvw_max / dqs_sampl_res - 1)
2461 * dqs_sampl_res) <= tdvw_min))) {
2462 /*
2463 * Data valid window width is lower than
2464 * sampling resolution and do not hit any
2465 * sampling point to be sure the sampling point
2466 * will be found the RE low pulse width will be
2467 * extended by one clock cycle.
2468 */
2469 trp_cnt = trp_cnt + 1;
2470 }
2471 } else {
2472 /*
2473 * There is no valid window to be able to sample data.
2474 * The tRP need to be widen.
2475 * Very safe calculations are performed here.
2476 */
2477 trp_cnt = (sdr->tREA_max + board_delay_skew_max
2478 + dqs_sampl_res) / clk_period;
2479 }
2480 }
2481
2482 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2483 sdr->tRHOH_min,
2484 board_delay_skew_min, ext_rd_mode);
2485
2486 if (sdr->tWC_min <= clk_period &&
2487 (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2488 (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2489 ext_wr_mode = 0;
2490 } else {
2491 u32 twh;
2492
2493 ext_wr_mode = 1;
2494 twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2495 if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2496 twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2497 clk_period);
2498
2499 twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2500 if (sdr->tWH_min >= twh)
2501 twh = sdr->tWH_min;
2502
2503 twh_cnt = calc_cycl(twh + if_skew, clk_period);
2504 }
2505
2506 reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2507 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2508 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2509 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2510 t->async_toggle_timings = reg;
2511 dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2512
2513 tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2514 tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2515 twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2516 trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2517 reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2518
2519 /*
2520 * If timing exceeds delay field in timing register
2521 * then use maximum value.
2522 */
2523 if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2524 reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2525 else
2526 reg |= TIMINGS0_TCCS;
2527
2528 reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2529 reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2530 t->timings0 = reg;
2531 dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2532
2533 /* The following is related to single signal so skew is not needed. */
2534 trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2535 trhz_cnt = trhz_cnt + 1;
2536 twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2537 /*
2538 * Because of the two stage syncflop the value must be increased by 3
2539 * first value is related with sync, second value is related
2540 * with output if delay.
2541 */
2542 twb_cnt = twb_cnt + 3 + 5;
2543 /*
2544 * The following is related to the we edge of the random data input
2545 * sequence so skew is not needed.
2546 */
2547 tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2548 reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2549 reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2550 reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2551 t->timings1 = reg;
2552 dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2553
2554 tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2555 if (tfeat_cnt < twb_cnt)
2556 tfeat_cnt = twb_cnt;
2557
2558 tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2559 tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2560
2561 reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2562 reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2563 reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2564 t->timings2 = reg;
2565 dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2566
2567 if (cdns_ctrl->caps2.is_phy_type_dll) {
2568 reg = DLL_PHY_CTRL_DLL_RST_N;
2569 if (ext_wr_mode)
2570 reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2571 if (ext_rd_mode)
2572 reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2573
2574 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2575 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2576 t->dll_phy_ctrl = reg;
2577 dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2578 }
2579
2580 /* Sampling point calculation. */
2581 if ((tdvw_max % dqs_sampl_res) > 0)
2582 sampling_point = tdvw_max / dqs_sampl_res;
2583 else
2584 sampling_point = (tdvw_max / dqs_sampl_res - 1);
2585
2586 if (sampling_point * dqs_sampl_res > tdvw_min) {
2587 dll_phy_dqs_timing =
2588 FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2589 dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2590 phony_dqs_timing = sampling_point / phony_dqs_mod;
2591
2592 if ((sampling_point % 2) > 0) {
2593 dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2594 if ((tdvw_max % dqs_sampl_res) == 0)
2595 /*
2596 * Calculation for sampling point at the edge
2597 * of data and being odd number.
2598 */
2599 phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2600 / phony_dqs_mod - 1;
2601
2602 if (!cdns_ctrl->caps2.is_phy_type_dll)
2603 phony_dqs_timing--;
2604
2605 } else {
2606 phony_dqs_timing--;
2607 }
2608 rd_del_sel = phony_dqs_timing + 3;
2609 } else {
2610 dev_warn(cdns_ctrl->dev,
2611 "ERROR : cannot find valid sampling point\n");
2612 }
2613
2614 reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2615 if (cdns_ctrl->caps2.is_phy_type_dll)
2616 reg |= PHY_CTRL_SDR_DQS;
2617 t->phy_ctrl = reg;
2618 dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2619
2620 if (cdns_ctrl->caps2.is_phy_type_dll) {
2621 dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2622 dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2623 dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2624 dll_phy_dqs_timing);
2625 t->phy_dqs_timing = dll_phy_dqs_timing;
2626
2627 reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2628 dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2629 reg);
2630 t->phy_gate_lpbk_ctrl = reg;
2631
2632 dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2633 PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2634 dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2635 }
2636
2637 return 0;
2638 }
2639
cadence_nand_attach_chip(struct nand_chip * chip)2640 static int cadence_nand_attach_chip(struct nand_chip *chip)
2641 {
2642 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2643 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2644 u32 ecc_size;
2645 struct mtd_info *mtd = nand_to_mtd(chip);
2646 int ret;
2647
2648 if (chip->options & NAND_BUSWIDTH_16) {
2649 ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2650 if (ret)
2651 return ret;
2652 }
2653
2654 chip->bbt_options |= NAND_BBT_USE_FLASH;
2655 chip->bbt_options |= NAND_BBT_NO_OOB;
2656 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2657
2658 chip->options |= NAND_NO_SUBPAGE_WRITE;
2659
2660 cdns_chip->bbm_offs = chip->badblockpos;
2661 cdns_chip->bbm_offs &= ~0x01;
2662 /* this value should be even number */
2663 cdns_chip->bbm_len = 2;
2664
2665 ret = nand_ecc_choose_conf(chip,
2666 &cdns_ctrl->ecc_caps,
2667 mtd->oobsize - cdns_chip->bbm_len);
2668 if (ret) {
2669 dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2670 return ret;
2671 }
2672
2673 dev_dbg(cdns_ctrl->dev,
2674 "chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2675 chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2676
2677 /* Error correction configuration. */
2678 cdns_chip->sector_size = chip->ecc.size;
2679 cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2680 ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2681
2682 cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2683
2684 if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2685 cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2686
2687 if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2688 > mtd->oobsize)
2689 cdns_chip->avail_oob_size -= 4;
2690
2691 ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2692 if (ret < 0)
2693 return -EINVAL;
2694
2695 cdns_chip->corr_str_idx = (u8)ret;
2696
2697 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2698 1000000,
2699 CTRL_STATUS_CTRL_BUSY, true))
2700 return -ETIMEDOUT;
2701
2702 cadence_nand_set_ecc_strength(cdns_ctrl,
2703 cdns_chip->corr_str_idx);
2704
2705 cadence_nand_set_erase_detection(cdns_ctrl, true,
2706 chip->ecc.strength);
2707
2708 /* Override the default read operations. */
2709 chip->ecc.read_page = cadence_nand_read_page;
2710 chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2711 chip->ecc.write_page = cadence_nand_write_page;
2712 chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2713 chip->ecc.read_oob = cadence_nand_read_oob;
2714 chip->ecc.write_oob = cadence_nand_write_oob;
2715 chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2716 chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2717
2718 if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2719 cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2720
2721 /* Is 32-bit DMA supported? */
2722 ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2723 if (ret) {
2724 dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2725 return ret;
2726 }
2727
2728 mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2729
2730 return 0;
2731 }
2732
2733 static const struct nand_controller_ops cadence_nand_controller_ops = {
2734 .attach_chip = cadence_nand_attach_chip,
2735 .exec_op = cadence_nand_exec_op,
2736 .setup_interface = cadence_nand_setup_interface,
2737 };
2738
cadence_nand_chip_init(struct cdns_nand_ctrl * cdns_ctrl,struct device_node * np)2739 static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2740 struct device_node *np)
2741 {
2742 struct cdns_nand_chip *cdns_chip;
2743 struct mtd_info *mtd;
2744 struct nand_chip *chip;
2745 int nsels, ret, i;
2746 u32 cs;
2747
2748 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2749 if (nsels <= 0) {
2750 dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2751 return -EINVAL;
2752 }
2753
2754 /* Allocate the nand chip structure. */
2755 cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2756 (nsels * sizeof(u8)),
2757 GFP_KERNEL);
2758 if (!cdns_chip) {
2759 dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2760 return -ENOMEM;
2761 }
2762
2763 cdns_chip->nsels = nsels;
2764
2765 for (i = 0; i < nsels; i++) {
2766 /* Retrieve CS id. */
2767 ret = of_property_read_u32_index(np, "reg", i, &cs);
2768 if (ret) {
2769 dev_err(cdns_ctrl->dev,
2770 "could not retrieve reg property: %d\n",
2771 ret);
2772 return ret;
2773 }
2774
2775 if (cs >= cdns_ctrl->caps2.max_banks) {
2776 dev_err(cdns_ctrl->dev,
2777 "invalid reg value: %u (max CS = %d)\n",
2778 cs, cdns_ctrl->caps2.max_banks);
2779 return -EINVAL;
2780 }
2781
2782 if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2783 dev_err(cdns_ctrl->dev,
2784 "CS %d already assigned\n", cs);
2785 return -EINVAL;
2786 }
2787
2788 cdns_chip->cs[i] = cs;
2789 }
2790
2791 chip = &cdns_chip->chip;
2792 chip->controller = &cdns_ctrl->controller;
2793 nand_set_flash_node(chip, np);
2794
2795 mtd = nand_to_mtd(chip);
2796 mtd->dev.parent = cdns_ctrl->dev;
2797
2798 /*
2799 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2800 * in the DT node, this entry will be overwritten in nand_scan_ident().
2801 */
2802 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2803
2804 ret = nand_scan(chip, cdns_chip->nsels);
2805 if (ret) {
2806 dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2807 return ret;
2808 }
2809
2810 ret = mtd_device_register(mtd, NULL, 0);
2811 if (ret) {
2812 dev_err(cdns_ctrl->dev,
2813 "failed to register mtd device: %d\n", ret);
2814 nand_cleanup(chip);
2815 return ret;
2816 }
2817
2818 list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2819
2820 return 0;
2821 }
2822
cadence_nand_chips_cleanup(struct cdns_nand_ctrl * cdns_ctrl)2823 static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2824 {
2825 struct cdns_nand_chip *entry, *temp;
2826 struct nand_chip *chip;
2827 int ret;
2828
2829 list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2830 chip = &entry->chip;
2831 ret = mtd_device_unregister(nand_to_mtd(chip));
2832 WARN_ON(ret);
2833 nand_cleanup(chip);
2834 list_del(&entry->node);
2835 }
2836 }
2837
cadence_nand_chips_init(struct cdns_nand_ctrl * cdns_ctrl)2838 static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2839 {
2840 struct device_node *np = cdns_ctrl->dev->of_node;
2841 int max_cs = cdns_ctrl->caps2.max_banks;
2842 int nchips, ret;
2843
2844 nchips = of_get_child_count(np);
2845
2846 if (nchips > max_cs) {
2847 dev_err(cdns_ctrl->dev,
2848 "too many NAND chips: %d (max = %d CS)\n",
2849 nchips, max_cs);
2850 return -EINVAL;
2851 }
2852
2853 for_each_child_of_node_scoped(np, nand_np) {
2854 ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2855 if (ret) {
2856 cadence_nand_chips_cleanup(cdns_ctrl);
2857 return ret;
2858 }
2859 }
2860
2861 return 0;
2862 }
2863
2864 static void
cadence_nand_irq_cleanup(int irqnum,struct cdns_nand_ctrl * cdns_ctrl)2865 cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2866 {
2867 /* Disable interrupts. */
2868 writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2869 }
2870
cadence_nand_init(struct cdns_nand_ctrl * cdns_ctrl)2871 static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2872 {
2873 dma_cap_mask_t mask;
2874 struct dma_device *dma_dev = cdns_ctrl->dmac->device;
2875 int ret;
2876
2877 cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2878 sizeof(*cdns_ctrl->cdma_desc),
2879 &cdns_ctrl->dma_cdma_desc,
2880 GFP_KERNEL);
2881 if (!cdns_ctrl->dma_cdma_desc)
2882 return -ENOMEM;
2883
2884 cdns_ctrl->buf_size = SZ_16K;
2885 cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2886 if (!cdns_ctrl->buf) {
2887 ret = -ENOMEM;
2888 goto free_buf_desc;
2889 }
2890
2891 if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2892 IRQF_SHARED, "cadence-nand-controller",
2893 cdns_ctrl)) {
2894 dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2895 ret = -ENODEV;
2896 goto free_buf;
2897 }
2898
2899 spin_lock_init(&cdns_ctrl->irq_lock);
2900 init_completion(&cdns_ctrl->complete);
2901
2902 ret = cadence_nand_hw_init(cdns_ctrl);
2903 if (ret)
2904 goto disable_irq;
2905
2906 dma_cap_zero(mask);
2907 dma_cap_set(DMA_MEMCPY, mask);
2908
2909 if (cdns_ctrl->caps1->has_dma) {
2910 cdns_ctrl->dmac = dma_request_chan_by_mask(&mask);
2911 if (IS_ERR(cdns_ctrl->dmac)) {
2912 ret = dev_err_probe(cdns_ctrl->dev, PTR_ERR(cdns_ctrl->dmac),
2913 "%d: Failed to get a DMA channel\n", ret);
2914 goto disable_irq;
2915 }
2916 }
2917
2918 cdns_ctrl->io.iova_dma = dma_map_resource(dma_dev->dev, cdns_ctrl->io.dma,
2919 cdns_ctrl->io.size,
2920 DMA_BIDIRECTIONAL, 0);
2921
2922 ret = dma_mapping_error(dma_dev->dev, cdns_ctrl->io.iova_dma);
2923 if (ret) {
2924 dev_err(cdns_ctrl->dev, "Failed to map I/O resource to DMA\n");
2925 goto dma_release_chnl;
2926 }
2927
2928 nand_controller_init(&cdns_ctrl->controller);
2929 INIT_LIST_HEAD(&cdns_ctrl->chips);
2930
2931 cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2932 cdns_ctrl->curr_corr_str_idx = 0xFF;
2933
2934 ret = cadence_nand_chips_init(cdns_ctrl);
2935 if (ret) {
2936 dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2937 ret);
2938 goto unmap_dma_resource;
2939 }
2940
2941 kfree(cdns_ctrl->buf);
2942 cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2943 if (!cdns_ctrl->buf) {
2944 ret = -ENOMEM;
2945 goto unmap_dma_resource;
2946 }
2947
2948 return 0;
2949
2950 unmap_dma_resource:
2951 dma_unmap_resource(dma_dev->dev, cdns_ctrl->io.iova_dma,
2952 cdns_ctrl->io.size, DMA_BIDIRECTIONAL, 0);
2953
2954 dma_release_chnl:
2955 if (cdns_ctrl->dmac)
2956 dma_release_channel(cdns_ctrl->dmac);
2957
2958 disable_irq:
2959 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2960
2961 free_buf:
2962 kfree(cdns_ctrl->buf);
2963
2964 free_buf_desc:
2965 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2966 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2967
2968 return ret;
2969 }
2970
2971 /* Driver exit point. */
cadence_nand_remove(struct cdns_nand_ctrl * cdns_ctrl)2972 static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2973 {
2974 cadence_nand_chips_cleanup(cdns_ctrl);
2975 if (cdns_ctrl->dmac)
2976 dma_unmap_resource(cdns_ctrl->dmac->device->dev,
2977 cdns_ctrl->io.iova_dma, cdns_ctrl->io.size,
2978 DMA_BIDIRECTIONAL, 0);
2979 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2980 kfree(cdns_ctrl->buf);
2981 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2982 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2983
2984 if (cdns_ctrl->dmac)
2985 dma_release_channel(cdns_ctrl->dmac);
2986 }
2987
2988 struct cadence_nand_dt {
2989 struct cdns_nand_ctrl cdns_ctrl;
2990 struct clk *clk;
2991 };
2992
2993 static const struct cadence_nand_dt_devdata cadence_nand_default = {
2994 .if_skew = 0,
2995 .has_dma = 1,
2996 };
2997
2998 static const struct of_device_id cadence_nand_dt_ids[] = {
2999 {
3000 .compatible = "cdns,hp-nfc",
3001 .data = &cadence_nand_default
3002 }, {}
3003 };
3004
3005 MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
3006
cadence_nand_dt_probe(struct platform_device * ofdev)3007 static int cadence_nand_dt_probe(struct platform_device *ofdev)
3008 {
3009 struct resource *res;
3010 struct cadence_nand_dt *dt;
3011 struct cdns_nand_ctrl *cdns_ctrl;
3012 int ret;
3013 const struct cadence_nand_dt_devdata *devdata;
3014 u32 val;
3015
3016 devdata = device_get_match_data(&ofdev->dev);
3017 if (!devdata) {
3018 pr_err("Failed to find the right device id.\n");
3019 return -ENOMEM;
3020 }
3021
3022 dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
3023 if (!dt)
3024 return -ENOMEM;
3025
3026 cdns_ctrl = &dt->cdns_ctrl;
3027 cdns_ctrl->caps1 = devdata;
3028
3029 cdns_ctrl->dev = &ofdev->dev;
3030 cdns_ctrl->irq = platform_get_irq(ofdev, 0);
3031 if (cdns_ctrl->irq < 0)
3032 return cdns_ctrl->irq;
3033
3034 dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
3035
3036 cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
3037 if (IS_ERR(cdns_ctrl->reg))
3038 return PTR_ERR(cdns_ctrl->reg);
3039
3040 cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
3041 if (IS_ERR(cdns_ctrl->io.virt))
3042 return PTR_ERR(cdns_ctrl->io.virt);
3043
3044 cdns_ctrl->io.dma = res->start;
3045 cdns_ctrl->io.size = resource_size(res);
3046
3047 dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
3048 if (IS_ERR(dt->clk))
3049 return PTR_ERR(dt->clk);
3050
3051 cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
3052
3053 ret = of_property_read_u32(ofdev->dev.of_node,
3054 "cdns,board-delay-ps", &val);
3055 if (ret) {
3056 val = 4830;
3057 dev_info(cdns_ctrl->dev,
3058 "missing cdns,board-delay-ps property, %d was set\n",
3059 val);
3060 }
3061 cdns_ctrl->board_delay = val;
3062
3063 ret = cadence_nand_init(cdns_ctrl);
3064 if (ret)
3065 return ret;
3066
3067 platform_set_drvdata(ofdev, dt);
3068 return 0;
3069 }
3070
cadence_nand_dt_remove(struct platform_device * ofdev)3071 static void cadence_nand_dt_remove(struct platform_device *ofdev)
3072 {
3073 struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3074
3075 cadence_nand_remove(&dt->cdns_ctrl);
3076 }
3077
3078 static struct platform_driver cadence_nand_dt_driver = {
3079 .probe = cadence_nand_dt_probe,
3080 .remove = cadence_nand_dt_remove,
3081 .driver = {
3082 .name = "cadence-nand-controller",
3083 .of_match_table = cadence_nand_dt_ids,
3084 },
3085 };
3086
3087 module_platform_driver(cadence_nand_dt_driver);
3088
3089 MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3090 MODULE_LICENSE("GPL v2");
3091 MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3092
3093