1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
4 * Copyright (c) 2020, Linaro Limited
5 */
6
7 #include <dt-bindings/dma/qcom-gpi.h>
8 #include <linux/bitfield.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/dmaengine.h>
11 #include <linux/module.h>
12 #include <linux/of_dma.h>
13 #include <linux/platform_device.h>
14 #include <linux/dma/qcom-gpi-dma.h>
15 #include <linux/scatterlist.h>
16 #include <linux/slab.h>
17 #include "../dmaengine.h"
18 #include "../virt-dma.h"
19
20 #define TRE_TYPE_DMA 0x10
21 #define TRE_TYPE_IMMEDIATE_DMA 0x11
22 #define TRE_TYPE_GO 0x20
23 #define TRE_TYPE_CONFIG0 0x22
24
25 /* TRE flags */
26 #define TRE_FLAGS_CHAIN BIT(0)
27 #define TRE_FLAGS_IEOB BIT(8)
28 #define TRE_FLAGS_IEOT BIT(9)
29 #define TRE_FLAGS_BEI BIT(10)
30 #define TRE_FLAGS_LINK BIT(11)
31 #define TRE_FLAGS_TYPE GENMASK(23, 16)
32
33 /* SPI CONFIG0 WD0 */
34 #define TRE_SPI_C0_WORD_SZ GENMASK(4, 0)
35 #define TRE_SPI_C0_LOOPBACK BIT(8)
36 #define TRE_SPI_C0_CS BIT(11)
37 #define TRE_SPI_C0_CPHA BIT(12)
38 #define TRE_SPI_C0_CPOL BIT(13)
39 #define TRE_SPI_C0_TX_PACK BIT(24)
40 #define TRE_SPI_C0_RX_PACK BIT(25)
41
42 /* CONFIG0 WD2 */
43 #define TRE_C0_CLK_DIV GENMASK(11, 0)
44 #define TRE_C0_CLK_SRC GENMASK(19, 16)
45
46 /* SPI GO WD0 */
47 #define TRE_SPI_GO_CMD GENMASK(4, 0)
48 #define TRE_SPI_GO_CS GENMASK(10, 8)
49 #define TRE_SPI_GO_FRAG BIT(26)
50
51 /* GO WD2 */
52 #define TRE_RX_LEN GENMASK(23, 0)
53
54 /* I2C Config0 WD0 */
55 #define TRE_I2C_C0_TLOW GENMASK(7, 0)
56 #define TRE_I2C_C0_THIGH GENMASK(15, 8)
57 #define TRE_I2C_C0_TCYL GENMASK(23, 16)
58 #define TRE_I2C_C0_TX_PACK BIT(24)
59 #define TRE_I2C_C0_RX_PACK BIT(25)
60
61 /* I2C GO WD0 */
62 #define TRE_I2C_GO_CMD GENMASK(4, 0)
63 #define TRE_I2C_GO_ADDR GENMASK(14, 8)
64 #define TRE_I2C_GO_STRETCH BIT(26)
65
66 /* DMA TRE */
67 #define TRE_DMA_LEN GENMASK(23, 0)
68 #define TRE_DMA_IMMEDIATE_LEN GENMASK(3, 0)
69
70 /* Register offsets from gpi-top */
71 #define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k)))
72 #define GPII_n_CH_k_CNTXT_0_EL_SIZE GENMASK(31, 24)
73 #define GPII_n_CH_k_CNTXT_0_CHSTATE GENMASK(23, 20)
74 #define GPII_n_CH_k_CNTXT_0_ERIDX GENMASK(18, 14)
75 #define GPII_n_CH_k_CNTXT_0_DIR BIT(3)
76 #define GPII_n_CH_k_CNTXT_0_PROTO GENMASK(2, 0)
77
78 #define GPII_n_CH_k_CNTXT_0(el_size, erindex, dir, chtype_proto) \
79 (FIELD_PREP(GPII_n_CH_k_CNTXT_0_EL_SIZE, el_size) | \
80 FIELD_PREP(GPII_n_CH_k_CNTXT_0_ERIDX, erindex) | \
81 FIELD_PREP(GPII_n_CH_k_CNTXT_0_DIR, dir) | \
82 FIELD_PREP(GPII_n_CH_k_CNTXT_0_PROTO, chtype_proto))
83
84 #define GPI_CHTYPE_DIR_IN (0)
85 #define GPI_CHTYPE_DIR_OUT (1)
86
87 #define GPI_CHTYPE_PROTO_GPI (0x2)
88
89 #define GPII_n_CH_k_DOORBELL_0_OFFS(n, k) (0x22000 + (0x4000 * (n)) + (0x8 * (k)))
90 #define GPII_n_CH_CMD_OFFS(n) (0x23008 + (0x4000 * (n)))
91 #define GPII_n_CH_CMD_OPCODE GENMASK(31, 24)
92 #define GPII_n_CH_CMD_CHID GENMASK(7, 0)
93 #define GPII_n_CH_CMD(opcode, chid) \
94 (FIELD_PREP(GPII_n_CH_CMD_OPCODE, opcode) | \
95 FIELD_PREP(GPII_n_CH_CMD_CHID, chid))
96
97 #define GPII_n_CH_CMD_ALLOCATE (0)
98 #define GPII_n_CH_CMD_START (1)
99 #define GPII_n_CH_CMD_STOP (2)
100 #define GPII_n_CH_CMD_RESET (9)
101 #define GPII_n_CH_CMD_DE_ALLOC (10)
102 #define GPII_n_CH_CMD_UART_SW_STALE (32)
103 #define GPII_n_CH_CMD_UART_RFR_READY (33)
104 #define GPII_n_CH_CMD_UART_RFR_NOT_READY (34)
105
106 /* EV Context Array */
107 #define GPII_n_EV_CH_k_CNTXT_0_OFFS(n, k) (0x21000 + (0x4000 * (n)) + (0x80 * (k)))
108 #define GPII_n_EV_k_CNTXT_0_EL_SIZE GENMASK(31, 24)
109 #define GPII_n_EV_k_CNTXT_0_CHSTATE GENMASK(23, 20)
110 #define GPII_n_EV_k_CNTXT_0_INTYPE BIT(16)
111 #define GPII_n_EV_k_CNTXT_0_CHTYPE GENMASK(3, 0)
112
113 #define GPII_n_EV_k_CNTXT_0(el_size, inttype, chtype) \
114 (FIELD_PREP(GPII_n_EV_k_CNTXT_0_EL_SIZE, el_size) | \
115 FIELD_PREP(GPII_n_EV_k_CNTXT_0_INTYPE, inttype) | \
116 FIELD_PREP(GPII_n_EV_k_CNTXT_0_CHTYPE, chtype))
117
118 #define GPI_INTTYPE_IRQ (1)
119 #define GPI_CHTYPE_GPI_EV (0x2)
120
121 enum CNTXT_OFFS {
122 CNTXT_0_CONFIG = 0x0,
123 CNTXT_1_R_LENGTH = 0x4,
124 CNTXT_2_RING_BASE_LSB = 0x8,
125 CNTXT_3_RING_BASE_MSB = 0xC,
126 CNTXT_4_RING_RP_LSB = 0x10,
127 CNTXT_5_RING_RP_MSB = 0x14,
128 CNTXT_6_RING_WP_LSB = 0x18,
129 CNTXT_7_RING_WP_MSB = 0x1C,
130 CNTXT_8_RING_INT_MOD = 0x20,
131 CNTXT_9_RING_INTVEC = 0x24,
132 CNTXT_10_RING_MSI_LSB = 0x28,
133 CNTXT_11_RING_MSI_MSB = 0x2C,
134 CNTXT_12_RING_RP_UPDATE_LSB = 0x30,
135 CNTXT_13_RING_RP_UPDATE_MSB = 0x34,
136 };
137
138 #define GPII_n_EV_CH_k_DOORBELL_0_OFFS(n, k) (0x22100 + (0x4000 * (n)) + (0x8 * (k)))
139 #define GPII_n_EV_CH_CMD_OFFS(n) (0x23010 + (0x4000 * (n)))
140 #define GPII_n_EV_CMD_OPCODE GENMASK(31, 24)
141 #define GPII_n_EV_CMD_CHID GENMASK(7, 0)
142 #define GPII_n_EV_CMD(opcode, chid) \
143 (FIELD_PREP(GPII_n_EV_CMD_OPCODE, opcode) | \
144 FIELD_PREP(GPII_n_EV_CMD_CHID, chid))
145
146 #define GPII_n_EV_CH_CMD_ALLOCATE (0x00)
147 #define GPII_n_EV_CH_CMD_RESET (0x09)
148 #define GPII_n_EV_CH_CMD_DE_ALLOC (0x0A)
149
150 #define GPII_n_CNTXT_TYPE_IRQ_OFFS(n) (0x23080 + (0x4000 * (n)))
151
152 /* mask type register */
153 #define GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) (0x23088 + (0x4000 * (n)))
154 #define GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK GENMASK(6, 0)
155 #define GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL BIT(6)
156 #define GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB BIT(3)
157 #define GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB BIT(2)
158 #define GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL BIT(1)
159 #define GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL BIT(0)
160
161 #define GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(n) (0x23090 + (0x4000 * (n)))
162 #define GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) (0x23094 + (0x4000 * (n)))
163
164 /* Mask channel control interrupt register */
165 #define GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(n) (0x23098 + (0x4000 * (n)))
166 #define GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK GENMASK(1, 0)
167
168 /* Mask event control interrupt register */
169 #define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) (0x2309C + (0x4000 * (n)))
170 #define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK BIT(0)
171
172 #define GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(n) (0x230A0 + (0x4000 * (n)))
173 #define GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) (0x230A4 + (0x4000 * (n)))
174
175 /* Mask event interrupt register */
176 #define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) (0x230B8 + (0x4000 * (n)))
177 #define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK BIT(0)
178
179 #define GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) (0x230C0 + (0x4000 * (n)))
180 #define GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) (0x23100 + (0x4000 * (n)))
181 #define GPI_GLOB_IRQ_ERROR_INT_MSK BIT(0)
182
183 /* GPII specific Global - Enable bit register */
184 #define GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(n) (0x23108 + (0x4000 * (n)))
185 #define GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) (0x23110 + (0x4000 * (n)))
186 #define GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(n) (0x23118 + (0x4000 * (n)))
187
188 /* GPII general interrupt - Enable bit register */
189 #define GPII_n_CNTXT_GPII_IRQ_EN_OFFS(n) (0x23120 + (0x4000 * (n)))
190 #define GPII_n_CNTXT_GPII_IRQ_EN_BMSK GENMASK(3, 0)
191
192 #define GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(n) (0x23128 + (0x4000 * (n)))
193
194 /* GPII Interrupt Type register */
195 #define GPII_n_CNTXT_INTSET_OFFS(n) (0x23180 + (0x4000 * (n)))
196 #define GPII_n_CNTXT_INTSET_BMSK BIT(0)
197
198 #define GPII_n_CNTXT_MSI_BASE_LSB_OFFS(n) (0x23188 + (0x4000 * (n)))
199 #define GPII_n_CNTXT_MSI_BASE_MSB_OFFS(n) (0x2318C + (0x4000 * (n)))
200 #define GPII_n_CNTXT_SCRATCH_0_OFFS(n) (0x23400 + (0x4000 * (n)))
201 #define GPII_n_CNTXT_SCRATCH_1_OFFS(n) (0x23404 + (0x4000 * (n)))
202
203 #define GPII_n_ERROR_LOG_OFFS(n) (0x23200 + (0x4000 * (n)))
204
205 /* QOS Registers */
206 #define GPII_n_CH_k_QOS_OFFS(n, k) (0x2005C + (0x4000 * (n)) + (0x80 * (k)))
207
208 /* Scratch registers */
209 #define GPII_n_CH_k_SCRATCH_0_OFFS(n, k) (0x20060 + (0x4000 * (n)) + (0x80 * (k)))
210 #define GPII_n_CH_k_SCRATCH_0_SEID GENMASK(2, 0)
211 #define GPII_n_CH_k_SCRATCH_0_PROTO GENMASK(7, 4)
212 #define GPII_n_CH_k_SCRATCH_0_PAIR GENMASK(20, 16)
213 #define GPII_n_CH_k_SCRATCH_0(pair, proto, seid) \
214 (FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PAIR, pair) | \
215 FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PROTO, proto) | \
216 FIELD_PREP(GPII_n_CH_k_SCRATCH_0_SEID, seid))
217 #define GPII_n_CH_k_SCRATCH_1_OFFS(n, k) (0x20064 + (0x4000 * (n)) + (0x80 * (k)))
218 #define GPII_n_CH_k_SCRATCH_2_OFFS(n, k) (0x20068 + (0x4000 * (n)) + (0x80 * (k)))
219 #define GPII_n_CH_k_SCRATCH_3_OFFS(n, k) (0x2006C + (0x4000 * (n)) + (0x80 * (k)))
220
221 struct __packed gpi_tre {
222 u32 dword[4];
223 };
224
225 enum msm_gpi_tce_code {
226 MSM_GPI_TCE_SUCCESS = 1,
227 MSM_GPI_TCE_EOT = 2,
228 MSM_GPI_TCE_EOB = 4,
229 MSM_GPI_TCE_UNEXP_ERR = 16,
230 };
231
232 #define CMD_TIMEOUT_MS (250)
233
234 #define MAX_CHANNELS_PER_GPII (2)
235 #define GPI_TX_CHAN (0)
236 #define GPI_RX_CHAN (1)
237 #define STATE_IGNORE (U32_MAX)
238 #define EV_FACTOR (2)
239 #define REQ_OF_DMA_ARGS (5) /* # of arguments required from client */
240 #define CHAN_TRES 64
241
242 struct __packed xfer_compl_event {
243 u64 ptr;
244 u32 length:24;
245 u8 code;
246 u16 status;
247 u8 type;
248 u8 chid;
249 };
250
251 struct __packed immediate_data_event {
252 u8 data_bytes[8];
253 u8 length:4;
254 u8 resvd:4;
255 u16 tre_index;
256 u8 code;
257 u16 status;
258 u8 type;
259 u8 chid;
260 };
261
262 struct __packed qup_notif_event {
263 u32 status;
264 u32 time;
265 u32 count:24;
266 u8 resvd;
267 u16 resvd1;
268 u8 type;
269 u8 chid;
270 };
271
272 struct __packed gpi_ere {
273 u32 dword[4];
274 };
275
276 enum GPI_EV_TYPE {
277 XFER_COMPLETE_EV_TYPE = 0x22,
278 IMMEDIATE_DATA_EV_TYPE = 0x30,
279 QUP_NOTIF_EV_TYPE = 0x31,
280 STALE_EV_TYPE = 0xFF,
281 };
282
283 union __packed gpi_event {
284 struct __packed xfer_compl_event xfer_compl_event;
285 struct __packed immediate_data_event immediate_data_event;
286 struct __packed qup_notif_event qup_notif_event;
287 struct __packed gpi_ere gpi_ere;
288 };
289
290 enum gpii_irq_settings {
291 DEFAULT_IRQ_SETTINGS,
292 MASK_IEOB_SETTINGS,
293 };
294
295 enum gpi_ev_state {
296 DEFAULT_EV_CH_STATE = 0,
297 EV_STATE_NOT_ALLOCATED = DEFAULT_EV_CH_STATE,
298 EV_STATE_ALLOCATED,
299 MAX_EV_STATES
300 };
301
302 static const char *const gpi_ev_state_str[MAX_EV_STATES] = {
303 [EV_STATE_NOT_ALLOCATED] = "NOT ALLOCATED",
304 [EV_STATE_ALLOCATED] = "ALLOCATED",
305 };
306
307 #define TO_GPI_EV_STATE_STR(_state) (((_state) >= MAX_EV_STATES) ? \
308 "INVALID" : gpi_ev_state_str[(_state)])
309
310 enum gpi_ch_state {
311 DEFAULT_CH_STATE = 0x0,
312 CH_STATE_NOT_ALLOCATED = DEFAULT_CH_STATE,
313 CH_STATE_ALLOCATED = 0x1,
314 CH_STATE_STARTED = 0x2,
315 CH_STATE_STOPPED = 0x3,
316 CH_STATE_STOP_IN_PROC = 0x4,
317 CH_STATE_ERROR = 0xf,
318 MAX_CH_STATES
319 };
320
321 enum gpi_cmd {
322 GPI_CH_CMD_BEGIN,
323 GPI_CH_CMD_ALLOCATE = GPI_CH_CMD_BEGIN,
324 GPI_CH_CMD_START,
325 GPI_CH_CMD_STOP,
326 GPI_CH_CMD_RESET,
327 GPI_CH_CMD_DE_ALLOC,
328 GPI_CH_CMD_UART_SW_STALE,
329 GPI_CH_CMD_UART_RFR_READY,
330 GPI_CH_CMD_UART_RFR_NOT_READY,
331 GPI_CH_CMD_END = GPI_CH_CMD_UART_RFR_NOT_READY,
332 GPI_EV_CMD_BEGIN,
333 GPI_EV_CMD_ALLOCATE = GPI_EV_CMD_BEGIN,
334 GPI_EV_CMD_RESET,
335 GPI_EV_CMD_DEALLOC,
336 GPI_EV_CMD_END = GPI_EV_CMD_DEALLOC,
337 GPI_MAX_CMD,
338 };
339
340 #define IS_CHAN_CMD(_cmd) ((_cmd) <= GPI_CH_CMD_END)
341
342 static const char *const gpi_cmd_str[GPI_MAX_CMD] = {
343 [GPI_CH_CMD_ALLOCATE] = "CH ALLOCATE",
344 [GPI_CH_CMD_START] = "CH START",
345 [GPI_CH_CMD_STOP] = "CH STOP",
346 [GPI_CH_CMD_RESET] = "CH_RESET",
347 [GPI_CH_CMD_DE_ALLOC] = "DE ALLOC",
348 [GPI_CH_CMD_UART_SW_STALE] = "UART SW STALE",
349 [GPI_CH_CMD_UART_RFR_READY] = "UART RFR READY",
350 [GPI_CH_CMD_UART_RFR_NOT_READY] = "UART RFR NOT READY",
351 [GPI_EV_CMD_ALLOCATE] = "EV ALLOCATE",
352 [GPI_EV_CMD_RESET] = "EV RESET",
353 [GPI_EV_CMD_DEALLOC] = "EV DEALLOC",
354 };
355
356 #define TO_GPI_CMD_STR(_cmd) (((_cmd) >= GPI_MAX_CMD) ? "INVALID" : \
357 gpi_cmd_str[(_cmd)])
358
359 /*
360 * @DISABLE_STATE: no register access allowed
361 * @CONFIG_STATE: client has configured the channel
362 * @PREP_HARDWARE: register access is allowed
363 * however, no processing EVENTS
364 * @ACTIVE_STATE: channels are fully operational
365 * @PREPARE_TERMINATE: graceful termination of channels
366 * register access is allowed
367 * @PAUSE_STATE: channels are active, but not processing any events
368 */
369 enum gpi_pm_state {
370 DISABLE_STATE,
371 CONFIG_STATE,
372 PREPARE_HARDWARE,
373 ACTIVE_STATE,
374 PREPARE_TERMINATE,
375 PAUSE_STATE,
376 MAX_PM_STATE
377 };
378
379 #define REG_ACCESS_VALID(_pm_state) ((_pm_state) >= PREPARE_HARDWARE)
380
381 static const char *const gpi_pm_state_str[MAX_PM_STATE] = {
382 [DISABLE_STATE] = "DISABLE",
383 [CONFIG_STATE] = "CONFIG",
384 [PREPARE_HARDWARE] = "PREPARE HARDWARE",
385 [ACTIVE_STATE] = "ACTIVE",
386 [PREPARE_TERMINATE] = "PREPARE TERMINATE",
387 [PAUSE_STATE] = "PAUSE",
388 };
389
390 #define TO_GPI_PM_STR(_state) (((_state) >= MAX_PM_STATE) ? \
391 "INVALID" : gpi_pm_state_str[(_state)])
392
393 static const struct {
394 enum gpi_cmd gpi_cmd;
395 u32 opcode;
396 u32 state;
397 } gpi_cmd_info[GPI_MAX_CMD] = {
398 {
399 GPI_CH_CMD_ALLOCATE,
400 GPII_n_CH_CMD_ALLOCATE,
401 CH_STATE_ALLOCATED,
402 },
403 {
404 GPI_CH_CMD_START,
405 GPII_n_CH_CMD_START,
406 CH_STATE_STARTED,
407 },
408 {
409 GPI_CH_CMD_STOP,
410 GPII_n_CH_CMD_STOP,
411 CH_STATE_STOPPED,
412 },
413 {
414 GPI_CH_CMD_RESET,
415 GPII_n_CH_CMD_RESET,
416 CH_STATE_ALLOCATED,
417 },
418 {
419 GPI_CH_CMD_DE_ALLOC,
420 GPII_n_CH_CMD_DE_ALLOC,
421 CH_STATE_NOT_ALLOCATED,
422 },
423 {
424 GPI_CH_CMD_UART_SW_STALE,
425 GPII_n_CH_CMD_UART_SW_STALE,
426 STATE_IGNORE,
427 },
428 {
429 GPI_CH_CMD_UART_RFR_READY,
430 GPII_n_CH_CMD_UART_RFR_READY,
431 STATE_IGNORE,
432 },
433 {
434 GPI_CH_CMD_UART_RFR_NOT_READY,
435 GPII_n_CH_CMD_UART_RFR_NOT_READY,
436 STATE_IGNORE,
437 },
438 {
439 GPI_EV_CMD_ALLOCATE,
440 GPII_n_EV_CH_CMD_ALLOCATE,
441 EV_STATE_ALLOCATED,
442 },
443 {
444 GPI_EV_CMD_RESET,
445 GPII_n_EV_CH_CMD_RESET,
446 EV_STATE_ALLOCATED,
447 },
448 {
449 GPI_EV_CMD_DEALLOC,
450 GPII_n_EV_CH_CMD_DE_ALLOC,
451 EV_STATE_NOT_ALLOCATED,
452 },
453 };
454
455 struct gpi_ring {
456 void *pre_aligned;
457 size_t alloc_size;
458 phys_addr_t phys_addr;
459 dma_addr_t dma_handle;
460 void *base;
461 void *wp;
462 void *rp;
463 u32 len;
464 u32 el_size;
465 u32 elements;
466 bool configured;
467 };
468
469 struct gpi_dev {
470 struct dma_device dma_device;
471 struct device *dev;
472 struct resource *res;
473 void __iomem *regs;
474 void __iomem *ee_base; /*ee register base address*/
475 u32 max_gpii; /* maximum # of gpii instances available per gpi block */
476 u32 gpii_mask; /* gpii instances available for apps */
477 u32 ev_factor; /* ev ring length factor */
478 struct gpii *gpiis;
479 };
480
481 struct gchan {
482 struct virt_dma_chan vc;
483 u32 chid;
484 u32 seid;
485 u32 protocol;
486 struct gpii *gpii;
487 enum gpi_ch_state ch_state;
488 enum gpi_pm_state pm_state;
489 void __iomem *ch_cntxt_base_reg;
490 void __iomem *ch_cntxt_db_reg;
491 void __iomem *ch_cmd_reg;
492 u32 dir;
493 struct gpi_ring ch_ring;
494 void *config;
495 };
496
497 struct gpii {
498 u32 gpii_id;
499 struct gchan gchan[MAX_CHANNELS_PER_GPII];
500 struct gpi_dev *gpi_dev;
501 int irq;
502 void __iomem *regs; /* points to gpi top */
503 void __iomem *ev_cntxt_base_reg;
504 void __iomem *ev_cntxt_db_reg;
505 void __iomem *ev_ring_rp_lsb_reg;
506 void __iomem *ev_cmd_reg;
507 void __iomem *ieob_clr_reg;
508 struct mutex ctrl_lock;
509 enum gpi_ev_state ev_state;
510 bool configured_irq;
511 enum gpi_pm_state pm_state;
512 rwlock_t pm_lock;
513 struct gpi_ring ev_ring;
514 struct tasklet_struct ev_task; /* event processing tasklet */
515 struct completion cmd_completion;
516 enum gpi_cmd gpi_cmd;
517 u32 cntxt_type_irq_msk;
518 bool ieob_set;
519 };
520
521 #define MAX_TRE 3
522
523 struct gpi_desc {
524 struct virt_dma_desc vd;
525 size_t len;
526 void *db; /* DB register to program */
527 struct gchan *gchan;
528 struct gpi_tre tre[MAX_TRE];
529 u32 num_tre;
530 };
531
532 static const u32 GPII_CHAN_DIR[MAX_CHANNELS_PER_GPII] = {
533 GPI_CHTYPE_DIR_OUT, GPI_CHTYPE_DIR_IN
534 };
535
536 static irqreturn_t gpi_handle_irq(int irq, void *data);
537 static void gpi_ring_recycle_ev_element(struct gpi_ring *ring);
538 static int gpi_ring_add_element(struct gpi_ring *ring, void **wp);
539 static void gpi_process_events(struct gpii *gpii);
540
to_gchan(struct dma_chan * dma_chan)541 static inline struct gchan *to_gchan(struct dma_chan *dma_chan)
542 {
543 return container_of(dma_chan, struct gchan, vc.chan);
544 }
545
to_gpi_desc(struct virt_dma_desc * vd)546 static inline struct gpi_desc *to_gpi_desc(struct virt_dma_desc *vd)
547 {
548 return container_of(vd, struct gpi_desc, vd);
549 }
550
to_physical(const struct gpi_ring * const ring,void * addr)551 static inline phys_addr_t to_physical(const struct gpi_ring *const ring,
552 void *addr)
553 {
554 return ring->phys_addr + (addr - ring->base);
555 }
556
to_virtual(const struct gpi_ring * const ring,phys_addr_t addr)557 static inline void *to_virtual(const struct gpi_ring *const ring, phys_addr_t addr)
558 {
559 return ring->base + (addr - ring->phys_addr);
560 }
561
gpi_read_reg(struct gpii * gpii,void __iomem * addr)562 static inline u32 gpi_read_reg(struct gpii *gpii, void __iomem *addr)
563 {
564 return readl_relaxed(addr);
565 }
566
gpi_write_reg(struct gpii * gpii,void __iomem * addr,u32 val)567 static inline void gpi_write_reg(struct gpii *gpii, void __iomem *addr, u32 val)
568 {
569 writel_relaxed(val, addr);
570 }
571
572 static __always_inline void
gpi_update_reg(struct gpii * gpii,u32 offset,u32 mask,u32 val)573 gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val)
574 {
575 void __iomem *addr = gpii->regs + offset;
576 u32 tmp = gpi_read_reg(gpii, addr);
577
578 tmp &= ~mask;
579 tmp |= u32_encode_bits(val, mask);
580
581 gpi_write_reg(gpii, addr, tmp);
582 }
583
gpi_disable_interrupts(struct gpii * gpii)584 static void gpi_disable_interrupts(struct gpii *gpii)
585 {
586 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
587 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, 0);
588 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),
589 GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK, 0);
590 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),
591 GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK, 0);
592 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),
593 GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK, 0);
594 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),
595 GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0);
596 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),
597 GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0);
598 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),
599 GPII_n_CNTXT_INTSET_BMSK, 0);
600
601 gpii->cntxt_type_irq_msk = 0;
602 devm_free_irq(gpii->gpi_dev->dev, gpii->irq, gpii);
603 gpii->configured_irq = false;
604 }
605
606 /* configure and enable interrupts */
gpi_config_interrupts(struct gpii * gpii,enum gpii_irq_settings settings,bool mask)607 static int gpi_config_interrupts(struct gpii *gpii, enum gpii_irq_settings settings, bool mask)
608 {
609 const u32 enable = (GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL |
610 GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB |
611 GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB |
612 GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL |
613 GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL);
614 int ret;
615
616 if (!gpii->configured_irq) {
617 ret = devm_request_irq(gpii->gpi_dev->dev, gpii->irq,
618 gpi_handle_irq, IRQF_TRIGGER_HIGH,
619 "gpi-dma", gpii);
620 if (ret < 0) {
621 dev_err(gpii->gpi_dev->dev, "error request irq:%d ret:%d\n",
622 gpii->irq, ret);
623 return ret;
624 }
625 }
626
627 if (settings == MASK_IEOB_SETTINGS) {
628 /*
629 * GPII only uses one EV ring per gpii so we can globally
630 * enable/disable IEOB interrupt
631 */
632 if (mask)
633 gpii->cntxt_type_irq_msk |= GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;
634 else
635 gpii->cntxt_type_irq_msk &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB);
636 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
637 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, gpii->cntxt_type_irq_msk);
638 } else {
639 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
640 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, enable);
641 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),
642 GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK,
643 GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK);
644 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),
645 GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK,
646 GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK);
647 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),
648 GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK,
649 GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK);
650 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),
651 GPII_n_CNTXT_GPII_IRQ_EN_BMSK,
652 GPII_n_CNTXT_GPII_IRQ_EN_BMSK);
653 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),
654 GPII_n_CNTXT_GPII_IRQ_EN_BMSK, GPII_n_CNTXT_GPII_IRQ_EN_BMSK);
655 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_LSB_OFFS(gpii->gpii_id), U32_MAX, 0);
656 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_MSB_OFFS(gpii->gpii_id), U32_MAX, 0);
657 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_0_OFFS(gpii->gpii_id), U32_MAX, 0);
658 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_1_OFFS(gpii->gpii_id), U32_MAX, 0);
659 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),
660 GPII_n_CNTXT_INTSET_BMSK, 1);
661 gpi_update_reg(gpii, GPII_n_ERROR_LOG_OFFS(gpii->gpii_id), U32_MAX, 0);
662
663 gpii->cntxt_type_irq_msk = enable;
664 }
665
666 gpii->configured_irq = true;
667 return 0;
668 }
669
670 /* Sends gpii event or channel command */
gpi_send_cmd(struct gpii * gpii,struct gchan * gchan,enum gpi_cmd gpi_cmd)671 static int gpi_send_cmd(struct gpii *gpii, struct gchan *gchan,
672 enum gpi_cmd gpi_cmd)
673 {
674 u32 chid = MAX_CHANNELS_PER_GPII;
675 unsigned long timeout;
676 void __iomem *cmd_reg;
677 u32 cmd;
678
679 if (gpi_cmd >= GPI_MAX_CMD)
680 return -EINVAL;
681 if (IS_CHAN_CMD(gpi_cmd))
682 chid = gchan->chid;
683
684 dev_dbg(gpii->gpi_dev->dev,
685 "sending cmd: %s:%u\n", TO_GPI_CMD_STR(gpi_cmd), chid);
686
687 /* send opcode and wait for completion */
688 reinit_completion(&gpii->cmd_completion);
689 gpii->gpi_cmd = gpi_cmd;
690
691 cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg;
692 cmd = IS_CHAN_CMD(gpi_cmd) ? GPII_n_CH_CMD(gpi_cmd_info[gpi_cmd].opcode, chid) :
693 GPII_n_EV_CMD(gpi_cmd_info[gpi_cmd].opcode, 0);
694 gpi_write_reg(gpii, cmd_reg, cmd);
695 timeout = wait_for_completion_timeout(&gpii->cmd_completion,
696 msecs_to_jiffies(CMD_TIMEOUT_MS));
697 if (!timeout) {
698 dev_err(gpii->gpi_dev->dev, "cmd: %s completion timeout:%u\n",
699 TO_GPI_CMD_STR(gpi_cmd), chid);
700 return -EIO;
701 }
702
703 /* confirm new ch state is correct , if the cmd is a state change cmd */
704 if (gpi_cmd_info[gpi_cmd].state == STATE_IGNORE)
705 return 0;
706
707 if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state)
708 return 0;
709
710 if (!IS_CHAN_CMD(gpi_cmd) && gpii->ev_state == gpi_cmd_info[gpi_cmd].state)
711 return 0;
712
713 return -EIO;
714 }
715
716 /* program transfer ring DB register */
gpi_write_ch_db(struct gchan * gchan,struct gpi_ring * ring,void * wp)717 static inline void gpi_write_ch_db(struct gchan *gchan,
718 struct gpi_ring *ring, void *wp)
719 {
720 struct gpii *gpii = gchan->gpii;
721 phys_addr_t p_wp;
722
723 p_wp = to_physical(ring, wp);
724 gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp);
725 }
726
727 /* program event ring DB register */
gpi_write_ev_db(struct gpii * gpii,struct gpi_ring * ring,void * wp)728 static inline void gpi_write_ev_db(struct gpii *gpii,
729 struct gpi_ring *ring, void *wp)
730 {
731 phys_addr_t p_wp;
732
733 p_wp = ring->phys_addr + (wp - ring->base);
734 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg, p_wp);
735 }
736
737 /* process transfer completion interrupt */
gpi_process_ieob(struct gpii * gpii)738 static void gpi_process_ieob(struct gpii *gpii)
739 {
740 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));
741
742 gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 0);
743 tasklet_hi_schedule(&gpii->ev_task);
744 }
745
746 /* process channel control interrupt */
gpi_process_ch_ctrl_irq(struct gpii * gpii)747 static void gpi_process_ch_ctrl_irq(struct gpii *gpii)
748 {
749 u32 gpii_id = gpii->gpii_id;
750 u32 offset = GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(gpii_id);
751 u32 ch_irq = gpi_read_reg(gpii, gpii->regs + offset);
752 struct gchan *gchan;
753 u32 chid, state;
754
755 /* clear the status */
756 offset = GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(gpii_id);
757 gpi_write_reg(gpii, gpii->regs + offset, (u32)ch_irq);
758
759 for (chid = 0; chid < MAX_CHANNELS_PER_GPII; chid++) {
760 if (!(BIT(chid) & ch_irq))
761 continue;
762
763 gchan = &gpii->gchan[chid];
764 state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg +
765 CNTXT_0_CONFIG);
766 state = FIELD_GET(GPII_n_CH_k_CNTXT_0_CHSTATE, state);
767
768 /*
769 * CH_CMD_DEALLOC cmd always successful. However cmd does
770 * not change hardware status. So overwriting software state
771 * to default state.
772 */
773 if (gpii->gpi_cmd == GPI_CH_CMD_DE_ALLOC)
774 state = DEFAULT_CH_STATE;
775 gchan->ch_state = state;
776
777 /*
778 * Triggering complete all if ch_state is not a stop in process.
779 * Stop in process is a transition state and we will wait for
780 * stop interrupt before notifying.
781 */
782 if (gchan->ch_state != CH_STATE_STOP_IN_PROC)
783 complete_all(&gpii->cmd_completion);
784 }
785 }
786
787 /* processing gpi general error interrupts */
gpi_process_gen_err_irq(struct gpii * gpii)788 static void gpi_process_gen_err_irq(struct gpii *gpii)
789 {
790 u32 gpii_id = gpii->gpii_id;
791 u32 offset = GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(gpii_id);
792 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);
793
794 /* clear the status */
795 dev_dbg(gpii->gpi_dev->dev, "irq_stts:0x%x\n", irq_stts);
796
797 /* Clear the register */
798 offset = GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(gpii_id);
799 gpi_write_reg(gpii, gpii->regs + offset, irq_stts);
800 }
801
802 /* processing gpi level error interrupts */
gpi_process_glob_err_irq(struct gpii * gpii)803 static void gpi_process_glob_err_irq(struct gpii *gpii)
804 {
805 u32 gpii_id = gpii->gpii_id;
806 u32 offset = GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(gpii_id);
807 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);
808
809 offset = GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(gpii_id);
810 gpi_write_reg(gpii, gpii->regs + offset, irq_stts);
811
812 /* only error interrupt should be set */
813 if (irq_stts & ~GPI_GLOB_IRQ_ERROR_INT_MSK) {
814 dev_err(gpii->gpi_dev->dev, "invalid error status:0x%x\n", irq_stts);
815 return;
816 }
817
818 offset = GPII_n_ERROR_LOG_OFFS(gpii_id);
819 gpi_write_reg(gpii, gpii->regs + offset, 0);
820 }
821
822 /* gpii interrupt handler */
gpi_handle_irq(int irq,void * data)823 static irqreturn_t gpi_handle_irq(int irq, void *data)
824 {
825 struct gpii *gpii = data;
826 u32 gpii_id = gpii->gpii_id;
827 u32 type, offset;
828 unsigned long flags;
829
830 read_lock_irqsave(&gpii->pm_lock, flags);
831
832 /*
833 * States are out of sync to receive interrupt
834 * while software state is in DISABLE state, bailing out.
835 */
836 if (!REG_ACCESS_VALID(gpii->pm_state)) {
837 dev_err(gpii->gpi_dev->dev, "receive interrupt while in %s state\n",
838 TO_GPI_PM_STR(gpii->pm_state));
839 goto exit_irq;
840 }
841
842 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);
843 type = gpi_read_reg(gpii, gpii->regs + offset);
844
845 do {
846 /* global gpii error */
847 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB) {
848 gpi_process_glob_err_irq(gpii);
849 type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB);
850 }
851
852 /* transfer complete interrupt */
853 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB) {
854 gpi_process_ieob(gpii);
855 type &= ~GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;
856 }
857
858 /* event control irq */
859 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL) {
860 u32 ev_state;
861 u32 ev_ch_irq;
862
863 dev_dbg(gpii->gpi_dev->dev,
864 "processing EV CTRL interrupt\n");
865 offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(gpii_id);
866 ev_ch_irq = gpi_read_reg(gpii, gpii->regs + offset);
867
868 offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS
869 (gpii_id);
870 gpi_write_reg(gpii, gpii->regs + offset, ev_ch_irq);
871 ev_state = gpi_read_reg(gpii, gpii->ev_cntxt_base_reg +
872 CNTXT_0_CONFIG);
873 ev_state = FIELD_GET(GPII_n_EV_k_CNTXT_0_CHSTATE, ev_state);
874
875 /*
876 * CMD EV_CMD_DEALLOC is always successful. However
877 * cmd does not change hardware status. So overwriting
878 * software state to default state.
879 */
880 if (gpii->gpi_cmd == GPI_EV_CMD_DEALLOC)
881 ev_state = DEFAULT_EV_CH_STATE;
882
883 gpii->ev_state = ev_state;
884 dev_dbg(gpii->gpi_dev->dev, "setting EV state to %s\n",
885 TO_GPI_EV_STATE_STR(gpii->ev_state));
886 complete_all(&gpii->cmd_completion);
887 type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL);
888 }
889
890 /* channel control irq */
891 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL) {
892 dev_dbg(gpii->gpi_dev->dev, "process CH CTRL interrupts\n");
893 gpi_process_ch_ctrl_irq(gpii);
894 type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL);
895 }
896
897 if (type) {
898 dev_err(gpii->gpi_dev->dev, "Unhandled interrupt status:0x%x\n", type);
899 gpi_process_gen_err_irq(gpii);
900 goto exit_irq;
901 }
902
903 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);
904 type = gpi_read_reg(gpii, gpii->regs + offset);
905 } while (type);
906
907 exit_irq:
908 read_unlock_irqrestore(&gpii->pm_lock, flags);
909
910 return IRQ_HANDLED;
911 }
912
913 /* process DMA Immediate completion data events */
gpi_process_imed_data_event(struct gchan * gchan,struct immediate_data_event * imed_event)914 static void gpi_process_imed_data_event(struct gchan *gchan,
915 struct immediate_data_event *imed_event)
916 {
917 struct gpii *gpii = gchan->gpii;
918 struct gpi_ring *ch_ring = &gchan->ch_ring;
919 void *tre = ch_ring->base + (ch_ring->el_size * imed_event->tre_index);
920 struct dmaengine_result result;
921 struct gpi_desc *gpi_desc;
922 struct virt_dma_desc *vd;
923 unsigned long flags;
924 u32 chid;
925
926 /*
927 * If channel not active don't process event
928 */
929 if (gchan->pm_state != ACTIVE_STATE) {
930 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",
931 TO_GPI_PM_STR(gchan->pm_state));
932 return;
933 }
934
935 spin_lock_irqsave(&gchan->vc.lock, flags);
936 vd = vchan_next_desc(&gchan->vc);
937 if (!vd) {
938 struct gpi_ere *gpi_ere;
939 struct gpi_tre *gpi_tre;
940
941 spin_unlock_irqrestore(&gchan->vc.lock, flags);
942 dev_dbg(gpii->gpi_dev->dev, "event without a pending descriptor!\n");
943 gpi_ere = (struct gpi_ere *)imed_event;
944 dev_dbg(gpii->gpi_dev->dev,
945 "Event: %08x %08x %08x %08x\n",
946 gpi_ere->dword[0], gpi_ere->dword[1],
947 gpi_ere->dword[2], gpi_ere->dword[3]);
948 gpi_tre = tre;
949 dev_dbg(gpii->gpi_dev->dev,
950 "Pending TRE: %08x %08x %08x %08x\n",
951 gpi_tre->dword[0], gpi_tre->dword[1],
952 gpi_tre->dword[2], gpi_tre->dword[3]);
953 return;
954 }
955 gpi_desc = to_gpi_desc(vd);
956 spin_unlock_irqrestore(&gchan->vc.lock, flags);
957
958 /*
959 * RP pointed by Event is to last TRE processed,
960 * we need to update ring rp to tre + 1
961 */
962 tre += ch_ring->el_size;
963 if (tre >= (ch_ring->base + ch_ring->len))
964 tre = ch_ring->base;
965 ch_ring->rp = tre;
966
967 /* make sure rp updates are immediately visible to all cores */
968 smp_wmb();
969
970 chid = imed_event->chid;
971 if (imed_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {
972 if (chid == GPI_RX_CHAN)
973 goto gpi_free_desc;
974 else
975 return;
976 }
977
978 if (imed_event->code == MSM_GPI_TCE_UNEXP_ERR)
979 result.result = DMA_TRANS_ABORTED;
980 else
981 result.result = DMA_TRANS_NOERROR;
982 result.residue = gpi_desc->len - imed_event->length;
983
984 dma_cookie_complete(&vd->tx);
985 dmaengine_desc_get_callback_invoke(&vd->tx, &result);
986
987 gpi_free_desc:
988 spin_lock_irqsave(&gchan->vc.lock, flags);
989 list_del(&vd->node);
990 spin_unlock_irqrestore(&gchan->vc.lock, flags);
991 kfree(gpi_desc);
992 gpi_desc = NULL;
993 }
994
995 /* processing transfer completion events */
gpi_process_xfer_compl_event(struct gchan * gchan,struct xfer_compl_event * compl_event)996 static void gpi_process_xfer_compl_event(struct gchan *gchan,
997 struct xfer_compl_event *compl_event)
998 {
999 struct gpii *gpii = gchan->gpii;
1000 struct gpi_ring *ch_ring = &gchan->ch_ring;
1001 void *ev_rp = to_virtual(ch_ring, compl_event->ptr);
1002 struct virt_dma_desc *vd;
1003 struct gpi_desc *gpi_desc;
1004 struct dmaengine_result result;
1005 unsigned long flags;
1006 u32 chid;
1007
1008 /* only process events on active channel */
1009 if (unlikely(gchan->pm_state != ACTIVE_STATE)) {
1010 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",
1011 TO_GPI_PM_STR(gchan->pm_state));
1012 return;
1013 }
1014
1015 spin_lock_irqsave(&gchan->vc.lock, flags);
1016 vd = vchan_next_desc(&gchan->vc);
1017 if (!vd) {
1018 struct gpi_ere *gpi_ere;
1019
1020 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1021 dev_err(gpii->gpi_dev->dev, "Event without a pending descriptor!\n");
1022 gpi_ere = (struct gpi_ere *)compl_event;
1023 dev_err(gpii->gpi_dev->dev,
1024 "Event: %08x %08x %08x %08x\n",
1025 gpi_ere->dword[0], gpi_ere->dword[1],
1026 gpi_ere->dword[2], gpi_ere->dword[3]);
1027 return;
1028 }
1029
1030 gpi_desc = to_gpi_desc(vd);
1031 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1032
1033 /*
1034 * RP pointed by Event is to last TRE processed,
1035 * we need to update ring rp to ev_rp + 1
1036 */
1037 ev_rp += ch_ring->el_size;
1038 if (ev_rp >= (ch_ring->base + ch_ring->len))
1039 ev_rp = ch_ring->base;
1040 ch_ring->rp = ev_rp;
1041
1042 /* update must be visible to other cores */
1043 smp_wmb();
1044
1045 chid = compl_event->chid;
1046 if (compl_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {
1047 if (chid == GPI_RX_CHAN)
1048 goto gpi_free_desc;
1049 else
1050 return;
1051 }
1052
1053 if (compl_event->code == MSM_GPI_TCE_UNEXP_ERR) {
1054 dev_err(gpii->gpi_dev->dev, "Error in Transaction\n");
1055 result.result = DMA_TRANS_ABORTED;
1056 } else {
1057 dev_dbg(gpii->gpi_dev->dev, "Transaction Success\n");
1058 result.result = DMA_TRANS_NOERROR;
1059 }
1060 result.residue = gpi_desc->len - compl_event->length;
1061 dev_dbg(gpii->gpi_dev->dev, "Residue %d\n", result.residue);
1062
1063 dma_cookie_complete(&vd->tx);
1064 dmaengine_desc_get_callback_invoke(&vd->tx, &result);
1065
1066 gpi_free_desc:
1067 spin_lock_irqsave(&gchan->vc.lock, flags);
1068 list_del(&vd->node);
1069 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1070 kfree(gpi_desc);
1071 gpi_desc = NULL;
1072 }
1073
1074 /* process all events */
gpi_process_events(struct gpii * gpii)1075 static void gpi_process_events(struct gpii *gpii)
1076 {
1077 struct gpi_ring *ev_ring = &gpii->ev_ring;
1078 phys_addr_t cntxt_rp;
1079 void *rp;
1080 union gpi_event *gpi_event;
1081 struct gchan *gchan;
1082 u32 chid, type;
1083
1084 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1085 rp = to_virtual(ev_ring, cntxt_rp);
1086
1087 do {
1088 while (rp != ev_ring->rp) {
1089 gpi_event = ev_ring->rp;
1090 chid = gpi_event->xfer_compl_event.chid;
1091 type = gpi_event->xfer_compl_event.type;
1092
1093 dev_dbg(gpii->gpi_dev->dev,
1094 "Event: CHID:%u, type:%x %08x %08x %08x %08x\n",
1095 chid, type, gpi_event->gpi_ere.dword[0],
1096 gpi_event->gpi_ere.dword[1], gpi_event->gpi_ere.dword[2],
1097 gpi_event->gpi_ere.dword[3]);
1098
1099 switch (type) {
1100 case XFER_COMPLETE_EV_TYPE:
1101 gchan = &gpii->gchan[chid];
1102 gpi_process_xfer_compl_event(gchan,
1103 &gpi_event->xfer_compl_event);
1104 break;
1105 case STALE_EV_TYPE:
1106 dev_dbg(gpii->gpi_dev->dev, "stale event, not processing\n");
1107 break;
1108 case IMMEDIATE_DATA_EV_TYPE:
1109 gchan = &gpii->gchan[chid];
1110 gpi_process_imed_data_event(gchan,
1111 &gpi_event->immediate_data_event);
1112 break;
1113 case QUP_NOTIF_EV_TYPE:
1114 dev_dbg(gpii->gpi_dev->dev, "QUP_NOTIF_EV_TYPE\n");
1115 break;
1116 default:
1117 dev_dbg(gpii->gpi_dev->dev,
1118 "not supported event type:0x%x\n", type);
1119 }
1120 gpi_ring_recycle_ev_element(ev_ring);
1121 }
1122 gpi_write_ev_db(gpii, ev_ring, ev_ring->wp);
1123
1124 /* clear pending IEOB events */
1125 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));
1126
1127 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1128 rp = to_virtual(ev_ring, cntxt_rp);
1129
1130 } while (rp != ev_ring->rp);
1131 }
1132
1133 /* processing events using tasklet */
gpi_ev_tasklet(unsigned long data)1134 static void gpi_ev_tasklet(unsigned long data)
1135 {
1136 struct gpii *gpii = (struct gpii *)data;
1137
1138 read_lock(&gpii->pm_lock);
1139 if (!REG_ACCESS_VALID(gpii->pm_state)) {
1140 read_unlock(&gpii->pm_lock);
1141 dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n",
1142 TO_GPI_PM_STR(gpii->pm_state));
1143 return;
1144 }
1145
1146 /* process the events */
1147 gpi_process_events(gpii);
1148
1149 /* enable IEOB, switching back to interrupts */
1150 gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1);
1151 read_unlock(&gpii->pm_lock);
1152 }
1153
1154 /* marks all pending events for the channel as stale */
gpi_mark_stale_events(struct gchan * gchan)1155 static void gpi_mark_stale_events(struct gchan *gchan)
1156 {
1157 struct gpii *gpii = gchan->gpii;
1158 struct gpi_ring *ev_ring = &gpii->ev_ring;
1159 u32 cntxt_rp, local_rp;
1160 void *ev_rp;
1161
1162 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1163
1164 ev_rp = ev_ring->rp;
1165 local_rp = (u32)to_physical(ev_ring, ev_rp);
1166 while (local_rp != cntxt_rp) {
1167 union gpi_event *gpi_event = ev_rp;
1168 u32 chid = gpi_event->xfer_compl_event.chid;
1169
1170 if (chid == gchan->chid)
1171 gpi_event->xfer_compl_event.type = STALE_EV_TYPE;
1172 ev_rp += ev_ring->el_size;
1173 if (ev_rp >= (ev_ring->base + ev_ring->len))
1174 ev_rp = ev_ring->base;
1175 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1176 local_rp = (u32)to_physical(ev_ring, ev_rp);
1177 }
1178 }
1179
1180 /* reset sw state and issue channel reset or de-alloc */
gpi_reset_chan(struct gchan * gchan,enum gpi_cmd gpi_cmd)1181 static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd)
1182 {
1183 struct gpii *gpii = gchan->gpii;
1184 struct gpi_ring *ch_ring = &gchan->ch_ring;
1185 LIST_HEAD(list);
1186 int ret;
1187
1188 ret = gpi_send_cmd(gpii, gchan, gpi_cmd);
1189 if (ret) {
1190 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1191 TO_GPI_CMD_STR(gpi_cmd), ret);
1192 return ret;
1193 }
1194
1195 /* initialize the local ring ptrs */
1196 ch_ring->rp = ch_ring->base;
1197 ch_ring->wp = ch_ring->base;
1198
1199 /* visible to other cores */
1200 smp_wmb();
1201
1202 /* check event ring for any stale events */
1203 write_lock_irq(&gpii->pm_lock);
1204 gpi_mark_stale_events(gchan);
1205
1206 /* remove all async descriptors */
1207 spin_lock(&gchan->vc.lock);
1208 vchan_get_all_descriptors(&gchan->vc, &list);
1209 spin_unlock(&gchan->vc.lock);
1210 write_unlock_irq(&gpii->pm_lock);
1211 vchan_dma_desc_free_list(&gchan->vc, &list);
1212
1213 return 0;
1214 }
1215
gpi_start_chan(struct gchan * gchan)1216 static int gpi_start_chan(struct gchan *gchan)
1217 {
1218 struct gpii *gpii = gchan->gpii;
1219 int ret;
1220
1221 ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_START);
1222 if (ret) {
1223 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1224 TO_GPI_CMD_STR(GPI_CH_CMD_START), ret);
1225 return ret;
1226 }
1227
1228 /* gpii CH is active now */
1229 write_lock_irq(&gpii->pm_lock);
1230 gchan->pm_state = ACTIVE_STATE;
1231 write_unlock_irq(&gpii->pm_lock);
1232
1233 return 0;
1234 }
1235
gpi_stop_chan(struct gchan * gchan)1236 static int gpi_stop_chan(struct gchan *gchan)
1237 {
1238 struct gpii *gpii = gchan->gpii;
1239 int ret;
1240
1241 ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_STOP);
1242 if (ret) {
1243 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1244 TO_GPI_CMD_STR(GPI_CH_CMD_STOP), ret);
1245 return ret;
1246 }
1247
1248 return 0;
1249 }
1250
1251 /* allocate and configure the transfer channel */
gpi_alloc_chan(struct gchan * chan,bool send_alloc_cmd)1252 static int gpi_alloc_chan(struct gchan *chan, bool send_alloc_cmd)
1253 {
1254 struct gpii *gpii = chan->gpii;
1255 struct gpi_ring *ring = &chan->ch_ring;
1256 int ret;
1257 u32 id = gpii->gpii_id;
1258 u32 chid = chan->chid;
1259 u32 pair_chid = !chid;
1260
1261 if (send_alloc_cmd) {
1262 ret = gpi_send_cmd(gpii, chan, GPI_CH_CMD_ALLOCATE);
1263 if (ret) {
1264 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1265 TO_GPI_CMD_STR(GPI_CH_CMD_ALLOCATE), ret);
1266 return ret;
1267 }
1268 }
1269
1270 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_0_CONFIG,
1271 GPII_n_CH_k_CNTXT_0(ring->el_size, 0, chan->dir, GPI_CHTYPE_PROTO_GPI));
1272 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_1_R_LENGTH, ring->len);
1273 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_2_RING_BASE_LSB, ring->phys_addr);
1274 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_3_RING_BASE_MSB,
1275 upper_32_bits(ring->phys_addr));
1276 gpi_write_reg(gpii, chan->ch_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,
1277 upper_32_bits(ring->phys_addr));
1278 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_0_OFFS(id, chid),
1279 GPII_n_CH_k_SCRATCH_0(pair_chid, chan->protocol, chan->seid));
1280 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_1_OFFS(id, chid), 0);
1281 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_2_OFFS(id, chid), 0);
1282 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_3_OFFS(id, chid), 0);
1283 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_QOS_OFFS(id, chid), 1);
1284
1285 /* flush all the writes */
1286 wmb();
1287 return 0;
1288 }
1289
1290 /* allocate and configure event ring */
gpi_alloc_ev_chan(struct gpii * gpii)1291 static int gpi_alloc_ev_chan(struct gpii *gpii)
1292 {
1293 struct gpi_ring *ring = &gpii->ev_ring;
1294 void __iomem *base = gpii->ev_cntxt_base_reg;
1295 int ret;
1296
1297 ret = gpi_send_cmd(gpii, NULL, GPI_EV_CMD_ALLOCATE);
1298 if (ret) {
1299 dev_err(gpii->gpi_dev->dev, "error with cmd:%s ret:%d\n",
1300 TO_GPI_CMD_STR(GPI_EV_CMD_ALLOCATE), ret);
1301 return ret;
1302 }
1303
1304 /* program event context */
1305 gpi_write_reg(gpii, base + CNTXT_0_CONFIG,
1306 GPII_n_EV_k_CNTXT_0(ring->el_size, GPI_INTTYPE_IRQ, GPI_CHTYPE_GPI_EV));
1307 gpi_write_reg(gpii, base + CNTXT_1_R_LENGTH, ring->len);
1308 gpi_write_reg(gpii, base + CNTXT_2_RING_BASE_LSB, lower_32_bits(ring->phys_addr));
1309 gpi_write_reg(gpii, base + CNTXT_3_RING_BASE_MSB, upper_32_bits(ring->phys_addr));
1310 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,
1311 upper_32_bits(ring->phys_addr));
1312 gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0);
1313 gpi_write_reg(gpii, base + CNTXT_10_RING_MSI_LSB, 0);
1314 gpi_write_reg(gpii, base + CNTXT_11_RING_MSI_MSB, 0);
1315 gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0);
1316 gpi_write_reg(gpii, base + CNTXT_12_RING_RP_UPDATE_LSB, 0);
1317 gpi_write_reg(gpii, base + CNTXT_13_RING_RP_UPDATE_MSB, 0);
1318
1319 /* add events to ring */
1320 ring->wp = (ring->base + ring->len - ring->el_size);
1321
1322 /* flush all the writes */
1323 wmb();
1324
1325 /* gpii is active now */
1326 write_lock_irq(&gpii->pm_lock);
1327 gpii->pm_state = ACTIVE_STATE;
1328 write_unlock_irq(&gpii->pm_lock);
1329 gpi_write_ev_db(gpii, ring, ring->wp);
1330
1331 return 0;
1332 }
1333
1334 /* calculate # of ERE/TRE available to queue */
gpi_ring_num_elements_avail(const struct gpi_ring * const ring)1335 static int gpi_ring_num_elements_avail(const struct gpi_ring * const ring)
1336 {
1337 int elements = 0;
1338
1339 if (ring->wp < ring->rp) {
1340 elements = ((ring->rp - ring->wp) / ring->el_size) - 1;
1341 } else {
1342 elements = (ring->rp - ring->base) / ring->el_size;
1343 elements += ((ring->base + ring->len - ring->wp) / ring->el_size) - 1;
1344 }
1345
1346 return elements;
1347 }
1348
gpi_ring_add_element(struct gpi_ring * ring,void ** wp)1349 static int gpi_ring_add_element(struct gpi_ring *ring, void **wp)
1350 {
1351 if (gpi_ring_num_elements_avail(ring) <= 0)
1352 return -ENOMEM;
1353
1354 *wp = ring->wp;
1355 ring->wp += ring->el_size;
1356 if (ring->wp >= (ring->base + ring->len))
1357 ring->wp = ring->base;
1358
1359 /* visible to other cores */
1360 smp_wmb();
1361
1362 return 0;
1363 }
1364
gpi_ring_recycle_ev_element(struct gpi_ring * ring)1365 static void gpi_ring_recycle_ev_element(struct gpi_ring *ring)
1366 {
1367 /* Update the WP */
1368 ring->wp += ring->el_size;
1369 if (ring->wp >= (ring->base + ring->len))
1370 ring->wp = ring->base;
1371
1372 /* Update the RP */
1373 ring->rp += ring->el_size;
1374 if (ring->rp >= (ring->base + ring->len))
1375 ring->rp = ring->base;
1376
1377 /* visible to other cores */
1378 smp_wmb();
1379 }
1380
gpi_free_ring(struct gpi_ring * ring,struct gpii * gpii)1381 static void gpi_free_ring(struct gpi_ring *ring,
1382 struct gpii *gpii)
1383 {
1384 dma_free_coherent(gpii->gpi_dev->dev, ring->alloc_size,
1385 ring->pre_aligned, ring->dma_handle);
1386 memset(ring, 0, sizeof(*ring));
1387 }
1388
1389 /* allocate memory for transfer and event rings */
gpi_alloc_ring(struct gpi_ring * ring,u32 elements,u32 el_size,struct gpii * gpii)1390 static int gpi_alloc_ring(struct gpi_ring *ring, u32 elements,
1391 u32 el_size, struct gpii *gpii)
1392 {
1393 u64 len = elements * el_size;
1394 int bit;
1395
1396 /* ring len must be power of 2 */
1397 bit = find_last_bit((unsigned long *)&len, 32);
1398 if (((1 << bit) - 1) & len)
1399 bit++;
1400 len = 1 << bit;
1401 ring->alloc_size = (len + (len - 1));
1402 dev_dbg(gpii->gpi_dev->dev,
1403 "#el:%u el_size:%u len:%u actual_len:%llu alloc_size:%zu\n",
1404 elements, el_size, (elements * el_size), len,
1405 ring->alloc_size);
1406
1407 ring->pre_aligned = dma_alloc_coherent(gpii->gpi_dev->dev,
1408 ring->alloc_size,
1409 &ring->dma_handle, GFP_KERNEL);
1410 if (!ring->pre_aligned) {
1411 dev_err(gpii->gpi_dev->dev, "could not alloc size:%zu mem for ring\n",
1412 ring->alloc_size);
1413 return -ENOMEM;
1414 }
1415
1416 /* align the physical mem */
1417 ring->phys_addr = (ring->dma_handle + (len - 1)) & ~(len - 1);
1418 ring->base = ring->pre_aligned + (ring->phys_addr - ring->dma_handle);
1419 ring->rp = ring->base;
1420 ring->wp = ring->base;
1421 ring->len = len;
1422 ring->el_size = el_size;
1423 ring->elements = ring->len / ring->el_size;
1424 memset(ring->base, 0, ring->len);
1425 ring->configured = true;
1426
1427 /* update to other cores */
1428 smp_wmb();
1429
1430 dev_dbg(gpii->gpi_dev->dev,
1431 "phy_pre:%pad phy_alig:%pa len:%u el_size:%u elements:%u\n",
1432 &ring->dma_handle, &ring->phys_addr, ring->len,
1433 ring->el_size, ring->elements);
1434
1435 return 0;
1436 }
1437
1438 /* copy tre into transfer ring */
gpi_queue_xfer(struct gpii * gpii,struct gchan * gchan,struct gpi_tre * gpi_tre,void ** wp)1439 static void gpi_queue_xfer(struct gpii *gpii, struct gchan *gchan,
1440 struct gpi_tre *gpi_tre, void **wp)
1441 {
1442 struct gpi_tre *ch_tre;
1443 int ret;
1444
1445 /* get next tre location we can copy */
1446 ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre);
1447 if (unlikely(ret)) {
1448 dev_err(gpii->gpi_dev->dev, "Error adding ring element to xfer ring\n");
1449 return;
1450 }
1451
1452 /* copy the tre info */
1453 memcpy(ch_tre, gpi_tre, sizeof(*ch_tre));
1454 *wp = ch_tre;
1455 }
1456
1457 /* reset and restart transfer channel */
gpi_terminate_all(struct dma_chan * chan)1458 static int gpi_terminate_all(struct dma_chan *chan)
1459 {
1460 struct gchan *gchan = to_gchan(chan);
1461 struct gpii *gpii = gchan->gpii;
1462 int schid, echid, i;
1463 int ret = 0;
1464
1465 mutex_lock(&gpii->ctrl_lock);
1466
1467 /*
1468 * treat both channels as a group if its protocol is not UART
1469 * STOP, RESET, or START needs to be in lockstep
1470 */
1471 schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0;
1472 echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII;
1473
1474 /* stop the channel */
1475 for (i = schid; i < echid; i++) {
1476 gchan = &gpii->gchan[i];
1477
1478 /* disable ch state so no more TRE processing */
1479 write_lock_irq(&gpii->pm_lock);
1480 gchan->pm_state = PREPARE_TERMINATE;
1481 write_unlock_irq(&gpii->pm_lock);
1482
1483 /* send command to Stop the channel */
1484 ret = gpi_stop_chan(gchan);
1485 }
1486
1487 /* reset the channels (clears any pending tre) */
1488 for (i = schid; i < echid; i++) {
1489 gchan = &gpii->gchan[i];
1490
1491 ret = gpi_reset_chan(gchan, GPI_CH_CMD_RESET);
1492 if (ret) {
1493 dev_err(gpii->gpi_dev->dev, "Error resetting channel ret:%d\n", ret);
1494 goto terminate_exit;
1495 }
1496
1497 /* reprogram channel CNTXT */
1498 ret = gpi_alloc_chan(gchan, false);
1499 if (ret) {
1500 dev_err(gpii->gpi_dev->dev, "Error alloc_channel ret:%d\n", ret);
1501 goto terminate_exit;
1502 }
1503 }
1504
1505 /* restart the channels */
1506 for (i = schid; i < echid; i++) {
1507 gchan = &gpii->gchan[i];
1508
1509 ret = gpi_start_chan(gchan);
1510 if (ret) {
1511 dev_err(gpii->gpi_dev->dev, "Error Starting Channel ret:%d\n", ret);
1512 goto terminate_exit;
1513 }
1514 }
1515
1516 terminate_exit:
1517 mutex_unlock(&gpii->ctrl_lock);
1518 return ret;
1519 }
1520
1521 /* pause dma transfer for all channels */
gpi_pause(struct dma_chan * chan)1522 static int gpi_pause(struct dma_chan *chan)
1523 {
1524 struct gchan *gchan = to_gchan(chan);
1525 struct gpii *gpii = gchan->gpii;
1526 int i, ret;
1527
1528 mutex_lock(&gpii->ctrl_lock);
1529
1530 /*
1531 * pause/resume are per gpii not per channel, so
1532 * client needs to call pause only once
1533 */
1534 if (gpii->pm_state == PAUSE_STATE) {
1535 dev_dbg(gpii->gpi_dev->dev, "channel is already paused\n");
1536 mutex_unlock(&gpii->ctrl_lock);
1537 return 0;
1538 }
1539
1540 /* send stop command to stop the channels */
1541 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1542 ret = gpi_stop_chan(&gpii->gchan[i]);
1543 if (ret) {
1544 mutex_unlock(&gpii->ctrl_lock);
1545 return ret;
1546 }
1547 }
1548
1549 disable_irq(gpii->irq);
1550
1551 /* Wait for threads to complete out */
1552 tasklet_kill(&gpii->ev_task);
1553
1554 write_lock_irq(&gpii->pm_lock);
1555 gpii->pm_state = PAUSE_STATE;
1556 write_unlock_irq(&gpii->pm_lock);
1557 mutex_unlock(&gpii->ctrl_lock);
1558
1559 return 0;
1560 }
1561
1562 /* resume dma transfer */
gpi_resume(struct dma_chan * chan)1563 static int gpi_resume(struct dma_chan *chan)
1564 {
1565 struct gchan *gchan = to_gchan(chan);
1566 struct gpii *gpii = gchan->gpii;
1567 int i, ret;
1568
1569 mutex_lock(&gpii->ctrl_lock);
1570 if (gpii->pm_state == ACTIVE_STATE) {
1571 dev_dbg(gpii->gpi_dev->dev, "channel is already active\n");
1572 mutex_unlock(&gpii->ctrl_lock);
1573 return 0;
1574 }
1575
1576 enable_irq(gpii->irq);
1577
1578 /* send start command to start the channels */
1579 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1580 ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START);
1581 if (ret) {
1582 dev_err(gpii->gpi_dev->dev, "Error starting chan, ret:%d\n", ret);
1583 mutex_unlock(&gpii->ctrl_lock);
1584 return ret;
1585 }
1586 }
1587
1588 write_lock_irq(&gpii->pm_lock);
1589 gpii->pm_state = ACTIVE_STATE;
1590 write_unlock_irq(&gpii->pm_lock);
1591 mutex_unlock(&gpii->ctrl_lock);
1592
1593 return 0;
1594 }
1595
gpi_desc_free(struct virt_dma_desc * vd)1596 static void gpi_desc_free(struct virt_dma_desc *vd)
1597 {
1598 struct gpi_desc *gpi_desc = to_gpi_desc(vd);
1599
1600 kfree(gpi_desc);
1601 gpi_desc = NULL;
1602 }
1603
1604 static int
gpi_peripheral_config(struct dma_chan * chan,struct dma_slave_config * config)1605 gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config)
1606 {
1607 struct gchan *gchan = to_gchan(chan);
1608
1609 if (!config->peripheral_config)
1610 return -EINVAL;
1611
1612 gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT);
1613 if (!gchan->config)
1614 return -ENOMEM;
1615
1616 memcpy(gchan->config, config->peripheral_config, config->peripheral_size);
1617
1618 return 0;
1619 }
1620
gpi_create_i2c_tre(struct gchan * chan,struct gpi_desc * desc,struct scatterlist * sgl,enum dma_transfer_direction direction,unsigned long flags)1621 static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc,
1622 struct scatterlist *sgl, enum dma_transfer_direction direction,
1623 unsigned long flags)
1624 {
1625 struct gpi_i2c_config *i2c = chan->config;
1626 struct device *dev = chan->gpii->gpi_dev->dev;
1627 unsigned int tre_idx = 0;
1628 dma_addr_t address;
1629 struct gpi_tre *tre;
1630 unsigned int i;
1631
1632 /* first create config tre if applicable */
1633 if (i2c->set_config) {
1634 tre = &desc->tre[tre_idx];
1635 tre_idx++;
1636
1637 tre->dword[0] = u32_encode_bits(i2c->low_count, TRE_I2C_C0_TLOW);
1638 tre->dword[0] |= u32_encode_bits(i2c->high_count, TRE_I2C_C0_THIGH);
1639 tre->dword[0] |= u32_encode_bits(i2c->cycle_count, TRE_I2C_C0_TCYL);
1640 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_TX_PACK);
1641 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_RX_PACK);
1642
1643 tre->dword[1] = 0;
1644
1645 tre->dword[2] = u32_encode_bits(i2c->clk_div, TRE_C0_CLK_DIV);
1646
1647 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);
1648 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1649 }
1650
1651 /* create the GO tre for Tx */
1652 if (i2c->op == I2C_WRITE) {
1653 tre = &desc->tre[tre_idx];
1654 tre_idx++;
1655
1656 if (i2c->multi_msg)
1657 tre->dword[0] = u32_encode_bits(I2C_READ, TRE_I2C_GO_CMD);
1658 else
1659 tre->dword[0] = u32_encode_bits(i2c->op, TRE_I2C_GO_CMD);
1660
1661 tre->dword[0] |= u32_encode_bits(i2c->addr, TRE_I2C_GO_ADDR);
1662 tre->dword[0] |= u32_encode_bits(i2c->stretch, TRE_I2C_GO_STRETCH);
1663
1664 tre->dword[1] = 0;
1665 tre->dword[2] = u32_encode_bits(i2c->rx_len, TRE_RX_LEN);
1666
1667 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1668
1669 if (i2c->multi_msg)
1670 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1671 else
1672 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1673 }
1674
1675 if (i2c->op == I2C_READ || i2c->multi_msg == false) {
1676 /* create the DMA TRE */
1677 tre = &desc->tre[tre_idx];
1678 tre_idx++;
1679
1680 address = sg_dma_address(sgl);
1681 tre->dword[0] = lower_32_bits(address);
1682 tre->dword[1] = upper_32_bits(address);
1683
1684 tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN);
1685
1686 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);
1687 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT);
1688
1689 if (!(flags & DMA_PREP_INTERRUPT))
1690 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_BEI);
1691 }
1692
1693 for (i = 0; i < tre_idx; i++)
1694 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],
1695 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);
1696
1697 return tre_idx;
1698 }
1699
gpi_create_spi_tre(struct gchan * chan,struct gpi_desc * desc,struct scatterlist * sgl,enum dma_transfer_direction direction)1700 static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc,
1701 struct scatterlist *sgl, enum dma_transfer_direction direction)
1702 {
1703 struct gpi_spi_config *spi = chan->config;
1704 struct device *dev = chan->gpii->gpi_dev->dev;
1705 unsigned int tre_idx = 0;
1706 dma_addr_t address;
1707 struct gpi_tre *tre;
1708 unsigned int i;
1709 int len;
1710
1711 /* first create config tre if applicable */
1712 if (direction == DMA_MEM_TO_DEV && spi->set_config) {
1713 tre = &desc->tre[tre_idx];
1714 tre_idx++;
1715
1716 tre->dword[0] = u32_encode_bits(spi->word_len, TRE_SPI_C0_WORD_SZ);
1717 tre->dword[0] |= u32_encode_bits(spi->loopback_en, TRE_SPI_C0_LOOPBACK);
1718 tre->dword[0] |= u32_encode_bits(spi->clock_pol_high, TRE_SPI_C0_CPOL);
1719 tre->dword[0] |= u32_encode_bits(spi->data_pol_high, TRE_SPI_C0_CPHA);
1720 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_TX_PACK);
1721 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_RX_PACK);
1722
1723 tre->dword[1] = 0;
1724
1725 tre->dword[2] = u32_encode_bits(spi->clk_div, TRE_C0_CLK_DIV);
1726 tre->dword[2] |= u32_encode_bits(spi->clk_src, TRE_C0_CLK_SRC);
1727
1728 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);
1729 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1730 }
1731
1732 /* create the GO tre for Tx */
1733 if (direction == DMA_MEM_TO_DEV) {
1734 tre = &desc->tre[tre_idx];
1735 tre_idx++;
1736
1737 tre->dword[0] = u32_encode_bits(spi->fragmentation, TRE_SPI_GO_FRAG);
1738 tre->dword[0] |= u32_encode_bits(spi->cs, TRE_SPI_GO_CS);
1739 tre->dword[0] |= u32_encode_bits(spi->cmd, TRE_SPI_GO_CMD);
1740
1741 tre->dword[1] = 0;
1742
1743 tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN);
1744
1745 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1746 if (spi->cmd == SPI_RX) {
1747 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB);
1748 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1749 } else if (spi->cmd == SPI_TX) {
1750 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1751 } else { /* SPI_DUPLEX */
1752 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1753 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1754 }
1755 }
1756
1757 /* create the dma tre */
1758 tre = &desc->tre[tre_idx];
1759 tre_idx++;
1760
1761 address = sg_dma_address(sgl);
1762 len = sg_dma_len(sgl);
1763
1764 /* Support Immediate dma for write transfers for data length up to 8 bytes */
1765 if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) {
1766 /*
1767 * For Immediate dma, data length may not always be length of 8 bytes,
1768 * it can be length less than 8, hence initialize both dword's with 0
1769 */
1770 tre->dword[0] = 0;
1771 tre->dword[1] = 0;
1772 memcpy(&tre->dword[0], sg_virt(sgl), len);
1773
1774 tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN);
1775 tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE);
1776 } else {
1777 tre->dword[0] = lower_32_bits(address);
1778 tre->dword[1] = upper_32_bits(address);
1779
1780 tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN);
1781 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);
1782 }
1783
1784 tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV,
1785 TRE_FLAGS_IEOT);
1786
1787 for (i = 0; i < tre_idx; i++)
1788 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],
1789 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);
1790
1791 return tre_idx;
1792 }
1793
1794 /* copy tre into transfer ring */
1795 static struct dma_async_tx_descriptor *
gpi_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)1796 gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1797 unsigned int sg_len, enum dma_transfer_direction direction,
1798 unsigned long flags, void *context)
1799 {
1800 struct gchan *gchan = to_gchan(chan);
1801 struct gpii *gpii = gchan->gpii;
1802 struct device *dev = gpii->gpi_dev->dev;
1803 struct gpi_ring *ch_ring = &gchan->ch_ring;
1804 struct gpi_desc *gpi_desc;
1805 u32 nr, nr_tre = 0;
1806 u8 set_config;
1807 int i;
1808
1809 gpii->ieob_set = false;
1810 if (!is_slave_direction(direction)) {
1811 dev_err(gpii->gpi_dev->dev, "invalid dma direction: %d\n", direction);
1812 return NULL;
1813 }
1814
1815 if (sg_len > 1) {
1816 dev_err(dev, "Multi sg sent, we support only one atm: %d\n", sg_len);
1817 return NULL;
1818 }
1819
1820 nr_tre = 3;
1821 set_config = *(u32 *)gchan->config;
1822 if (!set_config)
1823 nr_tre = 2;
1824 if (direction == DMA_DEV_TO_MEM) /* rx */
1825 nr_tre = 1;
1826
1827 /* calculate # of elements required & available */
1828 nr = gpi_ring_num_elements_avail(ch_ring);
1829 if (nr < nr_tre) {
1830 dev_err(dev, "not enough space in ring, avail:%u required:%u\n", nr, nr_tre);
1831 return NULL;
1832 }
1833
1834 if (!(flags & DMA_PREP_INTERRUPT) && (nr - nr_tre < 2))
1835 return NULL;
1836
1837 gpi_desc = kzalloc(sizeof(*gpi_desc), GFP_NOWAIT);
1838 if (!gpi_desc)
1839 return NULL;
1840
1841 /* create TREs for xfer */
1842 if (gchan->protocol == QCOM_GPI_SPI) {
1843 i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction);
1844 } else if (gchan->protocol == QCOM_GPI_I2C) {
1845 i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction, flags);
1846 } else {
1847 dev_err(dev, "invalid peripheral: %d\n", gchan->protocol);
1848 kfree(gpi_desc);
1849 return NULL;
1850 }
1851
1852 /* set up the descriptor */
1853 gpi_desc->gchan = gchan;
1854 gpi_desc->len = sg_dma_len(sgl);
1855 gpi_desc->num_tre = i;
1856
1857 return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags);
1858 }
1859
1860 /* rings transfer ring db to being transfer */
gpi_issue_pending(struct dma_chan * chan)1861 static void gpi_issue_pending(struct dma_chan *chan)
1862 {
1863 struct gchan *gchan = to_gchan(chan);
1864 struct gpii *gpii = gchan->gpii;
1865 unsigned long flags, pm_lock_flags;
1866 struct virt_dma_desc *vd = NULL;
1867 struct gpi_desc *gpi_desc;
1868 struct gpi_ring *ch_ring = &gchan->ch_ring;
1869 void *tre, *wp = NULL;
1870 int i;
1871
1872 read_lock_irqsave(&gpii->pm_lock, pm_lock_flags);
1873
1874 /* move all submitted descriptors to issued list */
1875 spin_lock_irqsave(&gchan->vc.lock, flags);
1876 if (vchan_issue_pending(&gchan->vc))
1877 vd = list_last_entry(&gchan->vc.desc_issued,
1878 struct virt_dma_desc, node);
1879 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1880
1881 /* nothing to do list is empty */
1882 if (!vd) {
1883 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);
1884 return;
1885 }
1886
1887 gpi_desc = to_gpi_desc(vd);
1888 for (i = 0; i < gpi_desc->num_tre; i++) {
1889 tre = &gpi_desc->tre[i];
1890 gpi_queue_xfer(gpii, gchan, tre, &wp);
1891 }
1892
1893 gpi_desc->db = ch_ring->wp;
1894 gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db);
1895 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);
1896 }
1897
gpi_ch_init(struct gchan * gchan)1898 static int gpi_ch_init(struct gchan *gchan)
1899 {
1900 struct gpii *gpii = gchan->gpii;
1901 const int ev_factor = gpii->gpi_dev->ev_factor;
1902 u32 elements;
1903 int i = 0, ret = 0;
1904
1905 gchan->pm_state = CONFIG_STATE;
1906
1907 /* check if both channels are configured before continue */
1908 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++)
1909 if (gpii->gchan[i].pm_state != CONFIG_STATE)
1910 goto exit_gpi_init;
1911
1912 /* protocol must be same for both channels */
1913 if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) {
1914 dev_err(gpii->gpi_dev->dev, "protocol did not match protocol %u != %u\n",
1915 gpii->gchan[0].protocol, gpii->gchan[1].protocol);
1916 ret = -EINVAL;
1917 goto exit_gpi_init;
1918 }
1919
1920 /* allocate memory for event ring */
1921 elements = CHAN_TRES << ev_factor;
1922 ret = gpi_alloc_ring(&gpii->ev_ring, elements,
1923 sizeof(union gpi_event), gpii);
1924 if (ret)
1925 goto exit_gpi_init;
1926
1927 /* configure interrupts */
1928 write_lock_irq(&gpii->pm_lock);
1929 gpii->pm_state = PREPARE_HARDWARE;
1930 write_unlock_irq(&gpii->pm_lock);
1931 ret = gpi_config_interrupts(gpii, DEFAULT_IRQ_SETTINGS, 0);
1932 if (ret) {
1933 dev_err(gpii->gpi_dev->dev, "error config. interrupts, ret:%d\n", ret);
1934 goto error_config_int;
1935 }
1936
1937 /* allocate event rings */
1938 ret = gpi_alloc_ev_chan(gpii);
1939 if (ret) {
1940 dev_err(gpii->gpi_dev->dev, "error alloc_ev_chan:%d\n", ret);
1941 goto error_alloc_ev_ring;
1942 }
1943
1944 /* Allocate all channels */
1945 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1946 ret = gpi_alloc_chan(&gpii->gchan[i], true);
1947 if (ret) {
1948 dev_err(gpii->gpi_dev->dev, "Error allocating chan:%d\n", ret);
1949 goto error_alloc_chan;
1950 }
1951 }
1952
1953 /* start channels */
1954 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1955 ret = gpi_start_chan(&gpii->gchan[i]);
1956 if (ret) {
1957 dev_err(gpii->gpi_dev->dev, "Error start chan:%d\n", ret);
1958 goto error_start_chan;
1959 }
1960 }
1961 return ret;
1962
1963 error_start_chan:
1964 for (i = i - 1; i >= 0; i--) {
1965 gpi_stop_chan(&gpii->gchan[i]);
1966 gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);
1967 }
1968 i = 2;
1969 error_alloc_chan:
1970 for (i = i - 1; i >= 0; i--)
1971 gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);
1972 error_alloc_ev_ring:
1973 gpi_disable_interrupts(gpii);
1974 error_config_int:
1975 gpi_free_ring(&gpii->ev_ring, gpii);
1976 exit_gpi_init:
1977 return ret;
1978 }
1979
1980 /* release all channel resources */
gpi_free_chan_resources(struct dma_chan * chan)1981 static void gpi_free_chan_resources(struct dma_chan *chan)
1982 {
1983 struct gchan *gchan = to_gchan(chan);
1984 struct gpii *gpii = gchan->gpii;
1985 enum gpi_pm_state cur_state;
1986 int ret, i;
1987
1988 mutex_lock(&gpii->ctrl_lock);
1989
1990 cur_state = gchan->pm_state;
1991
1992 /* disable ch state so no more TRE processing for this channel */
1993 write_lock_irq(&gpii->pm_lock);
1994 gchan->pm_state = PREPARE_TERMINATE;
1995 write_unlock_irq(&gpii->pm_lock);
1996
1997 /* attempt to do graceful hardware shutdown */
1998 if (cur_state == ACTIVE_STATE) {
1999 gpi_stop_chan(gchan);
2000
2001 ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);
2002 if (ret)
2003 dev_err(gpii->gpi_dev->dev, "error resetting channel:%d\n", ret);
2004
2005 gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);
2006 }
2007
2008 /* free all allocated memory */
2009 gpi_free_ring(&gchan->ch_ring, gpii);
2010 vchan_free_chan_resources(&gchan->vc);
2011 kfree(gchan->config);
2012
2013 write_lock_irq(&gpii->pm_lock);
2014 gchan->pm_state = DISABLE_STATE;
2015 write_unlock_irq(&gpii->pm_lock);
2016
2017 /* if other rings are still active exit */
2018 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++)
2019 if (gpii->gchan[i].ch_ring.configured)
2020 goto exit_free;
2021
2022 /* deallocate EV Ring */
2023 cur_state = gpii->pm_state;
2024 write_lock_irq(&gpii->pm_lock);
2025 gpii->pm_state = PREPARE_TERMINATE;
2026 write_unlock_irq(&gpii->pm_lock);
2027
2028 /* wait for threads to complete out */
2029 tasklet_kill(&gpii->ev_task);
2030
2031 /* send command to de allocate event ring */
2032 if (cur_state == ACTIVE_STATE)
2033 gpi_send_cmd(gpii, NULL, GPI_EV_CMD_DEALLOC);
2034
2035 gpi_free_ring(&gpii->ev_ring, gpii);
2036
2037 /* disable interrupts */
2038 if (cur_state == ACTIVE_STATE)
2039 gpi_disable_interrupts(gpii);
2040
2041 /* set final state to disable */
2042 write_lock_irq(&gpii->pm_lock);
2043 gpii->pm_state = DISABLE_STATE;
2044 write_unlock_irq(&gpii->pm_lock);
2045
2046 exit_free:
2047 mutex_unlock(&gpii->ctrl_lock);
2048 }
2049
2050 /* allocate channel resources */
gpi_alloc_chan_resources(struct dma_chan * chan)2051 static int gpi_alloc_chan_resources(struct dma_chan *chan)
2052 {
2053 struct gchan *gchan = to_gchan(chan);
2054 struct gpii *gpii = gchan->gpii;
2055 int ret;
2056
2057 mutex_lock(&gpii->ctrl_lock);
2058
2059 /* allocate memory for transfer ring */
2060 ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES,
2061 sizeof(struct gpi_tre), gpii);
2062 if (ret)
2063 goto xfer_alloc_err;
2064
2065 ret = gpi_ch_init(gchan);
2066
2067 mutex_unlock(&gpii->ctrl_lock);
2068
2069 return ret;
2070 xfer_alloc_err:
2071 mutex_unlock(&gpii->ctrl_lock);
2072
2073 return ret;
2074 }
2075
gpi_find_avail_gpii(struct gpi_dev * gpi_dev,u32 seid)2076 static int gpi_find_avail_gpii(struct gpi_dev *gpi_dev, u32 seid)
2077 {
2078 struct gchan *tx_chan, *rx_chan;
2079 unsigned int gpii;
2080
2081 /* check if same seid is already configured for another chid */
2082 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {
2083 if (!((1 << gpii) & gpi_dev->gpii_mask))
2084 continue;
2085
2086 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];
2087 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];
2088
2089 if (rx_chan->vc.chan.client_count && rx_chan->seid == seid)
2090 return gpii;
2091 if (tx_chan->vc.chan.client_count && tx_chan->seid == seid)
2092 return gpii;
2093 }
2094
2095 /* no channels configured with same seid, return next avail gpii */
2096 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {
2097 if (!((1 << gpii) & gpi_dev->gpii_mask))
2098 continue;
2099
2100 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];
2101 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];
2102
2103 /* check if gpii is configured */
2104 if (tx_chan->vc.chan.client_count ||
2105 rx_chan->vc.chan.client_count)
2106 continue;
2107
2108 /* found a free gpii */
2109 return gpii;
2110 }
2111
2112 /* no gpii instance available to use */
2113 return -EIO;
2114 }
2115
2116 /* gpi_of_dma_xlate: open client requested channel */
gpi_of_dma_xlate(struct of_phandle_args * args,struct of_dma * of_dma)2117 static struct dma_chan *gpi_of_dma_xlate(struct of_phandle_args *args,
2118 struct of_dma *of_dma)
2119 {
2120 struct gpi_dev *gpi_dev = (struct gpi_dev *)of_dma->of_dma_data;
2121 u32 seid, chid;
2122 int gpii;
2123 struct gchan *gchan;
2124
2125 if (args->args_count < 3) {
2126 dev_err(gpi_dev->dev, "gpii require minimum 2 args, client passed:%d args\n",
2127 args->args_count);
2128 return NULL;
2129 }
2130
2131 chid = args->args[0];
2132 if (chid >= MAX_CHANNELS_PER_GPII) {
2133 dev_err(gpi_dev->dev, "gpii channel:%d not valid\n", chid);
2134 return NULL;
2135 }
2136
2137 seid = args->args[1];
2138
2139 /* find next available gpii to use */
2140 gpii = gpi_find_avail_gpii(gpi_dev, seid);
2141 if (gpii < 0) {
2142 dev_err(gpi_dev->dev, "no available gpii instances\n");
2143 return NULL;
2144 }
2145
2146 gchan = &gpi_dev->gpiis[gpii].gchan[chid];
2147 if (gchan->vc.chan.client_count) {
2148 dev_err(gpi_dev->dev, "gpii:%d chid:%d seid:%d already configured\n",
2149 gpii, chid, gchan->seid);
2150 return NULL;
2151 }
2152
2153 gchan->seid = seid;
2154 gchan->protocol = args->args[2];
2155
2156 return dma_get_slave_channel(&gchan->vc.chan);
2157 }
2158
gpi_probe(struct platform_device * pdev)2159 static int gpi_probe(struct platform_device *pdev)
2160 {
2161 struct gpi_dev *gpi_dev;
2162 unsigned int i;
2163 u32 ee_offset;
2164 int ret;
2165
2166 gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL);
2167 if (!gpi_dev)
2168 return -ENOMEM;
2169
2170 gpi_dev->dev = &pdev->dev;
2171 gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res);
2172 if (IS_ERR(gpi_dev->regs))
2173 return PTR_ERR(gpi_dev->regs);
2174 gpi_dev->ee_base = gpi_dev->regs;
2175
2176 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channels",
2177 &gpi_dev->max_gpii);
2178 if (ret) {
2179 dev_err(gpi_dev->dev, "missing 'max-no-gpii' DT node\n");
2180 return ret;
2181 }
2182
2183 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channel-mask",
2184 &gpi_dev->gpii_mask);
2185 if (ret) {
2186 dev_err(gpi_dev->dev, "missing 'gpii-mask' DT node\n");
2187 return ret;
2188 }
2189
2190 ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev);
2191 gpi_dev->ee_base = gpi_dev->ee_base - ee_offset;
2192
2193 gpi_dev->ev_factor = EV_FACTOR;
2194
2195 ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64));
2196 if (ret) {
2197 dev_err(gpi_dev->dev, "Error setting dma_mask to 64, ret:%d\n", ret);
2198 return ret;
2199 }
2200
2201 gpi_dev->gpiis = devm_kzalloc(gpi_dev->dev, sizeof(*gpi_dev->gpiis) *
2202 gpi_dev->max_gpii, GFP_KERNEL);
2203 if (!gpi_dev->gpiis)
2204 return -ENOMEM;
2205
2206 /* setup all the supported gpii */
2207 INIT_LIST_HEAD(&gpi_dev->dma_device.channels);
2208 for (i = 0; i < gpi_dev->max_gpii; i++) {
2209 struct gpii *gpii = &gpi_dev->gpiis[i];
2210 int chan;
2211
2212 if (!((1 << i) & gpi_dev->gpii_mask))
2213 continue;
2214
2215 /* set up ev cntxt register map */
2216 gpii->ev_cntxt_base_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_CNTXT_0_OFFS(i, 0);
2217 gpii->ev_cntxt_db_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_DOORBELL_0_OFFS(i, 0);
2218 gpii->ev_ring_rp_lsb_reg = gpii->ev_cntxt_base_reg + CNTXT_4_RING_RP_LSB;
2219 gpii->ev_cmd_reg = gpi_dev->ee_base + GPII_n_EV_CH_CMD_OFFS(i);
2220 gpii->ieob_clr_reg = gpi_dev->ee_base + GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(i);
2221
2222 /* set up irq */
2223 ret = platform_get_irq(pdev, i);
2224 if (ret < 0)
2225 return ret;
2226 gpii->irq = ret;
2227
2228 /* set up channel specific register info */
2229 for (chan = 0; chan < MAX_CHANNELS_PER_GPII; chan++) {
2230 struct gchan *gchan = &gpii->gchan[chan];
2231
2232 /* set up ch cntxt register map */
2233 gchan->ch_cntxt_base_reg = gpi_dev->ee_base +
2234 GPII_n_CH_k_CNTXT_0_OFFS(i, chan);
2235 gchan->ch_cntxt_db_reg = gpi_dev->ee_base +
2236 GPII_n_CH_k_DOORBELL_0_OFFS(i, chan);
2237 gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i);
2238
2239 /* vchan setup */
2240 vchan_init(&gchan->vc, &gpi_dev->dma_device);
2241 gchan->vc.desc_free = gpi_desc_free;
2242 gchan->chid = chan;
2243 gchan->gpii = gpii;
2244 gchan->dir = GPII_CHAN_DIR[chan];
2245 }
2246 mutex_init(&gpii->ctrl_lock);
2247 rwlock_init(&gpii->pm_lock);
2248 tasklet_init(&gpii->ev_task, gpi_ev_tasklet,
2249 (unsigned long)gpii);
2250 init_completion(&gpii->cmd_completion);
2251 gpii->gpii_id = i;
2252 gpii->regs = gpi_dev->ee_base;
2253 gpii->gpi_dev = gpi_dev;
2254 }
2255
2256 platform_set_drvdata(pdev, gpi_dev);
2257
2258 /* clear and Set capabilities */
2259 dma_cap_zero(gpi_dev->dma_device.cap_mask);
2260 dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask);
2261
2262 /* configure dmaengine apis */
2263 gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2264 gpi_dev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
2265 gpi_dev->dma_device.src_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;
2266 gpi_dev->dma_device.dst_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;
2267 gpi_dev->dma_device.device_alloc_chan_resources = gpi_alloc_chan_resources;
2268 gpi_dev->dma_device.device_free_chan_resources = gpi_free_chan_resources;
2269 gpi_dev->dma_device.device_tx_status = dma_cookie_status;
2270 gpi_dev->dma_device.device_issue_pending = gpi_issue_pending;
2271 gpi_dev->dma_device.device_prep_slave_sg = gpi_prep_slave_sg;
2272 gpi_dev->dma_device.device_config = gpi_peripheral_config;
2273 gpi_dev->dma_device.device_terminate_all = gpi_terminate_all;
2274 gpi_dev->dma_device.dev = gpi_dev->dev;
2275 gpi_dev->dma_device.device_pause = gpi_pause;
2276 gpi_dev->dma_device.device_resume = gpi_resume;
2277
2278 /* register with dmaengine framework */
2279 ret = dma_async_device_register(&gpi_dev->dma_device);
2280 if (ret) {
2281 dev_err(gpi_dev->dev, "async_device_register failed ret:%d", ret);
2282 return ret;
2283 }
2284
2285 ret = of_dma_controller_register(gpi_dev->dev->of_node,
2286 gpi_of_dma_xlate, gpi_dev);
2287 if (ret) {
2288 dev_err(gpi_dev->dev, "of_dma_controller_reg failed ret:%d", ret);
2289 return ret;
2290 }
2291
2292 return ret;
2293 }
2294
2295 static const struct of_device_id gpi_of_match[] = {
2296 { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },
2297 { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 },
2298 /*
2299 * Do not grow the list for compatible devices. Instead use
2300 * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma
2301 * (for ee_offset = 0x10000).
2302 */
2303 { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },
2304 { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },
2305 { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },
2306 { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },
2307 { .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 },
2308 { },
2309 };
2310 MODULE_DEVICE_TABLE(of, gpi_of_match);
2311
2312 static struct platform_driver gpi_driver = {
2313 .probe = gpi_probe,
2314 .driver = {
2315 .name = KBUILD_MODNAME,
2316 .of_match_table = gpi_of_match,
2317 },
2318 };
2319
gpi_init(void)2320 static int __init gpi_init(void)
2321 {
2322 return platform_driver_register(&gpi_driver);
2323 }
2324 subsys_initcall(gpi_init)
2325
2326 MODULE_DESCRIPTION("QCOM GPI DMA engine driver");
2327 MODULE_LICENSE("GPL v2");
2328