xref: /linux/drivers/hwtracing/coresight/coresight-tmc-etf.c (revision 79790b6818e96c58fe2bffee1b418c16e64e7b80)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright(C) 2016 Linaro Limited. All rights reserved.
4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5  */
6 
7 #include <linux/atomic.h>
8 #include <linux/circ_buf.h>
9 #include <linux/coresight.h>
10 #include <linux/perf_event.h>
11 #include <linux/slab.h>
12 #include "coresight-priv.h"
13 #include "coresight-tmc.h"
14 #include "coresight-etm-perf.h"
15 
16 static int tmc_set_etf_buffer(struct coresight_device *csdev,
17 			      struct perf_output_handle *handle);
18 
__tmc_etb_enable_hw(struct tmc_drvdata * drvdata)19 static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
20 {
21 	int rc = 0;
22 
23 	CS_UNLOCK(drvdata->base);
24 
25 	/* Wait for TMCSReady bit to be set */
26 	rc = tmc_wait_for_tmcready(drvdata);
27 	if (rc) {
28 		dev_err(&drvdata->csdev->dev,
29 			"Failed to enable: TMC not ready\n");
30 		CS_LOCK(drvdata->base);
31 		return rc;
32 	}
33 
34 	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
35 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
36 		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
37 		       TMC_FFCR_TRIGON_TRIGIN,
38 		       drvdata->base + TMC_FFCR);
39 
40 	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
41 	tmc_enable_hw(drvdata);
42 
43 	CS_LOCK(drvdata->base);
44 	return rc;
45 }
46 
tmc_etb_enable_hw(struct tmc_drvdata * drvdata)47 static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
48 {
49 	int rc = coresight_claim_device(drvdata->csdev);
50 
51 	if (rc)
52 		return rc;
53 
54 	rc = __tmc_etb_enable_hw(drvdata);
55 	if (rc)
56 		coresight_disclaim_device(drvdata->csdev);
57 	return rc;
58 }
59 
tmc_etb_dump_hw(struct tmc_drvdata * drvdata)60 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
61 {
62 	char *bufp;
63 	u32 read_data, lost;
64 
65 	/* Check if the buffer wrapped around. */
66 	lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
67 	bufp = drvdata->buf;
68 	drvdata->len = 0;
69 	while (1) {
70 		read_data = readl_relaxed(drvdata->base + TMC_RRD);
71 		if (read_data == 0xFFFFFFFF)
72 			break;
73 		memcpy(bufp, &read_data, 4);
74 		bufp += 4;
75 		drvdata->len += 4;
76 	}
77 
78 	if (lost)
79 		coresight_insert_barrier_packet(drvdata->buf);
80 	return;
81 }
82 
__tmc_etb_disable_hw(struct tmc_drvdata * drvdata)83 static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
84 {
85 	CS_UNLOCK(drvdata->base);
86 
87 	tmc_flush_and_stop(drvdata);
88 	/*
89 	 * When operating in sysFS mode the content of the buffer needs to be
90 	 * read before the TMC is disabled.
91 	 */
92 	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
93 		tmc_etb_dump_hw(drvdata);
94 	tmc_disable_hw(drvdata);
95 
96 	CS_LOCK(drvdata->base);
97 }
98 
tmc_etb_disable_hw(struct tmc_drvdata * drvdata)99 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
100 {
101 	__tmc_etb_disable_hw(drvdata);
102 	coresight_disclaim_device(drvdata->csdev);
103 }
104 
__tmc_etf_enable_hw(struct tmc_drvdata * drvdata)105 static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
106 {
107 	int rc = 0;
108 
109 	CS_UNLOCK(drvdata->base);
110 
111 	/* Wait for TMCSReady bit to be set */
112 	rc = tmc_wait_for_tmcready(drvdata);
113 	if (rc) {
114 		dev_err(&drvdata->csdev->dev,
115 			"Failed to enable : TMC is not ready\n");
116 		CS_LOCK(drvdata->base);
117 		return rc;
118 	}
119 
120 	writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
121 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
122 		       drvdata->base + TMC_FFCR);
123 	writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
124 	tmc_enable_hw(drvdata);
125 
126 	CS_LOCK(drvdata->base);
127 	return rc;
128 }
129 
tmc_etf_enable_hw(struct tmc_drvdata * drvdata)130 static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
131 {
132 	int rc = coresight_claim_device(drvdata->csdev);
133 
134 	if (rc)
135 		return rc;
136 
137 	rc = __tmc_etf_enable_hw(drvdata);
138 	if (rc)
139 		coresight_disclaim_device(drvdata->csdev);
140 	return rc;
141 }
142 
tmc_etf_disable_hw(struct tmc_drvdata * drvdata)143 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
144 {
145 	struct coresight_device *csdev = drvdata->csdev;
146 
147 	CS_UNLOCK(drvdata->base);
148 
149 	tmc_flush_and_stop(drvdata);
150 	tmc_disable_hw(drvdata);
151 	coresight_disclaim_device_unlocked(csdev);
152 	CS_LOCK(drvdata->base);
153 }
154 
155 /*
156  * Return the available trace data in the buffer from @pos, with
157  * a maximum limit of @len, updating the @bufpp on where to
158  * find it.
159  */
tmc_etb_get_sysfs_trace(struct tmc_drvdata * drvdata,loff_t pos,size_t len,char ** bufpp)160 ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
161 				loff_t pos, size_t len, char **bufpp)
162 {
163 	ssize_t actual = len;
164 
165 	/* Adjust the len to available size @pos */
166 	if (pos + actual > drvdata->len)
167 		actual = drvdata->len - pos;
168 	if (actual > 0)
169 		*bufpp = drvdata->buf + pos;
170 	return actual;
171 }
172 
tmc_enable_etf_sink_sysfs(struct coresight_device * csdev)173 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
174 {
175 	int ret = 0;
176 	bool used = false;
177 	char *buf = NULL;
178 	unsigned long flags;
179 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
180 
181 	/*
182 	 * If we don't have a buffer release the lock and allocate memory.
183 	 * Otherwise keep the lock and move along.
184 	 */
185 	spin_lock_irqsave(&drvdata->spinlock, flags);
186 	if (!drvdata->buf) {
187 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
188 
189 		/* Allocating the memory here while outside of the spinlock */
190 		buf = kzalloc(drvdata->size, GFP_KERNEL);
191 		if (!buf)
192 			return -ENOMEM;
193 
194 		/* Let's try again */
195 		spin_lock_irqsave(&drvdata->spinlock, flags);
196 	}
197 
198 	if (drvdata->reading) {
199 		ret = -EBUSY;
200 		goto out;
201 	}
202 
203 	/*
204 	 * In sysFS mode we can have multiple writers per sink.  Since this
205 	 * sink is already enabled no memory is needed and the HW need not be
206 	 * touched.
207 	 */
208 	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
209 		csdev->refcnt++;
210 		goto out;
211 	}
212 
213 	/*
214 	 * If drvdata::buf isn't NULL, memory was allocated for a previous
215 	 * trace run but wasn't read.  If so simply zero-out the memory.
216 	 * Otherwise use the memory allocated above.
217 	 *
218 	 * The memory is freed when users read the buffer using the
219 	 * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
220 	 * details.
221 	 */
222 	if (drvdata->buf) {
223 		memset(drvdata->buf, 0, drvdata->size);
224 	} else {
225 		used = true;
226 		drvdata->buf = buf;
227 	}
228 
229 	ret = tmc_etb_enable_hw(drvdata);
230 	if (!ret) {
231 		coresight_set_mode(csdev, CS_MODE_SYSFS);
232 		csdev->refcnt++;
233 	} else {
234 		/* Free up the buffer if we failed to enable */
235 		used = false;
236 	}
237 out:
238 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
239 
240 	/* Free memory outside the spinlock if need be */
241 	if (!used)
242 		kfree(buf);
243 
244 	return ret;
245 }
246 
tmc_enable_etf_sink_perf(struct coresight_device * csdev,void * data)247 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
248 {
249 	int ret = 0;
250 	pid_t pid;
251 	unsigned long flags;
252 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
253 	struct perf_output_handle *handle = data;
254 	struct cs_buffers *buf = etm_perf_sink_config(handle);
255 
256 	spin_lock_irqsave(&drvdata->spinlock, flags);
257 	do {
258 		ret = -EINVAL;
259 		if (drvdata->reading)
260 			break;
261 		/*
262 		 * No need to continue if the ETB/ETF is already operated
263 		 * from sysFS.
264 		 */
265 		if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
266 			ret = -EBUSY;
267 			break;
268 		}
269 
270 		/* Get a handle on the pid of the process to monitor */
271 		pid = buf->pid;
272 
273 		if (drvdata->pid != -1 && drvdata->pid != pid) {
274 			ret = -EBUSY;
275 			break;
276 		}
277 
278 		ret = tmc_set_etf_buffer(csdev, handle);
279 		if (ret)
280 			break;
281 
282 		/*
283 		 * No HW configuration is needed if the sink is already in
284 		 * use for this session.
285 		 */
286 		if (drvdata->pid == pid) {
287 			csdev->refcnt++;
288 			break;
289 		}
290 
291 		ret  = tmc_etb_enable_hw(drvdata);
292 		if (!ret) {
293 			/* Associate with monitored process. */
294 			drvdata->pid = pid;
295 			coresight_set_mode(csdev, CS_MODE_PERF);
296 			csdev->refcnt++;
297 		}
298 	} while (0);
299 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
300 
301 	return ret;
302 }
303 
tmc_enable_etf_sink(struct coresight_device * csdev,enum cs_mode mode,void * data)304 static int tmc_enable_etf_sink(struct coresight_device *csdev,
305 			       enum cs_mode mode, void *data)
306 {
307 	int ret;
308 
309 	switch (mode) {
310 	case CS_MODE_SYSFS:
311 		ret = tmc_enable_etf_sink_sysfs(csdev);
312 		break;
313 	case CS_MODE_PERF:
314 		ret = tmc_enable_etf_sink_perf(csdev, data);
315 		break;
316 	/* We shouldn't be here */
317 	default:
318 		ret = -EINVAL;
319 		break;
320 	}
321 
322 	if (ret)
323 		return ret;
324 
325 	dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
326 	return 0;
327 }
328 
tmc_disable_etf_sink(struct coresight_device * csdev)329 static int tmc_disable_etf_sink(struct coresight_device *csdev)
330 {
331 	unsigned long flags;
332 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
333 
334 	spin_lock_irqsave(&drvdata->spinlock, flags);
335 
336 	if (drvdata->reading) {
337 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
338 		return -EBUSY;
339 	}
340 
341 	csdev->refcnt--;
342 	if (csdev->refcnt) {
343 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
344 		return -EBUSY;
345 	}
346 
347 	/* Complain if we (somehow) got out of sync */
348 	WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
349 	tmc_etb_disable_hw(drvdata);
350 	/* Dissociate from monitored process. */
351 	drvdata->pid = -1;
352 	coresight_set_mode(csdev, CS_MODE_DISABLED);
353 
354 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
355 
356 	dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
357 	return 0;
358 }
359 
tmc_enable_etf_link(struct coresight_device * csdev,struct coresight_connection * in,struct coresight_connection * out)360 static int tmc_enable_etf_link(struct coresight_device *csdev,
361 			       struct coresight_connection *in,
362 			       struct coresight_connection *out)
363 {
364 	int ret = 0;
365 	unsigned long flags;
366 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
367 	bool first_enable = false;
368 
369 	spin_lock_irqsave(&drvdata->spinlock, flags);
370 	if (drvdata->reading) {
371 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
372 		return -EBUSY;
373 	}
374 
375 	if (csdev->refcnt == 0) {
376 		ret = tmc_etf_enable_hw(drvdata);
377 		if (!ret) {
378 			coresight_set_mode(csdev, CS_MODE_SYSFS);
379 			first_enable = true;
380 		}
381 	}
382 	if (!ret)
383 		csdev->refcnt++;
384 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
385 
386 	if (first_enable)
387 		dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
388 	return ret;
389 }
390 
tmc_disable_etf_link(struct coresight_device * csdev,struct coresight_connection * in,struct coresight_connection * out)391 static void tmc_disable_etf_link(struct coresight_device *csdev,
392 				 struct coresight_connection *in,
393 				 struct coresight_connection *out)
394 {
395 	unsigned long flags;
396 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
397 	bool last_disable = false;
398 
399 	spin_lock_irqsave(&drvdata->spinlock, flags);
400 	if (drvdata->reading) {
401 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
402 		return;
403 	}
404 
405 	csdev->refcnt--;
406 	if (csdev->refcnt == 0) {
407 		tmc_etf_disable_hw(drvdata);
408 		coresight_set_mode(csdev, CS_MODE_DISABLED);
409 		last_disable = true;
410 	}
411 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
412 
413 	if (last_disable)
414 		dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
415 }
416 
tmc_alloc_etf_buffer(struct coresight_device * csdev,struct perf_event * event,void ** pages,int nr_pages,bool overwrite)417 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
418 				  struct perf_event *event, void **pages,
419 				  int nr_pages, bool overwrite)
420 {
421 	int node;
422 	struct cs_buffers *buf;
423 
424 	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
425 
426 	/* Allocate memory structure for interaction with Perf */
427 	buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
428 	if (!buf)
429 		return NULL;
430 
431 	buf->pid = task_pid_nr(event->owner);
432 	buf->snapshot = overwrite;
433 	buf->nr_pages = nr_pages;
434 	buf->data_pages = pages;
435 
436 	return buf;
437 }
438 
tmc_free_etf_buffer(void * config)439 static void tmc_free_etf_buffer(void *config)
440 {
441 	struct cs_buffers *buf = config;
442 
443 	kfree(buf);
444 }
445 
tmc_set_etf_buffer(struct coresight_device * csdev,struct perf_output_handle * handle)446 static int tmc_set_etf_buffer(struct coresight_device *csdev,
447 			      struct perf_output_handle *handle)
448 {
449 	int ret = 0;
450 	unsigned long head;
451 	struct cs_buffers *buf = etm_perf_sink_config(handle);
452 
453 	if (!buf)
454 		return -EINVAL;
455 
456 	/* wrap head around to the amount of space we have */
457 	head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1);
458 
459 	/* find the page to write to */
460 	buf->cur = head / PAGE_SIZE;
461 
462 	/* and offset within that page */
463 	buf->offset = head % PAGE_SIZE;
464 
465 	local_set(&buf->data_size, 0);
466 
467 	return ret;
468 }
469 
tmc_update_etf_buffer(struct coresight_device * csdev,struct perf_output_handle * handle,void * sink_config)470 static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
471 				  struct perf_output_handle *handle,
472 				  void *sink_config)
473 {
474 	bool lost = false;
475 	int i, cur;
476 	const u32 *barrier;
477 	u32 *buf_ptr;
478 	u64 read_ptr, write_ptr;
479 	u32 status;
480 	unsigned long offset, to_read = 0, flags;
481 	struct cs_buffers *buf = sink_config;
482 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
483 
484 	if (!buf)
485 		return 0;
486 
487 	/* This shouldn't happen */
488 	if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF))
489 		return 0;
490 
491 	spin_lock_irqsave(&drvdata->spinlock, flags);
492 
493 	/* Don't do anything if another tracer is using this sink */
494 	if (csdev->refcnt != 1)
495 		goto out;
496 
497 	CS_UNLOCK(drvdata->base);
498 
499 	tmc_flush_and_stop(drvdata);
500 
501 	read_ptr = tmc_read_rrp(drvdata);
502 	write_ptr = tmc_read_rwp(drvdata);
503 
504 	/*
505 	 * Get a hold of the status register and see if a wrap around
506 	 * has occurred.  If so adjust things accordingly.
507 	 */
508 	status = readl_relaxed(drvdata->base + TMC_STS);
509 	if (status & TMC_STS_FULL) {
510 		lost = true;
511 		to_read = drvdata->size;
512 	} else {
513 		to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
514 	}
515 
516 	/*
517 	 * The TMC RAM buffer may be bigger than the space available in the
518 	 * perf ring buffer (handle->size).  If so advance the RRP so that we
519 	 * get the latest trace data.  In snapshot mode none of that matters
520 	 * since we are expected to clobber stale data in favour of the latest
521 	 * traces.
522 	 */
523 	if (!buf->snapshot && to_read > handle->size) {
524 		u32 mask = tmc_get_memwidth_mask(drvdata);
525 
526 		/*
527 		 * Make sure the new size is aligned in accordance with the
528 		 * requirement explained in function tmc_get_memwidth_mask().
529 		 */
530 		to_read = handle->size & mask;
531 		/* Move the RAM read pointer up */
532 		read_ptr = (write_ptr + drvdata->size) - to_read;
533 		/* Make sure we are still within our limits */
534 		if (read_ptr > (drvdata->size - 1))
535 			read_ptr -= drvdata->size;
536 		/* Tell the HW */
537 		tmc_write_rrp(drvdata, read_ptr);
538 		lost = true;
539 	}
540 
541 	/*
542 	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
543 	 * captured buffer is expected to be truncated and 2) a full buffer
544 	 * prevents the event from being re-enabled by the perf core,
545 	 * resulting in stale data being send to user space.
546 	 */
547 	if (!buf->snapshot && lost)
548 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
549 
550 	cur = buf->cur;
551 	offset = buf->offset;
552 	barrier = coresight_barrier_pkt;
553 
554 	/* for every byte to read */
555 	for (i = 0; i < to_read; i += 4) {
556 		buf_ptr = buf->data_pages[cur] + offset;
557 		*buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
558 
559 		if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {
560 			*buf_ptr = *barrier;
561 			barrier++;
562 		}
563 
564 		offset += 4;
565 		if (offset >= PAGE_SIZE) {
566 			offset = 0;
567 			cur++;
568 			/* wrap around at the end of the buffer */
569 			cur &= buf->nr_pages - 1;
570 		}
571 	}
572 
573 	/*
574 	 * In snapshot mode we simply increment the head by the number of byte
575 	 * that were written.  User space will figure out how many bytes to get
576 	 * from the AUX buffer based on the position of the head.
577 	 */
578 	if (buf->snapshot)
579 		handle->head += to_read;
580 
581 	/*
582 	 * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace
583 	 * data before the aux_head is updated via perf_aux_output_end(), which
584 	 * is expected by the perf ring buffer.
585 	 */
586 	CS_LOCK(drvdata->base);
587 out:
588 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
589 
590 	return to_read;
591 }
592 
593 static const struct coresight_ops_sink tmc_etf_sink_ops = {
594 	.enable		= tmc_enable_etf_sink,
595 	.disable	= tmc_disable_etf_sink,
596 	.alloc_buffer	= tmc_alloc_etf_buffer,
597 	.free_buffer	= tmc_free_etf_buffer,
598 	.update_buffer	= tmc_update_etf_buffer,
599 };
600 
601 static const struct coresight_ops_link tmc_etf_link_ops = {
602 	.enable		= tmc_enable_etf_link,
603 	.disable	= tmc_disable_etf_link,
604 };
605 
606 const struct coresight_ops tmc_etb_cs_ops = {
607 	.sink_ops	= &tmc_etf_sink_ops,
608 };
609 
610 const struct coresight_ops tmc_etf_cs_ops = {
611 	.sink_ops	= &tmc_etf_sink_ops,
612 	.link_ops	= &tmc_etf_link_ops,
613 };
614 
tmc_read_prepare_etb(struct tmc_drvdata * drvdata)615 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
616 {
617 	enum tmc_mode mode;
618 	int ret = 0;
619 	unsigned long flags;
620 
621 	/* config types are set a boot time and never change */
622 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
623 			 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
624 		return -EINVAL;
625 
626 	spin_lock_irqsave(&drvdata->spinlock, flags);
627 
628 	if (drvdata->reading) {
629 		ret = -EBUSY;
630 		goto out;
631 	}
632 
633 	/* Don't interfere if operated from Perf */
634 	if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) {
635 		ret = -EINVAL;
636 		goto out;
637 	}
638 
639 	/* If drvdata::buf is NULL the trace data has been read already */
640 	if (drvdata->buf == NULL) {
641 		ret = -EINVAL;
642 		goto out;
643 	}
644 
645 	/* Disable the TMC if need be */
646 	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
647 		/* There is no point in reading a TMC in HW FIFO mode */
648 		mode = readl_relaxed(drvdata->base + TMC_MODE);
649 		if (mode != TMC_MODE_CIRCULAR_BUFFER) {
650 			ret = -EINVAL;
651 			goto out;
652 		}
653 		__tmc_etb_disable_hw(drvdata);
654 	}
655 
656 	drvdata->reading = true;
657 out:
658 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
659 
660 	return ret;
661 }
662 
tmc_read_unprepare_etb(struct tmc_drvdata * drvdata)663 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
664 {
665 	char *buf = NULL;
666 	enum tmc_mode mode;
667 	unsigned long flags;
668 	int rc = 0;
669 
670 	/* config types are set a boot time and never change */
671 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
672 			 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
673 		return -EINVAL;
674 
675 	spin_lock_irqsave(&drvdata->spinlock, flags);
676 
677 	/* Re-enable the TMC if need be */
678 	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
679 		/* There is no point in reading a TMC in HW FIFO mode */
680 		mode = readl_relaxed(drvdata->base + TMC_MODE);
681 		if (mode != TMC_MODE_CIRCULAR_BUFFER) {
682 			spin_unlock_irqrestore(&drvdata->spinlock, flags);
683 			return -EINVAL;
684 		}
685 		/*
686 		 * The trace run will continue with the same allocated trace
687 		 * buffer. As such zero-out the buffer so that we don't end
688 		 * up with stale data.
689 		 *
690 		 * Since the tracer is still enabled drvdata::buf
691 		 * can't be NULL.
692 		 */
693 		memset(drvdata->buf, 0, drvdata->size);
694 		rc = __tmc_etb_enable_hw(drvdata);
695 		if (rc) {
696 			spin_unlock_irqrestore(&drvdata->spinlock, flags);
697 			return rc;
698 		}
699 	} else {
700 		/*
701 		 * The ETB/ETF is not tracing and the buffer was just read.
702 		 * As such prepare to free the trace buffer.
703 		 */
704 		buf = drvdata->buf;
705 		drvdata->buf = NULL;
706 	}
707 
708 	drvdata->reading = false;
709 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
710 
711 	/*
712 	 * Free allocated memory outside of the spinlock.  There is no need
713 	 * to assert the validity of 'buf' since calling kfree(NULL) is safe.
714 	 */
715 	kfree(buf);
716 
717 	return 0;
718 }
719