xref: /linux/tools/perf/scripts/python/export-to-sqlite.py (revision da9cfb87a44da61f2403c4312916befcb6b6d7e8) 
1# export-to-sqlite.py: export perf data to a sqlite3 database
2# Copyright (c) 2017, Intel Corporation.
3#
4# This program is free software; you can redistribute it and/or modify it
5# under the terms and conditions of the GNU General Public License,
6# version 2, as published by the Free Software Foundation.
7#
8# This program is distributed in the hope it will be useful, but WITHOUT
9# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11# more details.
12
13import os
14import sys
15import struct
16import datetime
17
18# To use this script you will need to have installed package python-pyside which
19# provides LGPL-licensed Python bindings for Qt.  You will also need the package
20# libqt4-sql-sqlite for Qt sqlite3 support.
21#
22# An example of using this script with Intel PT:
23#
24#	$ perf record -e intel_pt//u ls
25#	$ perf script -s ~/libexec/perf-core/scripts/python/export-to-sqlite.py pt_example branches calls
26#	2017-07-31 14:26:07.326913 Creating database...
27#	2017-07-31 14:26:07.538097 Writing records...
28#	2017-07-31 14:26:09.889292 Adding indexes
29#	2017-07-31 14:26:09.958746 Done
30#
31# To browse the database, sqlite3 can be used e.g.
32#
33#	$ sqlite3 pt_example
34#	sqlite> .header on
35#	sqlite> select * from samples_view where id < 10;
36#	sqlite> .mode column
37#	sqlite> select * from samples_view where id < 10;
38#	sqlite> .tables
39#	sqlite> .schema samples_view
40#	sqlite> .quit
41#
42# An example of using the database is provided by the script
43# exported-sql-viewer.py.  Refer to that script for details.
44#
45# The database structure is practically the same as created by the script
46# export-to-postgresql.py. Refer to that script for details.  A notable
47# difference is  the 'transaction' column of the 'samples' table which is
48# renamed 'transaction_' in sqlite because 'transaction' is a reserved word.
49
50from PySide.QtSql import *
51
52sys.path.append(os.environ['PERF_EXEC_PATH'] + \
53	'/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
54
55# These perf imports are not used at present
56#from perf_trace_context import *
57#from Core import *
58
59perf_db_export_mode = True
60perf_db_export_calls = False
61perf_db_export_callchains = False
62
63def usage():
64	print >> sys.stderr, "Usage is: export-to-sqlite.py <database name> [<columns>] [<calls>] [<callchains>]"
65	print >> sys.stderr, "where:	columns		'all' or 'branches'"
66	print >> sys.stderr, "		calls		'calls' => create calls and call_paths table"
67	print >> sys.stderr, "		callchains	'callchains' => create call_paths table"
68	raise Exception("Too few arguments")
69
70if (len(sys.argv) < 2):
71	usage()
72
73dbname = sys.argv[1]
74
75if (len(sys.argv) >= 3):
76	columns = sys.argv[2]
77else:
78	columns = "all"
79
80if columns not in ("all", "branches"):
81	usage()
82
83branches = (columns == "branches")
84
85for i in range(3,len(sys.argv)):
86	if (sys.argv[i] == "calls"):
87		perf_db_export_calls = True
88	elif (sys.argv[i] == "callchains"):
89		perf_db_export_callchains = True
90	else:
91		usage()
92
93def do_query(q, s):
94	if (q.exec_(s)):
95		return
96	raise Exception("Query failed: " + q.lastError().text())
97
98def do_query_(q):
99	if (q.exec_()):
100		return
101	raise Exception("Query failed: " + q.lastError().text())
102
103print datetime.datetime.today(), "Creating database..."
104
105db_exists = False
106try:
107	f = open(dbname)
108	f.close()
109	db_exists = True
110except:
111	pass
112
113if db_exists:
114	raise Exception(dbname + " already exists")
115
116db = QSqlDatabase.addDatabase('QSQLITE')
117db.setDatabaseName(dbname)
118db.open()
119
120query = QSqlQuery(db)
121
122do_query(query, 'PRAGMA journal_mode = OFF')
123do_query(query, 'BEGIN TRANSACTION')
124
125do_query(query, 'CREATE TABLE selected_events ('
126		'id		integer		NOT NULL	PRIMARY KEY,'
127		'name		varchar(80))')
128do_query(query, 'CREATE TABLE machines ('
129		'id		integer		NOT NULL	PRIMARY KEY,'
130		'pid		integer,'
131		'root_dir 	varchar(4096))')
132do_query(query, 'CREATE TABLE threads ('
133		'id		integer		NOT NULL	PRIMARY KEY,'
134		'machine_id	bigint,'
135		'process_id	bigint,'
136		'pid		integer,'
137		'tid		integer)')
138do_query(query, 'CREATE TABLE comms ('
139		'id		integer		NOT NULL	PRIMARY KEY,'
140		'comm		varchar(16))')
141do_query(query, 'CREATE TABLE comm_threads ('
142		'id		integer		NOT NULL	PRIMARY KEY,'
143		'comm_id	bigint,'
144		'thread_id	bigint)')
145do_query(query, 'CREATE TABLE dsos ('
146		'id		integer		NOT NULL	PRIMARY KEY,'
147		'machine_id	bigint,'
148		'short_name	varchar(256),'
149		'long_name	varchar(4096),'
150		'build_id	varchar(64))')
151do_query(query, 'CREATE TABLE symbols ('
152		'id		integer		NOT NULL	PRIMARY KEY,'
153		'dso_id		bigint,'
154		'sym_start	bigint,'
155		'sym_end	bigint,'
156		'binding	integer,'
157		'name		varchar(2048))')
158do_query(query, 'CREATE TABLE branch_types ('
159		'id		integer		NOT NULL	PRIMARY KEY,'
160		'name		varchar(80))')
161
162if branches:
163	do_query(query, 'CREATE TABLE samples ('
164		'id		integer		NOT NULL	PRIMARY KEY,'
165		'evsel_id	bigint,'
166		'machine_id	bigint,'
167		'thread_id	bigint,'
168		'comm_id	bigint,'
169		'dso_id		bigint,'
170		'symbol_id	bigint,'
171		'sym_offset	bigint,'
172		'ip		bigint,'
173		'time		bigint,'
174		'cpu		integer,'
175		'to_dso_id	bigint,'
176		'to_symbol_id	bigint,'
177		'to_sym_offset	bigint,'
178		'to_ip		bigint,'
179		'branch_type	integer,'
180		'in_tx		boolean,'
181		'call_path_id	bigint)')
182else:
183	do_query(query, 'CREATE TABLE samples ('
184		'id		integer		NOT NULL	PRIMARY KEY,'
185		'evsel_id	bigint,'
186		'machine_id	bigint,'
187		'thread_id	bigint,'
188		'comm_id	bigint,'
189		'dso_id		bigint,'
190		'symbol_id	bigint,'
191		'sym_offset	bigint,'
192		'ip		bigint,'
193		'time		bigint,'
194		'cpu		integer,'
195		'to_dso_id	bigint,'
196		'to_symbol_id	bigint,'
197		'to_sym_offset	bigint,'
198		'to_ip		bigint,'
199		'period		bigint,'
200		'weight		bigint,'
201		'transaction_	bigint,'
202		'data_src	bigint,'
203		'branch_type	integer,'
204		'in_tx		boolean,'
205		'call_path_id	bigint)')
206
207if perf_db_export_calls or perf_db_export_callchains:
208	do_query(query, 'CREATE TABLE call_paths ('
209		'id		integer		NOT NULL	PRIMARY KEY,'
210		'parent_id	bigint,'
211		'symbol_id	bigint,'
212		'ip		bigint)')
213if perf_db_export_calls:
214	do_query(query, 'CREATE TABLE calls ('
215		'id		integer		NOT NULL	PRIMARY KEY,'
216		'thread_id	bigint,'
217		'comm_id	bigint,'
218		'call_path_id	bigint,'
219		'call_time	bigint,'
220		'return_time	bigint,'
221		'branch_count	bigint,'
222		'call_id	bigint,'
223		'return_id	bigint,'
224		'parent_call_path_id	bigint,'
225		'flags		integer,'
226		'parent_id	bigint)')
227
228# printf was added to sqlite in version 3.8.3
229sqlite_has_printf = False
230try:
231	do_query(query, 'SELECT printf("") FROM machines')
232	sqlite_has_printf = True
233except:
234	pass
235
236def emit_to_hex(x):
237	if sqlite_has_printf:
238		return 'printf("%x", ' + x + ')'
239	else:
240		return x
241
242do_query(query, 'CREATE VIEW machines_view AS '
243	'SELECT '
244		'id,'
245		'pid,'
246		'root_dir,'
247		'CASE WHEN id=0 THEN \'unknown\' WHEN pid=-1 THEN \'host\' ELSE \'guest\' END AS host_or_guest'
248	' FROM machines')
249
250do_query(query, 'CREATE VIEW dsos_view AS '
251	'SELECT '
252		'id,'
253		'machine_id,'
254		'(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
255		'short_name,'
256		'long_name,'
257		'build_id'
258	' FROM dsos')
259
260do_query(query, 'CREATE VIEW symbols_view AS '
261	'SELECT '
262		'id,'
263		'name,'
264		'(SELECT short_name FROM dsos WHERE id=dso_id) AS dso,'
265		'dso_id,'
266		'sym_start,'
267		'sym_end,'
268		'CASE WHEN binding=0 THEN \'local\' WHEN binding=1 THEN \'global\' ELSE \'weak\' END AS binding'
269	' FROM symbols')
270
271do_query(query, 'CREATE VIEW threads_view AS '
272	'SELECT '
273		'id,'
274		'machine_id,'
275		'(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
276		'process_id,'
277		'pid,'
278		'tid'
279	' FROM threads')
280
281do_query(query, 'CREATE VIEW comm_threads_view AS '
282	'SELECT '
283		'comm_id,'
284		'(SELECT comm FROM comms WHERE id = comm_id) AS command,'
285		'thread_id,'
286		'(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
287		'(SELECT tid FROM threads WHERE id = thread_id) AS tid'
288	' FROM comm_threads')
289
290if perf_db_export_calls or perf_db_export_callchains:
291	do_query(query, 'CREATE VIEW call_paths_view AS '
292		'SELECT '
293			'c.id,'
294			+ emit_to_hex('c.ip') + ' AS ip,'
295			'c.symbol_id,'
296			'(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol,'
297			'(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id,'
298			'(SELECT dso FROM symbols_view  WHERE id = c.symbol_id) AS dso_short_name,'
299			'c.parent_id,'
300			+ emit_to_hex('p.ip') + ' AS parent_ip,'
301			'p.symbol_id AS parent_symbol_id,'
302			'(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol,'
303			'(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id,'
304			'(SELECT dso FROM symbols_view  WHERE id = p.symbol_id) AS parent_dso_short_name'
305		' FROM call_paths c INNER JOIN call_paths p ON p.id = c.parent_id')
306if perf_db_export_calls:
307	do_query(query, 'CREATE VIEW calls_view AS '
308		'SELECT '
309			'calls.id,'
310			'thread_id,'
311			'(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
312			'(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
313			'(SELECT comm FROM comms WHERE id = comm_id) AS command,'
314			'call_path_id,'
315			+ emit_to_hex('ip') + ' AS ip,'
316			'symbol_id,'
317			'(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
318			'call_time,'
319			'return_time,'
320			'return_time - call_time AS elapsed_time,'
321			'branch_count,'
322			'call_id,'
323			'return_id,'
324			'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE flags END AS flags,'
325			'parent_call_path_id,'
326			'parent_id'
327		' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
328
329do_query(query, 'CREATE VIEW samples_view AS '
330	'SELECT '
331		'id,'
332		'time,'
333		'cpu,'
334		'(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
335		'(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
336		'(SELECT comm FROM comms WHERE id = comm_id) AS command,'
337		'(SELECT name FROM selected_events WHERE id = evsel_id) AS event,'
338		+ emit_to_hex('ip') + ' AS ip_hex,'
339		'(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
340		'sym_offset,'
341		'(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name,'
342		+ emit_to_hex('to_ip') + ' AS to_ip_hex,'
343		'(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,'
344		'to_sym_offset,'
345		'(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,'
346		'(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,'
347		'in_tx'
348	' FROM samples')
349
350do_query(query, 'END TRANSACTION')
351
352evsel_query = QSqlQuery(db)
353evsel_query.prepare("INSERT INTO selected_events VALUES (?, ?)")
354machine_query = QSqlQuery(db)
355machine_query.prepare("INSERT INTO machines VALUES (?, ?, ?)")
356thread_query = QSqlQuery(db)
357thread_query.prepare("INSERT INTO threads VALUES (?, ?, ?, ?, ?)")
358comm_query = QSqlQuery(db)
359comm_query.prepare("INSERT INTO comms VALUES (?, ?)")
360comm_thread_query = QSqlQuery(db)
361comm_thread_query.prepare("INSERT INTO comm_threads VALUES (?, ?, ?)")
362dso_query = QSqlQuery(db)
363dso_query.prepare("INSERT INTO dsos VALUES (?, ?, ?, ?, ?)")
364symbol_query = QSqlQuery(db)
365symbol_query.prepare("INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?)")
366branch_type_query = QSqlQuery(db)
367branch_type_query.prepare("INSERT INTO branch_types VALUES (?, ?)")
368sample_query = QSqlQuery(db)
369if branches:
370	sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
371else:
372	sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
373if perf_db_export_calls or perf_db_export_callchains:
374	call_path_query = QSqlQuery(db)
375	call_path_query.prepare("INSERT INTO call_paths VALUES (?, ?, ?, ?)")
376if perf_db_export_calls:
377	call_query = QSqlQuery(db)
378	call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
379
380def trace_begin():
381	print datetime.datetime.today(), "Writing records..."
382	do_query(query, 'BEGIN TRANSACTION')
383	# id == 0 means unknown.  It is easier to create records for them than replace the zeroes with NULLs
384	evsel_table(0, "unknown")
385	machine_table(0, 0, "unknown")
386	thread_table(0, 0, 0, -1, -1)
387	comm_table(0, "unknown")
388	dso_table(0, 0, "unknown", "unknown", "")
389	symbol_table(0, 0, 0, 0, 0, "unknown")
390	sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
391	if perf_db_export_calls or perf_db_export_callchains:
392		call_path_table(0, 0, 0, 0)
393		call_return_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
394
395unhandled_count = 0
396
397def trace_end():
398	do_query(query, 'END TRANSACTION')
399
400	print datetime.datetime.today(), "Adding indexes"
401	if perf_db_export_calls:
402		do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
403		do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)')
404
405	if (unhandled_count):
406		print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
407	print datetime.datetime.today(), "Done"
408
409def trace_unhandled(event_name, context, event_fields_dict):
410	global unhandled_count
411	unhandled_count += 1
412
413def sched__sched_switch(*x):
414	pass
415
416def bind_exec(q, n, x):
417	for xx in x[0:n]:
418		q.addBindValue(str(xx))
419	do_query_(q)
420
421def evsel_table(*x):
422	bind_exec(evsel_query, 2, x)
423
424def machine_table(*x):
425	bind_exec(machine_query, 3, x)
426
427def thread_table(*x):
428	bind_exec(thread_query, 5, x)
429
430def comm_table(*x):
431	bind_exec(comm_query, 2, x)
432
433def comm_thread_table(*x):
434	bind_exec(comm_thread_query, 3, x)
435
436def dso_table(*x):
437	bind_exec(dso_query, 5, x)
438
439def symbol_table(*x):
440	bind_exec(symbol_query, 6, x)
441
442def branch_type_table(*x):
443	bind_exec(branch_type_query, 2, x)
444
445def sample_table(*x):
446	if branches:
447		for xx in x[0:15]:
448			sample_query.addBindValue(str(xx))
449		for xx in x[19:22]:
450			sample_query.addBindValue(str(xx))
451		do_query_(sample_query)
452	else:
453		bind_exec(sample_query, 22, x)
454
455def call_path_table(*x):
456	bind_exec(call_path_query, 4, x)
457
458def call_return_table(*x):
459	bind_exec(call_query, 12, x)
460