1# firefox-gecko-converter.py - Convert perf record output to Firefox's gecko profile format 2# SPDX-License-Identifier: GPL-2.0 3# 4# The script converts perf.data to Gecko Profile Format, 5# which can be read by https://profiler.firefox.com/. 6# 7# Usage: 8# 9# perf record -a -g -F 99 sleep 60 10# perf script report gecko > output.json 11 12import os 13import sys 14import json 15import argparse 16from functools import reduce 17from dataclasses import dataclass, field 18from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any 19 20# Add the Perf-Trace-Util library to the Python path 21sys.path.append(os.environ['PERF_EXEC_PATH'] + \ 22 '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') 23 24from perf_trace_context import * 25from Core import * 26 27StringID = int 28StackID = int 29FrameID = int 30CategoryID = int 31Milliseconds = float 32 33# start_time is intialiazed only once for the all event traces. 34start_time = None 35 36# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425 37# Follow Brendan Gregg's Flamegraph convention: orange for kernel and yellow for user space by default. 38CATEGORIES = None 39 40# The product name is used by the profiler UI to show the Operating system and Processor. 41PRODUCT = os.popen('uname -op').read().strip() 42 43# Here key = tid, value = Thread 44tid_to_thread = dict() 45 46# The category index is used by the profiler UI to show the color of the flame graph. 47USER_CATEGORY_INDEX = 0 48KERNEL_CATEGORY_INDEX = 1 49 50# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 51class Frame(NamedTuple): 52 string_id: StringID 53 relevantForJS: bool 54 innerWindowID: int 55 implementation: None 56 optimizations: None 57 line: None 58 column: None 59 category: CategoryID 60 subcategory: int 61 62# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 63class Stack(NamedTuple): 64 prefix_id: Optional[StackID] 65 frame_id: FrameID 66 67# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 68class Sample(NamedTuple): 69 stack_id: Optional[StackID] 70 time_ms: Milliseconds 71 responsiveness: int 72 73@dataclass 74class Thread: 75 """A builder for a profile of the thread. 76 77 Attributes: 78 comm: Thread command-line (name). 79 pid: process ID of containing process. 80 tid: thread ID. 81 samples: Timeline of profile samples. 82 frameTable: interned stack frame ID -> stack frame. 83 stringTable: interned string ID -> string. 84 stringMap: interned string -> string ID. 85 stackTable: interned stack ID -> stack. 86 stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID. 87 frameMap: Stack Frame string -> interned Frame ID. 88 comm: str 89 pid: int 90 tid: int 91 samples: List[Sample] = field(default_factory=list) 92 frameTable: List[Frame] = field(default_factory=list) 93 stringTable: List[str] = field(default_factory=list) 94 stringMap: Dict[str, int] = field(default_factory=dict) 95 stackTable: List[Stack] = field(default_factory=list) 96 stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) 97 frameMap: Dict[str, int] = field(default_factory=dict) 98 """ 99 comm: str 100 pid: int 101 tid: int 102 samples: List[Sample] = field(default_factory=list) 103 frameTable: List[Frame] = field(default_factory=list) 104 stringTable: List[str] = field(default_factory=list) 105 stringMap: Dict[str, int] = field(default_factory=dict) 106 stackTable: List[Stack] = field(default_factory=list) 107 stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) 108 frameMap: Dict[str, int] = field(default_factory=dict) 109 110 def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: 111 """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" 112 key = f"{frame_id}" if prefix_id is None else f"{frame_id},{prefix_id}" 113 # key = (prefix_id, frame_id) 114 stack_id = self.stackMap.get(key) 115 if stack_id is None: 116 # return stack_id 117 stack_id = len(self.stackTable) 118 self.stackTable.append(Stack(prefix_id=prefix_id, frame_id=frame_id)) 119 self.stackMap[key] = stack_id 120 return stack_id 121 122 def _intern_string(self, string: str) -> int: 123 """Gets a matching string, or saves the new string. Returns a String ID.""" 124 string_id = self.stringMap.get(string) 125 if string_id is not None: 126 return string_id 127 string_id = len(self.stringTable) 128 self.stringTable.append(string) 129 self.stringMap[string] = string_id 130 return string_id 131 132 def _intern_frame(self, frame_str: str) -> int: 133 """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" 134 frame_id = self.frameMap.get(frame_str) 135 if frame_id is not None: 136 return frame_id 137 frame_id = len(self.frameTable) 138 self.frameMap[frame_str] = frame_id 139 string_id = self._intern_string(frame_str) 140 141 symbol_name_to_category = KERNEL_CATEGORY_INDEX if frame_str.find('kallsyms') != -1 \ 142 or frame_str.find('/vmlinux') != -1 \ 143 or frame_str.endswith('.ko)') \ 144 else USER_CATEGORY_INDEX 145 146 self.frameTable.append(Frame( 147 string_id=string_id, 148 relevantForJS=False, 149 innerWindowID=0, 150 implementation=None, 151 optimizations=None, 152 line=None, 153 column=None, 154 category=symbol_name_to_category, 155 subcategory=None, 156 )) 157 return frame_id 158 159 def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: 160 """Add a timestamped stack trace sample to the thread builder. 161 Args: 162 comm: command-line (name) of the thread at this sample 163 stack: sampled stack frames. Root first, leaf last. 164 time_ms: timestamp of sample in milliseconds. 165 """ 166 # Ihreads may not set their names right after they are created. 167 # Instead, they might do it later. In such situations, to use the latest name they have set. 168 if self.comm != comm: 169 self.comm = comm 170 171 prefix_stack_id = reduce(lambda prefix_id, frame: self._intern_stack 172 (self._intern_frame(frame), prefix_id), stack, None) 173 if prefix_stack_id is not None: 174 self.samples.append(Sample(stack_id=prefix_stack_id, 175 time_ms=time_ms, 176 responsiveness=0)) 177 178 def _to_json_dict(self) -> Dict: 179 """Converts current Thread to GeckoThread JSON format.""" 180 # Gecko profile format is row-oriented data as List[List], 181 # And a schema for interpreting each index. 182 # Schema: 183 # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md 184 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230 185 return { 186 "tid": self.tid, 187 "pid": self.pid, 188 "name": self.comm, 189 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51 190 "markers": { 191 "schema": { 192 "name": 0, 193 "startTime": 1, 194 "endTime": 2, 195 "phase": 3, 196 "category": 4, 197 "data": 5, 198 }, 199 "data": [], 200 }, 201 202 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 203 "samples": { 204 "schema": { 205 "stack": 0, 206 "time": 1, 207 "responsiveness": 2, 208 }, 209 "data": self.samples 210 }, 211 212 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 213 "frameTable": { 214 "schema": { 215 "location": 0, 216 "relevantForJS": 1, 217 "innerWindowID": 2, 218 "implementation": 3, 219 "optimizations": 4, 220 "line": 5, 221 "column": 6, 222 "category": 7, 223 "subcategory": 8, 224 }, 225 "data": self.frameTable, 226 }, 227 228 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 229 "stackTable": { 230 "schema": { 231 "prefix": 0, 232 "frame": 1, 233 }, 234 "data": self.stackTable, 235 }, 236 "stringTable": self.stringTable, 237 "registerTime": 0, 238 "unregisterTime": None, 239 "processType": "default", 240 } 241 242# Uses perf script python interface to parse each 243# event and store the data in the thread builder. 244def process_event(param_dict: Dict) -> None: 245 global start_time 246 global tid_to_thread 247 time_stamp = (param_dict['sample']['time'] // 1000) / 1000 248 pid = param_dict['sample']['pid'] 249 tid = param_dict['sample']['tid'] 250 comm = param_dict['comm'] 251 252 # Start time is the time of the first sample 253 if not start_time: 254 start_time = time_stamp 255 256 # Parse and append the callchain of the current sample into a stack. 257 stack = [] 258 if param_dict['callchain']: 259 for call in param_dict['callchain']: 260 if 'sym' not in call: 261 continue 262 stack.append(f'{call["sym"]["name"]} (in {call["dso"]})') 263 if len(stack) != 0: 264 # Reverse the stack, as root come first and the leaf at the end. 265 stack = stack[::-1] 266 267 # During perf record if -g is not used, the callchain is not available. 268 # In that case, the symbol and dso are available in the event parameters. 269 else: 270 func = param_dict['symbol'] if 'symbol' in param_dict else '[unknown]' 271 dso = param_dict['dso'] if 'dso' in param_dict else '[unknown]' 272 stack.append(f'{func} (in {dso})') 273 274 # Add sample to the specific thread. 275 thread = tid_to_thread.get(tid) 276 if thread is None: 277 thread = Thread(comm=comm, pid=pid, tid=tid) 278 tid_to_thread[tid] = thread 279 thread._add_sample(comm=comm, stack=stack, time_ms=time_stamp) 280 281# Trace_end runs at the end and will be used to aggregate 282# the data into the final json object and print it out to stdout. 283def trace_end() -> None: 284 threads = [thread._to_json_dict() for thread in tid_to_thread.values()] 285 286 # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 287 gecko_profile_with_meta = { 288 "meta": { 289 "interval": 1, 290 "processType": 0, 291 "product": PRODUCT, 292 "stackwalk": 1, 293 "debug": 0, 294 "gcpoison": 0, 295 "asyncstack": 1, 296 "startTime": start_time, 297 "shutdownTime": None, 298 "version": 24, 299 "presymbolicated": True, 300 "categories": CATEGORIES, 301 "markerSchema": [], 302 }, 303 "libs": [], 304 "threads": threads, 305 "processes": [], 306 "pausedRanges": [], 307 } 308 json.dump(gecko_profile_with_meta, sys.stdout, indent=2) 309 310def main() -> None: 311 global CATEGORIES 312 parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format") 313 314 # Add the command-line options 315 # Colors must be defined according to this: 316 # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css 317 parser.add_argument('--user-color', default='yellow', help='Color for the User category') 318 parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category') 319 # Parse the command-line arguments 320 args = parser.parse_args() 321 # Access the values provided by the user 322 user_color = args.user_color 323 kernel_color = args.kernel_color 324 325 CATEGORIES = [ 326 { 327 "name": 'User', 328 "color": user_color, 329 "subcategories": ['Other'] 330 }, 331 { 332 "name": 'Kernel', 333 "color": kernel_color, 334 "subcategories": ['Other'] 335 }, 336 ] 337 338if __name__ == '__main__': 339 main() 340