xref: /linux/tools/perf/pmu-events/jevents.py (revision 3df692169e8486fc3dd91fcd5ea81c27a0bac033)
1#!/usr/bin/env python3
2# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
3"""Convert directories of JSON events to C code."""
4import argparse
5import csv
6from functools import lru_cache
7import json
8import metric
9import os
10import sys
11from typing import (Callable, Dict, Optional, Sequence, Set, Tuple)
12import collections
13
14# Global command line arguments.
15_args = None
16# List of regular event tables.
17_event_tables = []
18# List of event tables generated from "/sys" directories.
19_sys_event_tables = []
20# List of regular metric tables.
21_metric_tables = []
22# List of metric tables generated from "/sys" directories.
23_sys_metric_tables = []
24# Mapping between sys event table names and sys metric table names.
25_sys_event_table_to_metric_table_mapping = {}
26# Map from an event name to an architecture standard
27# JsonEvent. Architecture standard events are in json files in the top
28# f'{_args.starting_dir}/{_args.arch}' directory.
29_arch_std_events = {}
30# Events to write out when the table is closed
31_pending_events = []
32# Name of events table to be written out
33_pending_events_tblname = None
34# Metrics to write out when the table is closed
35_pending_metrics = []
36# Name of metrics table to be written out
37_pending_metrics_tblname = None
38# Global BigCString shared by all structures.
39_bcs = None
40# Map from the name of a metric group to a description of the group.
41_metricgroups = {}
42# Order specific JsonEvent attributes will be visited.
43_json_event_attributes = [
44    # cmp_sevent related attributes.
45    'name', 'topic', 'desc',
46    # Seems useful, put it early.
47    'event',
48    # Short things in alphabetical order.
49    'compat', 'deprecated', 'perpkg', 'unit',
50    # Longer things (the last won't be iterated over during decompress).
51    'long_desc'
52]
53
54# Attributes that are in pmu_metric rather than pmu_event.
55_json_metric_attributes = [
56    'metric_name', 'metric_group', 'metric_expr', 'metric_threshold',
57    'desc', 'long_desc', 'unit', 'compat', 'metricgroup_no_group',
58    'default_metricgroup_name', 'aggr_mode', 'event_grouping'
59]
60# Attributes that are bools or enum int values, encoded as '0', '1',...
61_json_enum_attributes = ['aggr_mode', 'deprecated', 'event_grouping', 'perpkg']
62
63def removesuffix(s: str, suffix: str) -> str:
64  """Remove the suffix from a string
65
66  The removesuffix function is added to str in Python 3.9. We aim for 3.6
67  compatibility and so provide our own function here.
68  """
69  return s[0:-len(suffix)] if s.endswith(suffix) else s
70
71
72def file_name_to_table_name(prefix: str, parents: Sequence[str],
73                            dirname: str) -> str:
74  """Generate a C table name from directory names."""
75  tblname = prefix
76  for p in parents:
77    tblname += '_' + p
78  tblname += '_' + dirname
79  return tblname.replace('-', '_')
80
81
82def c_len(s: str) -> int:
83  """Return the length of s a C string
84
85  This doesn't handle all escape characters properly. It first assumes
86  all \ are for escaping, it then adjusts as it will have over counted
87  \\. The code uses \000 rather than \0 as a terminator as an adjacent
88  number would be folded into a string of \0 (ie. "\0" + "5" doesn't
89  equal a terminator followed by the number 5 but the escape of
90  \05). The code adjusts for \000 but not properly for all octal, hex
91  or unicode values.
92  """
93  try:
94    utf = s.encode(encoding='utf-8',errors='strict')
95  except:
96    print(f'broken string {s}')
97    raise
98  return len(utf) - utf.count(b'\\') + utf.count(b'\\\\') - (utf.count(b'\\000') * 2)
99
100class BigCString:
101  """A class to hold many strings concatenated together.
102
103  Generating a large number of stand-alone C strings creates a large
104  number of relocations in position independent code. The BigCString
105  is a helper for this case. It builds a single string which within it
106  are all the other C strings (to avoid memory issues the string
107  itself is held as a list of strings). The offsets within the big
108  string are recorded and when stored to disk these don't need
109  relocation. To reduce the size of the string further, identical
110  strings are merged. If a longer string ends-with the same value as a
111  shorter string, these entries are also merged.
112  """
113  strings: Set[str]
114  big_string: Sequence[str]
115  offsets: Dict[str, int]
116  insert_number: int
117  insert_point: Dict[str, int]
118  metrics: Set[str]
119
120  def __init__(self):
121    self.strings = set()
122    self.insert_number = 0;
123    self.insert_point = {}
124    self.metrics = set()
125
126  def add(self, s: str, metric: bool) -> None:
127    """Called to add to the big string."""
128    if s not in self.strings:
129      self.strings.add(s)
130      self.insert_point[s] = self.insert_number
131      self.insert_number += 1
132      if metric:
133        self.metrics.add(s)
134
135  def compute(self) -> None:
136    """Called once all strings are added to compute the string and offsets."""
137
138    folded_strings = {}
139    # Determine if two strings can be folded, ie. let 1 string use the
140    # end of another. First reverse all strings and sort them.
141    sorted_reversed_strings = sorted([x[::-1] for x in self.strings])
142
143    # Strings 'xyz' and 'yz' will now be [ 'zy', 'zyx' ]. Scan forward
144    # for each string to see if there is a better candidate to fold it
145    # into, in the example rather than using 'yz' we can use'xyz' at
146    # an offset of 1. We record which string can be folded into which
147    # in folded_strings, we don't need to record the offset as it is
148    # trivially computed from the string lengths.
149    for pos,s in enumerate(sorted_reversed_strings):
150      best_pos = pos
151      for check_pos in range(pos + 1, len(sorted_reversed_strings)):
152        if sorted_reversed_strings[check_pos].startswith(s):
153          best_pos = check_pos
154        else:
155          break
156      if pos != best_pos:
157        folded_strings[s[::-1]] = sorted_reversed_strings[best_pos][::-1]
158
159    # Compute reverse mappings for debugging.
160    fold_into_strings = collections.defaultdict(set)
161    for key, val in folded_strings.items():
162      if key != val:
163        fold_into_strings[val].add(key)
164
165    # big_string_offset is the current location within the C string
166    # being appended to - comments, etc. don't count. big_string is
167    # the string contents represented as a list. Strings are immutable
168    # in Python and so appending to one causes memory issues, while
169    # lists are mutable.
170    big_string_offset = 0
171    self.big_string = []
172    self.offsets = {}
173
174    def string_cmp_key(s: str) -> Tuple[bool, int, str]:
175      return (s in self.metrics, self.insert_point[s], s)
176
177    # Emit all strings that aren't folded in a sorted manner.
178    for s in sorted(self.strings, key=string_cmp_key):
179      if s not in folded_strings:
180        self.offsets[s] = big_string_offset
181        self.big_string.append(f'/* offset={big_string_offset} */ "')
182        self.big_string.append(s)
183        self.big_string.append('"')
184        if s in fold_into_strings:
185          self.big_string.append(' /* also: ' + ', '.join(fold_into_strings[s]) + ' */')
186        self.big_string.append('\n')
187        big_string_offset += c_len(s)
188        continue
189
190    # Compute the offsets of the folded strings.
191    for s in folded_strings.keys():
192      assert s not in self.offsets
193      folded_s = folded_strings[s]
194      self.offsets[s] = self.offsets[folded_s] + c_len(folded_s) - c_len(s)
195
196_bcs = BigCString()
197
198class JsonEvent:
199  """Representation of an event loaded from a json file dictionary."""
200
201  def __init__(self, jd: dict):
202    """Constructor passed the dictionary of parsed json values."""
203
204    def llx(x: int) -> str:
205      """Convert an int to a string similar to a printf modifier of %#llx."""
206      return '0' if x == 0 else hex(x)
207
208    def fixdesc(s: str) -> str:
209      """Fix formatting issue for the desc string."""
210      if s is None:
211        return None
212      return removesuffix(removesuffix(removesuffix(s, '.  '),
213                                       '. '), '.').replace('\n', '\\n').replace(
214                                           '\"', '\\"').replace('\r', '\\r')
215
216    def convert_aggr_mode(aggr_mode: str) -> Optional[str]:
217      """Returns the aggr_mode_class enum value associated with the JSON string."""
218      if not aggr_mode:
219        return None
220      aggr_mode_to_enum = {
221          'PerChip': '1',
222          'PerCore': '2',
223      }
224      return aggr_mode_to_enum[aggr_mode]
225
226    def convert_metric_constraint(metric_constraint: str) -> Optional[str]:
227      """Returns the metric_event_groups enum value associated with the JSON string."""
228      if not metric_constraint:
229        return None
230      metric_constraint_to_enum = {
231          'NO_GROUP_EVENTS': '1',
232          'NO_GROUP_EVENTS_NMI': '2',
233          'NO_NMI_WATCHDOG': '2',
234          'NO_GROUP_EVENTS_SMT': '3',
235      }
236      return metric_constraint_to_enum[metric_constraint]
237
238    def lookup_msr(num: str) -> Optional[str]:
239      """Converts the msr number, or first in a list to the appropriate event field."""
240      if not num:
241        return None
242      msrmap = {
243          0x3F6: 'ldlat=',
244          0x1A6: 'offcore_rsp=',
245          0x1A7: 'offcore_rsp=',
246          0x3F7: 'frontend=',
247      }
248      return msrmap[int(num.split(',', 1)[0], 0)]
249
250    def real_event(name: str, event: str) -> Optional[str]:
251      """Convert well known event names to an event string otherwise use the event argument."""
252      fixed = {
253          'inst_retired.any': 'event=0xc0,period=2000003',
254          'inst_retired.any_p': 'event=0xc0,period=2000003',
255          'cpu_clk_unhalted.ref': 'event=0x0,umask=0x03,period=2000003',
256          'cpu_clk_unhalted.thread': 'event=0x3c,period=2000003',
257          'cpu_clk_unhalted.core': 'event=0x3c,period=2000003',
258          'cpu_clk_unhalted.thread_any': 'event=0x3c,any=1,period=2000003',
259      }
260      if not name:
261        return None
262      if name.lower() in fixed:
263        return fixed[name.lower()]
264      return event
265
266    def unit_to_pmu(unit: str) -> Optional[str]:
267      """Convert a JSON Unit to Linux PMU name."""
268      if not unit:
269        return 'default_core'
270      # Comment brought over from jevents.c:
271      # it's not realistic to keep adding these, we need something more scalable ...
272      table = {
273          'CBO': 'uncore_cbox',
274          'QPI LL': 'uncore_qpi',
275          'SBO': 'uncore_sbox',
276          'iMPH-U': 'uncore_arb',
277          'CPU-M-CF': 'cpum_cf',
278          'CPU-M-SF': 'cpum_sf',
279          'PAI-CRYPTO' : 'pai_crypto',
280          'PAI-EXT' : 'pai_ext',
281          'UPI LL': 'uncore_upi',
282          'hisi_sicl,cpa': 'hisi_sicl,cpa',
283          'hisi_sccl,ddrc': 'hisi_sccl,ddrc',
284          'hisi_sccl,hha': 'hisi_sccl,hha',
285          'hisi_sccl,l3c': 'hisi_sccl,l3c',
286          'imx8_ddr': 'imx8_ddr',
287          'L3PMC': 'amd_l3',
288          'DFPMC': 'amd_df',
289          'cpu_core': 'cpu_core',
290          'cpu_atom': 'cpu_atom',
291          'ali_drw': 'ali_drw',
292          'arm_cmn': 'arm_cmn',
293      }
294      return table[unit] if unit in table else f'uncore_{unit.lower()}'
295
296    eventcode = 0
297    if 'EventCode' in jd:
298      eventcode = int(jd['EventCode'].split(',', 1)[0], 0)
299    if 'ExtSel' in jd:
300      eventcode |= int(jd['ExtSel']) << 8
301    configcode = int(jd['ConfigCode'], 0) if 'ConfigCode' in jd else None
302    eventidcode = int(jd['EventidCode'], 0) if 'EventidCode' in jd else None
303    self.name = jd['EventName'].lower() if 'EventName' in jd else None
304    self.topic = ''
305    self.compat = jd.get('Compat')
306    self.desc = fixdesc(jd.get('BriefDescription'))
307    self.long_desc = fixdesc(jd.get('PublicDescription'))
308    precise = jd.get('PEBS')
309    msr = lookup_msr(jd.get('MSRIndex'))
310    msrval = jd.get('MSRValue')
311    extra_desc = ''
312    if 'Data_LA' in jd:
313      extra_desc += '  Supports address when precise'
314      if 'Errata' in jd:
315        extra_desc += '.'
316    if 'Errata' in jd:
317      extra_desc += '  Spec update: ' + jd['Errata']
318    self.pmu = unit_to_pmu(jd.get('Unit'))
319    filter = jd.get('Filter')
320    self.unit = jd.get('ScaleUnit')
321    self.perpkg = jd.get('PerPkg')
322    self.aggr_mode = convert_aggr_mode(jd.get('AggregationMode'))
323    self.deprecated = jd.get('Deprecated')
324    self.metric_name = jd.get('MetricName')
325    self.metric_group = jd.get('MetricGroup')
326    self.metricgroup_no_group = jd.get('MetricgroupNoGroup')
327    self.default_metricgroup_name = jd.get('DefaultMetricgroupName')
328    self.event_grouping = convert_metric_constraint(jd.get('MetricConstraint'))
329    self.metric_expr = None
330    if 'MetricExpr' in jd:
331      self.metric_expr = metric.ParsePerfJson(jd['MetricExpr']).Simplify()
332    # Note, the metric formula for the threshold isn't parsed as the &
333    # and > have incorrect precedence.
334    self.metric_threshold = jd.get('MetricThreshold')
335
336    arch_std = jd.get('ArchStdEvent')
337    if precise and self.desc and '(Precise Event)' not in self.desc:
338      extra_desc += ' (Must be precise)' if precise == '2' else (' (Precise '
339                                                                 'event)')
340    event = None
341    if configcode is not None:
342      event = f'config={llx(configcode)}'
343    elif eventidcode is not None:
344      event = f'eventid={llx(eventidcode)}'
345    else:
346      event = f'event={llx(eventcode)}'
347    event_fields = [
348        ('AnyThread', 'any='),
349        ('PortMask', 'ch_mask='),
350        ('CounterMask', 'cmask='),
351        ('EdgeDetect', 'edge='),
352        ('FCMask', 'fc_mask='),
353        ('Invert', 'inv='),
354        ('SampleAfterValue', 'period='),
355        ('UMask', 'umask='),
356        ('NodeType', 'type='),
357    ]
358    for key, value in event_fields:
359      if key in jd and jd[key] != '0':
360        event += ',' + value + jd[key]
361    if filter:
362      event += f',{filter}'
363    if msr:
364      event += f',{msr}{msrval}'
365    if self.desc and extra_desc:
366      self.desc += extra_desc
367    if self.long_desc and extra_desc:
368      self.long_desc += extra_desc
369    if arch_std:
370      if arch_std.lower() in _arch_std_events:
371        event = _arch_std_events[arch_std.lower()].event
372        # Copy from the architecture standard event to self for undefined fields.
373        for attr, value in _arch_std_events[arch_std.lower()].__dict__.items():
374          if hasattr(self, attr) and not getattr(self, attr):
375            setattr(self, attr, value)
376      else:
377        raise argparse.ArgumentTypeError('Cannot find arch std event:', arch_std)
378
379    self.event = real_event(self.name, event)
380
381  def __repr__(self) -> str:
382    """String representation primarily for debugging."""
383    s = '{\n'
384    for attr, value in self.__dict__.items():
385      if value:
386        s += f'\t{attr} = {value},\n'
387    return s + '}'
388
389  def build_c_string(self, metric: bool) -> str:
390    s = ''
391    for attr in _json_metric_attributes if metric else _json_event_attributes:
392      x = getattr(self, attr)
393      if metric and x and attr == 'metric_expr':
394        # Convert parsed metric expressions into a string. Slashes
395        # must be doubled in the file.
396        x = x.ToPerfJson().replace('\\', '\\\\')
397      if metric and x and attr == 'metric_threshold':
398        x = x.replace('\\', '\\\\')
399      if attr in _json_enum_attributes:
400        s += x if x else '0'
401      else:
402        s += f'{x}\\000' if x else '\\000'
403    return s
404
405  def to_c_string(self, metric: bool) -> str:
406    """Representation of the event as a C struct initializer."""
407
408    s = self.build_c_string(metric)
409    return f'{{ { _bcs.offsets[s] } }}, /* {s} */\n'
410
411
412@lru_cache(maxsize=None)
413def read_json_events(path: str, topic: str) -> Sequence[JsonEvent]:
414  """Read json events from the specified file."""
415  try:
416    events = json.load(open(path), object_hook=JsonEvent)
417  except BaseException as err:
418    print(f"Exception processing {path}")
419    raise
420  metrics: list[Tuple[str, str, metric.Expression]] = []
421  for event in events:
422    event.topic = topic
423    if event.metric_name and '-' not in event.metric_name:
424      metrics.append((event.pmu, event.metric_name, event.metric_expr))
425  updates = metric.RewriteMetricsInTermsOfOthers(metrics)
426  if updates:
427    for event in events:
428      if event.metric_name in updates:
429        # print(f'Updated {event.metric_name} from\n"{event.metric_expr}"\n'
430        #       f'to\n"{updates[event.metric_name]}"')
431        event.metric_expr = updates[event.metric_name]
432
433  return events
434
435def preprocess_arch_std_files(archpath: str) -> None:
436  """Read in all architecture standard events."""
437  global _arch_std_events
438  for item in os.scandir(archpath):
439    if item.is_file() and item.name.endswith('.json'):
440      for event in read_json_events(item.path, topic=''):
441        if event.name:
442          _arch_std_events[event.name.lower()] = event
443        if event.metric_name:
444          _arch_std_events[event.metric_name.lower()] = event
445
446
447def add_events_table_entries(item: os.DirEntry, topic: str) -> None:
448  """Add contents of file to _pending_events table."""
449  for e in read_json_events(item.path, topic):
450    if e.name:
451      _pending_events.append(e)
452    if e.metric_name:
453      _pending_metrics.append(e)
454
455
456def print_pending_events() -> None:
457  """Optionally close events table."""
458
459  def event_cmp_key(j: JsonEvent) -> Tuple[str, str, bool, str, str]:
460    def fix_none(s: Optional[str]) -> str:
461      if s is None:
462        return ''
463      return s
464
465    return (fix_none(j.pmu).replace(',','_'), fix_none(j.name), j.desc is not None, fix_none(j.topic),
466            fix_none(j.metric_name))
467
468  global _pending_events
469  if not _pending_events:
470    return
471
472  global _pending_events_tblname
473  if _pending_events_tblname.endswith('_sys'):
474    global _sys_event_tables
475    _sys_event_tables.append(_pending_events_tblname)
476  else:
477    global event_tables
478    _event_tables.append(_pending_events_tblname)
479
480  first = True
481  last_pmu = None
482  pmus = set()
483  for event in sorted(_pending_events, key=event_cmp_key):
484    if event.pmu != last_pmu:
485      if not first:
486        _args.output_file.write('};\n')
487      pmu_name = event.pmu.replace(',', '_')
488      _args.output_file.write(
489          f'static const struct compact_pmu_event {_pending_events_tblname}_{pmu_name}[] = {{\n')
490      first = False
491      last_pmu = event.pmu
492      pmus.add((event.pmu, pmu_name))
493
494    _args.output_file.write(event.to_c_string(metric=False))
495  _pending_events = []
496
497  _args.output_file.write(f"""
498}};
499
500const struct pmu_table_entry {_pending_events_tblname}[] = {{
501""")
502  for (pmu, tbl_pmu) in sorted(pmus):
503    pmu_name = f"{pmu}\\000"
504    _args.output_file.write(f"""{{
505     .entries = {_pending_events_tblname}_{tbl_pmu},
506     .num_entries = ARRAY_SIZE({_pending_events_tblname}_{tbl_pmu}),
507     .pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }},
508}},
509""")
510  _args.output_file.write('};\n\n')
511
512def print_pending_metrics() -> None:
513  """Optionally close metrics table."""
514
515  def metric_cmp_key(j: JsonEvent) -> Tuple[bool, str, str]:
516    def fix_none(s: Optional[str]) -> str:
517      if s is None:
518        return ''
519      return s
520
521    return (j.desc is not None, fix_none(j.pmu), fix_none(j.metric_name))
522
523  global _pending_metrics
524  if not _pending_metrics:
525    return
526
527  global _pending_metrics_tblname
528  if _pending_metrics_tblname.endswith('_sys'):
529    global _sys_metric_tables
530    _sys_metric_tables.append(_pending_metrics_tblname)
531  else:
532    global metric_tables
533    _metric_tables.append(_pending_metrics_tblname)
534
535  first = True
536  last_pmu = None
537  pmus = set()
538  for metric in sorted(_pending_metrics, key=metric_cmp_key):
539    if metric.pmu != last_pmu:
540      if not first:
541        _args.output_file.write('};\n')
542      pmu_name = metric.pmu.replace(',', '_')
543      _args.output_file.write(
544          f'static const struct compact_pmu_event {_pending_metrics_tblname}_{pmu_name}[] = {{\n')
545      first = False
546      last_pmu = metric.pmu
547      pmus.add((metric.pmu, pmu_name))
548
549    _args.output_file.write(metric.to_c_string(metric=True))
550  _pending_metrics = []
551
552  _args.output_file.write(f"""
553}};
554
555const struct pmu_table_entry {_pending_metrics_tblname}[] = {{
556""")
557  for (pmu, tbl_pmu) in sorted(pmus):
558    pmu_name = f"{pmu}\\000"
559    _args.output_file.write(f"""{{
560     .entries = {_pending_metrics_tblname}_{tbl_pmu},
561     .num_entries = ARRAY_SIZE({_pending_metrics_tblname}_{tbl_pmu}),
562     .pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }},
563}},
564""")
565  _args.output_file.write('};\n\n')
566
567def get_topic(topic: str) -> str:
568  if topic.endswith('metrics.json'):
569    return 'metrics'
570  return removesuffix(topic, '.json').replace('-', ' ')
571
572def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
573
574  if item.is_dir():
575    return
576
577  # base dir or too deep
578  level = len(parents)
579  if level == 0 or level > 4:
580    return
581
582  # Ignore other directories. If the file name does not have a .json
583  # extension, ignore it. It could be a readme.txt for instance.
584  if not item.is_file() or not item.name.endswith('.json'):
585    return
586
587  if item.name == 'metricgroups.json':
588    metricgroup_descriptions = json.load(open(item.path))
589    for mgroup in metricgroup_descriptions:
590      assert len(mgroup) > 1, parents
591      description = f"{metricgroup_descriptions[mgroup]}\\000"
592      mgroup = f"{mgroup}\\000"
593      _bcs.add(mgroup, metric=True)
594      _bcs.add(description, metric=True)
595      _metricgroups[mgroup] = description
596    return
597
598  topic = get_topic(item.name)
599  for event in read_json_events(item.path, topic):
600    pmu_name = f"{event.pmu}\\000"
601    if event.name:
602      _bcs.add(pmu_name, metric=False)
603      _bcs.add(event.build_c_string(metric=False), metric=False)
604    if event.metric_name:
605      _bcs.add(pmu_name, metric=True)
606      _bcs.add(event.build_c_string(metric=True), metric=True)
607
608def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
609  """Process a JSON file during the main walk."""
610  def is_leaf_dir(path: str) -> bool:
611    for item in os.scandir(path):
612      if item.is_dir():
613        return False
614    return True
615
616  # model directory, reset topic
617  if item.is_dir() and is_leaf_dir(item.path):
618    print_pending_events()
619    print_pending_metrics()
620
621    global _pending_events_tblname
622    _pending_events_tblname = file_name_to_table_name('pmu_events_', parents, item.name)
623    global _pending_metrics_tblname
624    _pending_metrics_tblname = file_name_to_table_name('pmu_metrics_', parents, item.name)
625
626    if item.name == 'sys':
627      _sys_event_table_to_metric_table_mapping[_pending_events_tblname] = _pending_metrics_tblname
628    return
629
630  # base dir or too deep
631  level = len(parents)
632  if level == 0 or level > 4:
633    return
634
635  # Ignore other directories. If the file name does not have a .json
636  # extension, ignore it. It could be a readme.txt for instance.
637  if not item.is_file() or not item.name.endswith('.json') or item.name == 'metricgroups.json':
638    return
639
640  add_events_table_entries(item, get_topic(item.name))
641
642
643def print_mapping_table(archs: Sequence[str]) -> None:
644  """Read the mapfile and generate the struct from cpuid string to event table."""
645  _args.output_file.write("""
646/* Struct used to make the PMU event table implementation opaque to callers. */
647struct pmu_events_table {
648        const struct pmu_table_entry *pmus;
649        uint32_t num_pmus;
650};
651
652/* Struct used to make the PMU metric table implementation opaque to callers. */
653struct pmu_metrics_table {
654        const struct pmu_table_entry *pmus;
655        uint32_t num_pmus;
656};
657
658/*
659 * Map a CPU to its table of PMU events. The CPU is identified by the
660 * cpuid field, which is an arch-specific identifier for the CPU.
661 * The identifier specified in tools/perf/pmu-events/arch/xxx/mapfile
662 * must match the get_cpuid_str() in tools/perf/arch/xxx/util/header.c)
663 *
664 * The  cpuid can contain any character other than the comma.
665 */
666struct pmu_events_map {
667        const char *arch;
668        const char *cpuid;
669        struct pmu_events_table event_table;
670        struct pmu_metrics_table metric_table;
671};
672
673/*
674 * Global table mapping each known CPU for the architecture to its
675 * table of PMU events.
676 */
677const struct pmu_events_map pmu_events_map[] = {
678""")
679  for arch in archs:
680    if arch == 'test':
681      _args.output_file.write("""{
682\t.arch = "testarch",
683\t.cpuid = "testcpu",
684\t.event_table = {
685\t\t.pmus = pmu_events__test_soc_cpu,
686\t\t.num_pmus = ARRAY_SIZE(pmu_events__test_soc_cpu),
687\t},
688\t.metric_table = {
689\t\t.pmus = pmu_metrics__test_soc_cpu,
690\t\t.num_pmus = ARRAY_SIZE(pmu_metrics__test_soc_cpu),
691\t}
692},
693""")
694    else:
695      with open(f'{_args.starting_dir}/{arch}/mapfile.csv') as csvfile:
696        table = csv.reader(csvfile)
697        first = True
698        for row in table:
699          # Skip the first row or any row beginning with #.
700          if not first and len(row) > 0 and not row[0].startswith('#'):
701            event_tblname = file_name_to_table_name('pmu_events_', [], row[2].replace('/', '_'))
702            if event_tblname in _event_tables:
703              event_size = f'ARRAY_SIZE({event_tblname})'
704            else:
705              event_tblname = 'NULL'
706              event_size = '0'
707            metric_tblname = file_name_to_table_name('pmu_metrics_', [], row[2].replace('/', '_'))
708            if metric_tblname in _metric_tables:
709              metric_size = f'ARRAY_SIZE({metric_tblname})'
710            else:
711              metric_tblname = 'NULL'
712              metric_size = '0'
713            if event_size == '0' and metric_size == '0':
714              continue
715            cpuid = row[0].replace('\\', '\\\\')
716            _args.output_file.write(f"""{{
717\t.arch = "{arch}",
718\t.cpuid = "{cpuid}",
719\t.event_table = {{
720\t\t.pmus = {event_tblname},
721\t\t.num_pmus = {event_size}
722\t}},
723\t.metric_table = {{
724\t\t.pmus = {metric_tblname},
725\t\t.num_pmus = {metric_size}
726\t}}
727}},
728""")
729          first = False
730
731  _args.output_file.write("""{
732\t.arch = 0,
733\t.cpuid = 0,
734\t.event_table = { 0, 0 },
735\t.metric_table = { 0, 0 },
736}
737};
738""")
739
740
741def print_system_mapping_table() -> None:
742  """C struct mapping table array for tables from /sys directories."""
743  _args.output_file.write("""
744struct pmu_sys_events {
745\tconst char *name;
746\tstruct pmu_events_table event_table;
747\tstruct pmu_metrics_table metric_table;
748};
749
750static const struct pmu_sys_events pmu_sys_event_tables[] = {
751""")
752  printed_metric_tables = []
753  for tblname in _sys_event_tables:
754    _args.output_file.write(f"""\t{{
755\t\t.event_table = {{
756\t\t\t.pmus = {tblname},
757\t\t\t.num_pmus = ARRAY_SIZE({tblname})
758\t\t}},""")
759    metric_tblname = _sys_event_table_to_metric_table_mapping[tblname]
760    if metric_tblname in _sys_metric_tables:
761      _args.output_file.write(f"""
762\t\t.metric_table = {{
763\t\t\t.pmus = {metric_tblname},
764\t\t\t.num_pmus = ARRAY_SIZE({metric_tblname})
765\t\t}},""")
766      printed_metric_tables.append(metric_tblname)
767    _args.output_file.write(f"""
768\t\t.name = \"{tblname}\",
769\t}},
770""")
771  for tblname in _sys_metric_tables:
772    if tblname in printed_metric_tables:
773      continue
774    _args.output_file.write(f"""\t{{
775\t\t.metric_table = {{
776\t\t\t.pmus = {tblname},
777\t\t\t.num_pmus = ARRAY_SIZE({tblname})
778\t\t}},
779\t\t.name = \"{tblname}\",
780\t}},
781""")
782  _args.output_file.write("""\t{
783\t\t.event_table = { 0, 0 },
784\t\t.metric_table = { 0, 0 },
785\t},
786};
787
788static void decompress_event(int offset, struct pmu_event *pe)
789{
790\tconst char *p = &big_c_string[offset];
791""")
792  for attr in _json_event_attributes:
793    _args.output_file.write(f'\n\tpe->{attr} = ')
794    if attr in _json_enum_attributes:
795      _args.output_file.write("*p - '0';\n")
796    else:
797      _args.output_file.write("(*p == '\\0' ? NULL : p);\n")
798    if attr == _json_event_attributes[-1]:
799      continue
800    if attr in _json_enum_attributes:
801      _args.output_file.write('\tp++;')
802    else:
803      _args.output_file.write('\twhile (*p++);')
804  _args.output_file.write("""}
805
806static void decompress_metric(int offset, struct pmu_metric *pm)
807{
808\tconst char *p = &big_c_string[offset];
809""")
810  for attr in _json_metric_attributes:
811    _args.output_file.write(f'\n\tpm->{attr} = ')
812    if attr in _json_enum_attributes:
813      _args.output_file.write("*p - '0';\n")
814    else:
815      _args.output_file.write("(*p == '\\0' ? NULL : p);\n")
816    if attr == _json_metric_attributes[-1]:
817      continue
818    if attr in _json_enum_attributes:
819      _args.output_file.write('\tp++;')
820    else:
821      _args.output_file.write('\twhile (*p++);')
822  _args.output_file.write("""}
823
824static int pmu_events_table__for_each_event_pmu(const struct pmu_events_table *table,
825                                                const struct pmu_table_entry *pmu,
826                                                pmu_event_iter_fn fn,
827                                                void *data)
828{
829        int ret;
830        struct pmu_event pe = {
831                .pmu = &big_c_string[pmu->pmu_name.offset],
832        };
833
834        for (uint32_t i = 0; i < pmu->num_entries; i++) {
835                decompress_event(pmu->entries[i].offset, &pe);
836                if (!pe.name)
837                        continue;
838                ret = fn(&pe, table, data);
839                if (ret)
840                        return ret;
841        }
842        return 0;
843 }
844
845static int pmu_events_table__find_event_pmu(const struct pmu_events_table *table,
846                                            const struct pmu_table_entry *pmu,
847                                            const char *name,
848                                            pmu_event_iter_fn fn,
849                                            void *data)
850{
851        struct pmu_event pe = {
852                .pmu = &big_c_string[pmu->pmu_name.offset],
853        };
854        int low = 0, high = pmu->num_entries - 1;
855
856        while (low <= high) {
857                int cmp, mid = (low + high) / 2;
858
859                decompress_event(pmu->entries[mid].offset, &pe);
860
861                if (!pe.name && !name)
862                        goto do_call;
863
864                if (!pe.name && name) {
865                        low = mid + 1;
866                        continue;
867                }
868                if (pe.name && !name) {
869                        high = mid - 1;
870                        continue;
871                }
872
873                cmp = strcasecmp(pe.name, name);
874                if (cmp < 0) {
875                        low = mid + 1;
876                        continue;
877                }
878                if (cmp > 0) {
879                        high = mid - 1;
880                        continue;
881                }
882  do_call:
883                return fn ? fn(&pe, table, data) : 0;
884        }
885        return -1000;
886}
887
888int pmu_events_table__for_each_event(const struct pmu_events_table *table,
889                                    struct perf_pmu *pmu,
890                                    pmu_event_iter_fn fn,
891                                    void *data)
892{
893        for (size_t i = 0; i < table->num_pmus; i++) {
894                const struct pmu_table_entry *table_pmu = &table->pmus[i];
895                const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
896                int ret;
897
898                if (pmu && !pmu__name_match(pmu, pmu_name))
899                        continue;
900
901                ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data);
902                if (pmu || ret)
903                        return ret;
904        }
905        return 0;
906}
907
908int pmu_events_table__find_event(const struct pmu_events_table *table,
909                                 struct perf_pmu *pmu,
910                                 const char *name,
911                                 pmu_event_iter_fn fn,
912                                 void *data)
913{
914        for (size_t i = 0; i < table->num_pmus; i++) {
915                const struct pmu_table_entry *table_pmu = &table->pmus[i];
916                const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
917                int ret;
918
919                if (!pmu__name_match(pmu, pmu_name))
920                        continue;
921
922                ret = pmu_events_table__find_event_pmu(table, table_pmu, name, fn, data);
923                if (ret != -1000)
924                        return ret;
925        }
926        return -1000;
927}
928
929size_t pmu_events_table__num_events(const struct pmu_events_table *table,
930                                    struct perf_pmu *pmu)
931{
932        size_t count = 0;
933
934        for (size_t i = 0; i < table->num_pmus; i++) {
935                const struct pmu_table_entry *table_pmu = &table->pmus[i];
936                const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
937
938                if (pmu__name_match(pmu, pmu_name))
939                        count += table_pmu->num_entries;
940        }
941        return count;
942}
943
944static int pmu_metrics_table__for_each_metric_pmu(const struct pmu_metrics_table *table,
945                                                const struct pmu_table_entry *pmu,
946                                                pmu_metric_iter_fn fn,
947                                                void *data)
948{
949        int ret;
950        struct pmu_metric pm = {
951                .pmu = &big_c_string[pmu->pmu_name.offset],
952        };
953
954        for (uint32_t i = 0; i < pmu->num_entries; i++) {
955                decompress_metric(pmu->entries[i].offset, &pm);
956                if (!pm.metric_expr)
957                        continue;
958                ret = fn(&pm, table, data);
959                if (ret)
960                        return ret;
961        }
962        return 0;
963}
964
965int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table,
966                                     pmu_metric_iter_fn fn,
967                                     void *data)
968{
969        for (size_t i = 0; i < table->num_pmus; i++) {
970                int ret = pmu_metrics_table__for_each_metric_pmu(table, &table->pmus[i],
971                                                                 fn, data);
972
973                if (ret)
974                        return ret;
975        }
976        return 0;
977}
978
979static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu)
980{
981        static struct {
982                const struct pmu_events_map *map;
983                struct perf_pmu *pmu;
984        } last_result;
985        static struct {
986                const struct pmu_events_map *map;
987                char *cpuid;
988        } last_map_search;
989        static bool has_last_result, has_last_map_search;
990        const struct pmu_events_map *map = NULL;
991        char *cpuid = NULL;
992        size_t i;
993
994        if (has_last_result && last_result.pmu == pmu)
995                return last_result.map;
996
997        cpuid = perf_pmu__getcpuid(pmu);
998
999        /*
1000         * On some platforms which uses cpus map, cpuid can be NULL for
1001         * PMUs other than CORE PMUs.
1002         */
1003        if (!cpuid)
1004                goto out_update_last_result;
1005
1006        if (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) {
1007                map = last_map_search.map;
1008                free(cpuid);
1009        } else {
1010                i = 0;
1011                for (;;) {
1012                        map = &pmu_events_map[i++];
1013
1014                        if (!map->arch) {
1015                                map = NULL;
1016                                break;
1017                        }
1018
1019                        if (!strcmp_cpuid_str(map->cpuid, cpuid))
1020                                break;
1021               }
1022               free(last_map_search.cpuid);
1023               last_map_search.cpuid = cpuid;
1024               last_map_search.map = map;
1025               has_last_map_search = true;
1026        }
1027out_update_last_result:
1028        last_result.pmu = pmu;
1029        last_result.map = map;
1030        has_last_result = true;
1031        return map;
1032}
1033
1034const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu)
1035{
1036        const struct pmu_events_map *map = map_for_pmu(pmu);
1037
1038        if (!map)
1039                return NULL;
1040
1041        if (!pmu)
1042                return &map->event_table;
1043
1044        for (size_t i = 0; i < map->event_table.num_pmus; i++) {
1045                const struct pmu_table_entry *table_pmu = &map->event_table.pmus[i];
1046                const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
1047
1048                if (pmu__name_match(pmu, pmu_name))
1049                         return &map->event_table;
1050        }
1051        return NULL;
1052}
1053
1054const struct pmu_metrics_table *perf_pmu__find_metrics_table(struct perf_pmu *pmu)
1055{
1056        const struct pmu_events_map *map = map_for_pmu(pmu);
1057
1058        if (!map)
1059                return NULL;
1060
1061        if (!pmu)
1062                return &map->metric_table;
1063
1064        for (size_t i = 0; i < map->metric_table.num_pmus; i++) {
1065                const struct pmu_table_entry *table_pmu = &map->metric_table.pmus[i];
1066                const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset];
1067
1068                if (pmu__name_match(pmu, pmu_name))
1069                           return &map->metric_table;
1070        }
1071        return NULL;
1072}
1073
1074const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid)
1075{
1076        for (const struct pmu_events_map *tables = &pmu_events_map[0];
1077             tables->arch;
1078             tables++) {
1079                if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid))
1080                        return &tables->event_table;
1081        }
1082        return NULL;
1083}
1084
1085const struct pmu_metrics_table *find_core_metrics_table(const char *arch, const char *cpuid)
1086{
1087        for (const struct pmu_events_map *tables = &pmu_events_map[0];
1088             tables->arch;
1089             tables++) {
1090                if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid))
1091                        return &tables->metric_table;
1092        }
1093        return NULL;
1094}
1095
1096int pmu_for_each_core_event(pmu_event_iter_fn fn, void *data)
1097{
1098        for (const struct pmu_events_map *tables = &pmu_events_map[0];
1099             tables->arch;
1100             tables++) {
1101                int ret = pmu_events_table__for_each_event(&tables->event_table,
1102                                                           /*pmu=*/ NULL, fn, data);
1103
1104                if (ret)
1105                        return ret;
1106        }
1107        return 0;
1108}
1109
1110int pmu_for_each_core_metric(pmu_metric_iter_fn fn, void *data)
1111{
1112        for (const struct pmu_events_map *tables = &pmu_events_map[0];
1113             tables->arch;
1114             tables++) {
1115                int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data);
1116
1117                if (ret)
1118                        return ret;
1119        }
1120        return 0;
1121}
1122
1123const struct pmu_events_table *find_sys_events_table(const char *name)
1124{
1125        for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
1126             tables->name;
1127             tables++) {
1128                if (!strcmp(tables->name, name))
1129                        return &tables->event_table;
1130        }
1131        return NULL;
1132}
1133
1134int pmu_for_each_sys_event(pmu_event_iter_fn fn, void *data)
1135{
1136        for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
1137             tables->name;
1138             tables++) {
1139                int ret = pmu_events_table__for_each_event(&tables->event_table,
1140                                                           /*pmu=*/ NULL, fn, data);
1141
1142                if (ret)
1143                        return ret;
1144        }
1145        return 0;
1146}
1147
1148int pmu_for_each_sys_metric(pmu_metric_iter_fn fn, void *data)
1149{
1150        for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0];
1151             tables->name;
1152             tables++) {
1153                int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data);
1154
1155                if (ret)
1156                        return ret;
1157        }
1158        return 0;
1159}
1160""")
1161
1162def print_metricgroups() -> None:
1163  _args.output_file.write("""
1164static const int metricgroups[][2] = {
1165""")
1166  for mgroup in sorted(_metricgroups):
1167    description = _metricgroups[mgroup]
1168    _args.output_file.write(
1169        f'\t{{ {_bcs.offsets[mgroup]}, {_bcs.offsets[description]} }}, /* {mgroup} => {description} */\n'
1170    )
1171  _args.output_file.write("""
1172};
1173
1174const char *describe_metricgroup(const char *group)
1175{
1176        int low = 0, high = (int)ARRAY_SIZE(metricgroups) - 1;
1177
1178        while (low <= high) {
1179                int mid = (low + high) / 2;
1180                const char *mgroup = &big_c_string[metricgroups[mid][0]];
1181                int cmp = strcmp(mgroup, group);
1182
1183                if (cmp == 0) {
1184                        return &big_c_string[metricgroups[mid][1]];
1185                } else if (cmp < 0) {
1186                        low = mid + 1;
1187                } else {
1188                        high = mid - 1;
1189                }
1190        }
1191        return NULL;
1192}
1193""")
1194
1195def main() -> None:
1196  global _args
1197
1198  def dir_path(path: str) -> str:
1199    """Validate path is a directory for argparse."""
1200    if os.path.isdir(path):
1201      return path
1202    raise argparse.ArgumentTypeError(f'\'{path}\' is not a valid directory')
1203
1204  def ftw(path: str, parents: Sequence[str],
1205          action: Callable[[Sequence[str], os.DirEntry], None]) -> None:
1206    """Replicate the directory/file walking behavior of C's file tree walk."""
1207    for item in sorted(os.scandir(path), key=lambda e: e.name):
1208      if _args.model != 'all' and item.is_dir():
1209        # Check if the model matches one in _args.model.
1210        if len(parents) == _args.model.split(',')[0].count('/'):
1211          # We're testing the correct directory.
1212          item_path = '/'.join(parents) + ('/' if len(parents) > 0 else '') + item.name
1213          if 'test' not in item_path and item_path not in _args.model.split(','):
1214            continue
1215      action(parents, item)
1216      if item.is_dir():
1217        ftw(item.path, parents + [item.name], action)
1218
1219  ap = argparse.ArgumentParser()
1220  ap.add_argument('arch', help='Architecture name like x86')
1221  ap.add_argument('model', help='''Select a model such as skylake to
1222reduce the code size.  Normally set to "all". For architectures like
1223ARM64 with an implementor/model, the model must include the implementor
1224such as "arm/cortex-a34".''',
1225                  default='all')
1226  ap.add_argument(
1227      'starting_dir',
1228      type=dir_path,
1229      help='Root of tree containing architecture directories containing json files'
1230  )
1231  ap.add_argument(
1232      'output_file', type=argparse.FileType('w', encoding='utf-8'), nargs='?', default=sys.stdout)
1233  _args = ap.parse_args()
1234
1235  _args.output_file.write("""
1236#include <pmu-events/pmu-events.h>
1237#include "util/header.h"
1238#include "util/pmu.h"
1239#include <string.h>
1240#include <stddef.h>
1241
1242struct compact_pmu_event {
1243        int offset;
1244};
1245
1246struct pmu_table_entry {
1247        const struct compact_pmu_event *entries;
1248        uint32_t num_entries;
1249        struct compact_pmu_event pmu_name;
1250};
1251
1252""")
1253  archs = []
1254  for item in os.scandir(_args.starting_dir):
1255    if not item.is_dir():
1256      continue
1257    if item.name == _args.arch or _args.arch == 'all' or item.name == 'test':
1258      archs.append(item.name)
1259
1260  if len(archs) < 2:
1261    raise IOError(f'Missing architecture directory \'{_args.arch}\'')
1262
1263  archs.sort()
1264  for arch in archs:
1265    arch_path = f'{_args.starting_dir}/{arch}'
1266    preprocess_arch_std_files(arch_path)
1267    ftw(arch_path, [], preprocess_one_file)
1268
1269  _bcs.compute()
1270  _args.output_file.write('static const char *const big_c_string =\n')
1271  for s in _bcs.big_string:
1272    _args.output_file.write(s)
1273  _args.output_file.write(';\n\n')
1274  for arch in archs:
1275    arch_path = f'{_args.starting_dir}/{arch}'
1276    ftw(arch_path, [], process_one_file)
1277    print_pending_events()
1278    print_pending_metrics()
1279
1280  print_mapping_table(archs)
1281  print_system_mapping_table()
1282  print_metricgroups()
1283
1284if __name__ == '__main__':
1285  main()
1286