1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# Copyright (C) Google LLC, 2018 5# 6# Author: Tom Roeder <tmroeder@google.com> 7# 8"""A tool for generating compile_commands.json in the Linux kernel.""" 9 10import argparse 11import json 12import logging 13import os 14import re 15import subprocess 16 17_DEFAULT_OUTPUT = 'compile_commands.json' 18_DEFAULT_LOG_LEVEL = 'WARNING' 19 20_FILENAME_PATTERN = r'^\..*\.cmd$' 21_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' 22_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] 23# The tools/ directory adopts a different build system, and produces .cmd 24# files in a different format. Do not support it. 25_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools'] 26 27def parse_arguments(): 28 """Sets up and parses command-line arguments. 29 30 Returns: 31 log_level: A logging level to filter log output. 32 directory: The work directory where the objects were built. 33 ar: Command used for parsing .a archives. 34 output: Where to write the compile-commands JSON file. 35 paths: The list of files/directories to handle to find .cmd files. 36 """ 37 usage = 'Creates a compile_commands.json database from kernel .cmd files' 38 parser = argparse.ArgumentParser(description=usage) 39 40 directory_help = ('specify the output directory used for the kernel build ' 41 '(defaults to the working directory)') 42 parser.add_argument('-d', '--directory', type=str, default='.', 43 help=directory_help) 44 45 output_help = ('path to the output command database (defaults to ' + 46 _DEFAULT_OUTPUT + ')') 47 parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, 48 help=output_help) 49 50 log_level_help = ('the level of log messages to produce (defaults to ' + 51 _DEFAULT_LOG_LEVEL + ')') 52 parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, 53 default=_DEFAULT_LOG_LEVEL, help=log_level_help) 54 55 ar_help = 'command used for parsing .a archives' 56 parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) 57 58 paths_help = ('directories to search or files to parse ' 59 '(files should be *.o, *.a, or modules.order). ' 60 'If nothing is specified, the current directory is searched') 61 parser.add_argument('paths', type=str, nargs='*', help=paths_help) 62 63 args = parser.parse_args() 64 65 return (args.log_level, 66 os.path.abspath(args.directory), 67 args.output, 68 args.ar, 69 args.paths if len(args.paths) > 0 else [args.directory]) 70 71 72def cmdfiles_in_dir(directory): 73 """Generate the iterator of .cmd files found under the directory. 74 75 Walk under the given directory, and yield every .cmd file found. 76 77 Args: 78 directory: The directory to search for .cmd files. 79 80 Yields: 81 The path to a .cmd file. 82 """ 83 84 filename_matcher = re.compile(_FILENAME_PATTERN) 85 exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ] 86 87 for dirpath, dirnames, filenames in os.walk(directory, topdown=True): 88 # Prune unwanted directories. 89 if dirpath in exclude_dirs: 90 dirnames[:] = [] 91 continue 92 93 for filename in filenames: 94 if filename_matcher.match(filename): 95 yield os.path.join(dirpath, filename) 96 97 98def to_cmdfile(path): 99 """Return the path of .cmd file used for the given build artifact 100 101 Args: 102 Path: file path 103 104 Returns: 105 The path to .cmd file 106 """ 107 dir, base = os.path.split(path) 108 return os.path.join(dir, '.' + base + '.cmd') 109 110 111def cmdfiles_for_o(obj): 112 """Generate the iterator of .cmd files associated with the object 113 114 Yield the .cmd file used to build the given object 115 116 Args: 117 obj: The object path 118 119 Yields: 120 The path to .cmd file 121 """ 122 yield to_cmdfile(obj) 123 124 125def cmdfiles_for_a(archive, ar): 126 """Generate the iterator of .cmd files associated with the archive. 127 128 Parse the given archive, and yield every .cmd file used to build it. 129 130 Args: 131 archive: The archive to parse 132 133 Yields: 134 The path to every .cmd file found 135 """ 136 for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): 137 yield to_cmdfile(obj) 138 139 140def cmdfiles_for_modorder(modorder): 141 """Generate the iterator of .cmd files associated with the modules.order. 142 143 Parse the given modules.order, and yield every .cmd file used to build the 144 contained modules. 145 146 Args: 147 modorder: The modules.order file to parse 148 149 Yields: 150 The path to every .cmd file found 151 """ 152 with open(modorder) as f: 153 for line in f: 154 ko = line.rstrip() 155 base, ext = os.path.splitext(ko) 156 if ext != '.ko': 157 sys.exit('{}: module path must end with .ko'.format(ko)) 158 mod = base + '.mod' 159 # The first line of *.mod lists the objects that compose the module. 160 with open(mod) as m: 161 for obj in m.readline().split(): 162 yield to_cmdfile(obj) 163 164 165def process_line(root_directory, command_prefix, file_path): 166 """Extracts information from a .cmd line and creates an entry from it. 167 168 Args: 169 root_directory: The directory that was searched for .cmd files. Usually 170 used directly in the "directory" entry in compile_commands.json. 171 command_prefix: The extracted command line, up to the last element. 172 file_path: The .c file from the end of the extracted command. 173 Usually relative to root_directory, but sometimes absolute. 174 175 Returns: 176 An entry to append to compile_commands. 177 178 Raises: 179 ValueError: Could not find the extracted file based on file_path and 180 root_directory or file_directory. 181 """ 182 # The .cmd files are intended to be included directly by Make, so they 183 # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the 184 # kernel version). The compile_commands.json file is not interepreted 185 # by Make, so this code replaces the escaped version with '#'. 186 prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') 187 188 # Use os.path.abspath() to normalize the path resolving '.' and '..' . 189 abs_path = os.path.abspath(os.path.join(root_directory, file_path)) 190 if not os.path.exists(abs_path): 191 raise ValueError('File %s not found' % abs_path) 192 return { 193 'directory': root_directory, 194 'file': abs_path, 195 'command': prefix + file_path, 196 } 197 198 199def main(): 200 """Walks through the directory and finds and parses .cmd files.""" 201 log_level, directory, output, ar, paths = parse_arguments() 202 203 level = getattr(logging, log_level) 204 logging.basicConfig(format='%(levelname)s: %(message)s', level=level) 205 206 line_matcher = re.compile(_LINE_PATTERN) 207 208 compile_commands = [] 209 210 for path in paths: 211 # If 'path' is a directory, handle all .cmd files under it. 212 # Otherwise, handle .cmd files associated with the file. 213 # Most of built-in objects are linked via archives (built-in.a or lib.a) 214 # but some objects are linked to vmlinux directly. 215 # Modules are listed in modules.order. 216 if os.path.isdir(path): 217 cmdfiles = cmdfiles_in_dir(path) 218 elif path.endswith('.o'): 219 cmdfiles = cmdfiles_for_o(path) 220 elif path.endswith('.a'): 221 cmdfiles = cmdfiles_for_a(path, ar) 222 elif path.endswith('modules.order'): 223 cmdfiles = cmdfiles_for_modorder(path) 224 else: 225 sys.exit('{}: unknown file type'.format(path)) 226 227 for cmdfile in cmdfiles: 228 with open(cmdfile, 'rt') as f: 229 result = line_matcher.match(f.readline()) 230 if result: 231 try: 232 entry = process_line(directory, result.group(1), 233 result.group(2)) 234 compile_commands.append(entry) 235 except ValueError as err: 236 logging.info('Could not add line from %s: %s', 237 cmdfile, err) 238 239 with open(output, 'wt') as f: 240 json.dump(compile_commands, f, indent=2, sort_keys=True) 241 242 243if __name__ == '__main__': 244 main() 245