1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3"""generate_rust_analyzer - Generates the `rust-project.json` file for `rust-analyzer`. 4""" 5 6import argparse 7import json 8import logging 9import os 10import pathlib 11import subprocess 12import sys 13from typing import Dict, Iterable, List, Literal, Optional, TypedDict 14 15def invoke_rustc(args: List[str]) -> str: 16 return subprocess.check_output( 17 [os.environ["RUSTC"]] + args, 18 stdin=subprocess.DEVNULL, 19 ).decode('utf-8').strip() 20 21def args_crates_cfgs(cfgs: List[str]) -> Dict[str, List[str]]: 22 crates_cfgs = {} 23 for cfg in cfgs: 24 crate, vals = cfg.split("=", 1) 25 crates_cfgs[crate] = vals.split() 26 27 return crates_cfgs 28 29def args_crates_envs(envs: List[str]) -> Dict[str, Dict[str, str]]: 30 crates_envs = {} 31 for env in envs: 32 crate, vals = env.split("=", 1) 33 crates_envs[crate] = dict(v.split("=", 1) for v in vals.split()) 34 35 return crates_envs 36 37class Dependency(TypedDict): 38 crate: int 39 name: str 40 41 42class Source(TypedDict): 43 include_dirs: List[str] 44 exclude_dirs: List[str] 45 46 47class Crate(TypedDict): 48 display_name: str 49 root_module: str 50 is_workspace_member: bool 51 deps: List[Dependency] 52 cfg: List[str] 53 edition: str 54 env: Dict[str, str] 55 56 57class ProcMacroCrate(Crate): 58 is_proc_macro: Literal[True] 59 proc_macro_dylib_path: str # `pathlib.Path` is not JSON serializable. 60 61 62class CrateWithGenerated(Crate): 63 source: Source 64 65 66def generate_crates( 67 srctree: pathlib.Path, 68 objtree: pathlib.Path, 69 sysroot_src: pathlib.Path, 70 external_src: Optional[pathlib.Path], 71 cfgs: List[str], 72 envs: List[str], 73 core_edition: str, 74) -> List[Crate]: 75 # Generate the configuration list. 76 generated_cfg = [] 77 with open(objtree / "include" / "generated" / "rustc_cfg") as fd: 78 for line in fd: 79 line = line.replace("--cfg=", "") 80 line = line.replace("\n", "") 81 generated_cfg.append(line) 82 83 # Now fill the crates list. 84 crates: List[Crate] = [] 85 crates_cfgs = args_crates_cfgs(cfgs) 86 crates_envs = args_crates_envs(envs) 87 88 def get_crate_name(path: pathlib.Path) -> str: 89 return invoke_rustc(["--print", "crate-name", str(path)]) 90 91 def build_crate( 92 display_name: str, 93 root_module: pathlib.Path, 94 deps: List[Dependency], 95 *, 96 cfg: Optional[List[str]], 97 is_workspace_member: Optional[bool], 98 edition: Optional[str], 99 ) -> Crate: 100 cfg = cfg if cfg is not None else crates_cfgs.get(display_name, []) 101 is_workspace_member = ( 102 is_workspace_member if is_workspace_member is not None else True 103 ) 104 edition = edition if edition is not None else "2021" 105 crate_env = { 106 "RUST_MODFILE": "This is only for rust-analyzer", 107 **crates_envs.get(display_name, {}), 108 } 109 return { 110 "display_name": display_name, 111 "root_module": str(root_module), 112 "is_workspace_member": is_workspace_member, 113 "deps": deps, 114 "cfg": cfg, 115 "edition": edition, 116 "env": crate_env, 117 } 118 119 def append_proc_macro_crate( 120 display_name: str, 121 root_module: pathlib.Path, 122 deps: List[Dependency], 123 *, 124 cfg: Optional[List[str]] = None, 125 is_workspace_member: Optional[bool] = None, 126 edition: Optional[str] = None, 127 ) -> Dependency: 128 crate = build_crate( 129 display_name, 130 root_module, 131 deps, 132 cfg=cfg, 133 is_workspace_member=is_workspace_member, 134 edition=edition, 135 ) 136 proc_macro_dylib_name = invoke_rustc([ 137 "--print", 138 "file-names", 139 "--crate-name", 140 display_name, 141 "--crate-type", 142 "proc-macro", 143 "-", 144 ]) 145 proc_macro_crate: ProcMacroCrate = { 146 **crate, 147 "is_proc_macro": True, 148 "proc_macro_dylib_path": str(objtree / "rust" / proc_macro_dylib_name), 149 } 150 return register_crate(proc_macro_crate) 151 152 def register_crate(crate: Crate) -> Dependency: 153 index = len(crates) 154 crates.append(crate) 155 return {"crate": index, "name": crate["display_name"]} 156 157 def append_crate( 158 display_name: str, 159 root_module: pathlib.Path, 160 deps: List[Dependency], 161 *, 162 cfg: Optional[List[str]] = None, 163 is_workspace_member: Optional[bool] = None, 164 edition: Optional[str] = None, 165 ) -> Dependency: 166 return register_crate( 167 build_crate( 168 display_name, 169 root_module, 170 deps, 171 cfg=cfg, 172 is_workspace_member=is_workspace_member, 173 edition=edition, 174 ) 175 ) 176 177 def append_sysroot_crate( 178 display_name: str, 179 deps: List[Dependency], 180 *, 181 cfg: Optional[List[str]] = None, 182 ) -> Dependency: 183 return append_crate( 184 display_name, 185 sysroot_src / display_name / "src" / "lib.rs", 186 deps, 187 cfg=cfg, 188 is_workspace_member=False, 189 # Miguel Ojeda writes: 190 # 191 # > ... in principle even the sysroot crates may have different 192 # > editions. 193 # > 194 # > For instance, in the move to 2024, it seems all happened at once 195 # > in 1.87.0 in these upstream commits: 196 # > 197 # > 0e071c2c6a58 ("Migrate core to Rust 2024") 198 # > f505d4e8e380 ("Migrate alloc to Rust 2024") 199 # > 0b2489c226c3 ("Migrate proc_macro to Rust 2024") 200 # > 993359e70112 ("Migrate std to Rust 2024") 201 # > 202 # > But in the previous move to 2021, `std` moved in 1.59.0, while 203 # > the others in 1.60.0: 204 # > 205 # > b656384d8398 ("Update stdlib to the 2021 edition") 206 # > 06a1c14d52a8 ("Switch all libraries to the 2021 edition") 207 # 208 # Link: https://lore.kernel.org/all/CANiq72kd9bHdKaAm=8xCUhSHMy2csyVed69bOc4dXyFAW4sfuw@mail.gmail.com/ 209 # 210 # At the time of writing all rust versions we support build the 211 # sysroot crates with the same edition. We may need to relax this 212 # assumption if future edition moves span multiple rust versions. 213 edition=core_edition, 214 ) 215 216 # NB: sysroot crates reexport items from one another so setting up our transitive dependencies 217 # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth 218 # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`. 219 core = append_sysroot_crate("core", []) 220 alloc = append_sysroot_crate("alloc", [core]) 221 std = append_sysroot_crate("std", [alloc, core]) 222 proc_macro = append_sysroot_crate("proc_macro", [core, std]) 223 224 compiler_builtins = append_crate( 225 "compiler_builtins", 226 srctree / "rust" / "compiler_builtins.rs", 227 [core], 228 ) 229 230 proc_macro2 = append_crate( 231 "proc_macro2", 232 srctree / "rust" / "proc-macro2" / "lib.rs", 233 [core, alloc, std, proc_macro], 234 ) 235 236 quote = append_crate( 237 "quote", 238 srctree / "rust" / "quote" / "lib.rs", 239 [core, alloc, std, proc_macro, proc_macro2], 240 edition="2018", 241 ) 242 243 syn = append_crate( 244 "syn", 245 srctree / "rust" / "syn" / "lib.rs", 246 [std, proc_macro, proc_macro2, quote], 247 ) 248 249 macros = append_proc_macro_crate( 250 "macros", 251 srctree / "rust" / "macros" / "lib.rs", 252 [std, proc_macro, proc_macro2, quote, syn], 253 ) 254 255 zerocopy_derive = append_proc_macro_crate( 256 "zerocopy_derive", 257 srctree / "rust" / "zerocopy-derive" / "lib.rs", 258 [std, proc_macro, proc_macro2, quote, syn], 259 ) 260 261 build_error = append_crate( 262 "build_error", 263 srctree / "rust" / "build_error.rs", 264 [core, compiler_builtins], 265 ) 266 267 pin_init_internal = append_proc_macro_crate( 268 "pin_init_internal", 269 srctree / "rust" / "pin-init" / "internal" / "src" / "lib.rs", 270 [std, proc_macro, proc_macro2, quote, syn], 271 ) 272 273 pin_init = append_crate( 274 "pin_init", 275 srctree / "rust" / "pin-init" / "src" / "lib.rs", 276 [core, compiler_builtins, pin_init_internal, macros], 277 ) 278 279 ffi = append_crate( 280 "ffi", 281 srctree / "rust" / "ffi.rs", 282 [core, compiler_builtins], 283 ) 284 285 zerocopy = append_crate( 286 "zerocopy", 287 srctree / "rust" / "zerocopy" / "src" / "lib.rs", 288 [core, compiler_builtins], 289 ) 290 291 def append_crate_with_generated( 292 display_name: str, 293 deps: List[Dependency], 294 ) -> Dependency: 295 crate = build_crate( 296 display_name, 297 srctree / "rust"/ display_name / "lib.rs", 298 deps, 299 cfg=generated_cfg, 300 is_workspace_member=True, 301 edition=None, 302 ) 303 crate["env"]["OBJTREE"] = str(objtree.resolve(True)) 304 crate_with_generated: CrateWithGenerated = { 305 **crate, 306 "source": { 307 "include_dirs": [ 308 str(srctree / "rust" / display_name), 309 str(objtree / "rust"), 310 ], 311 "exclude_dirs": [], 312 }, 313 } 314 return register_crate(crate_with_generated) 315 316 bindings = append_crate_with_generated("bindings", [core, ffi, pin_init]) 317 uapi = append_crate_with_generated("uapi", [core, ffi, pin_init]) 318 kernel = append_crate_with_generated( 319 "kernel", [core, macros, build_error, pin_init, ffi, bindings, uapi, zerocopy, zerocopy_derive] 320 ) 321 322 scripts = srctree / "scripts" 323 makefile = (scripts / "Makefile").read_text() 324 for path in scripts.glob("*.rs"): 325 name = path.stem 326 if f"{name}-rust" not in makefile: 327 continue 328 append_crate( 329 name, 330 path, 331 [std], 332 ) 333 334 def is_root_crate(build_file: pathlib.Path, target: str) -> bool: 335 try: 336 contents = build_file.read_text() 337 except FileNotFoundError: 338 return False 339 return f"{target}.o" in contents 340 341 # Then, the rest outside of `rust/`. 342 # 343 # We explicitly mention the top-level folders we want to cover. 344 extra_dirs: Iterable[pathlib.Path] = ( 345 srctree / dir for dir in ("samples", "drivers") 346 ) 347 if external_src is not None: 348 extra_dirs = [external_src] 349 for folder in extra_dirs: 350 for path in folder.rglob("*.rs"): 351 logging.info("Checking %s", path) 352 file_name = path.stem 353 354 # Skip those that are not crate roots. 355 if not is_root_crate(path.parent / "Makefile", file_name) and \ 356 not is_root_crate(path.parent / "Kbuild", file_name): 357 continue 358 359 crate_name = get_crate_name(path) 360 logging.info("Adding %s", crate_name) 361 append_crate( 362 crate_name, 363 path, 364 [core, kernel, pin_init, zerocopy, zerocopy_derive], 365 cfg=generated_cfg, 366 ) 367 368 return crates 369 370def main() -> None: 371 parser = argparse.ArgumentParser() 372 parser.add_argument('--verbose', '-v', action='store_true') 373 parser.add_argument('--cfgs', action='append', default=[]) 374 parser.add_argument('--envs', action='append', default=[]) 375 parser.add_argument("core_edition") 376 parser.add_argument("srctree", type=pathlib.Path) 377 parser.add_argument("objtree", type=pathlib.Path) 378 parser.add_argument("sysroot", type=pathlib.Path) 379 parser.add_argument("sysroot_src", type=pathlib.Path) 380 parser.add_argument("exttree", type=pathlib.Path, nargs="?") 381 382 class Args(argparse.Namespace): 383 verbose: bool 384 cfgs: List[str] 385 envs: List[str] 386 srctree: pathlib.Path 387 objtree: pathlib.Path 388 sysroot: pathlib.Path 389 sysroot_src: pathlib.Path 390 exttree: Optional[pathlib.Path] 391 core_edition: str 392 393 args = parser.parse_args(namespace=Args()) 394 395 logging.basicConfig( 396 format="[%(asctime)s] [%(levelname)s] %(message)s", 397 level=logging.INFO if args.verbose else logging.WARNING 398 ) 399 400 rust_project = { 401 "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.envs, args.core_edition), 402 "sysroot": str(args.sysroot), 403 } 404 405 json.dump(rust_project, sys.stdout, sort_keys=True, indent=4) 406 407if __name__ == "__main__": 408 main() 409