xref: /linux/scripts/generate_rust_analyzer.py (revision 7e9535ebd05d7e8de155164b7c97a370d4646e06) 
1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3"""generate_rust_analyzer - Generates the `rust-project.json` file for `rust-analyzer`.
4"""
5
6import argparse
7import json
8import logging
9import os
10import pathlib
11import subprocess
12import sys
13from typing import Dict, Iterable, List, Literal, Optional, TypedDict
14
15def invoke_rustc(args: List[str]) -> str:
16    return subprocess.check_output(
17        [os.environ["RUSTC"]] + args,
18        stdin=subprocess.DEVNULL,
19    ).decode('utf-8').strip()
20
21def args_crates_cfgs(cfgs: List[str]) -> Dict[str, List[str]]:
22    crates_cfgs = {}
23    for cfg in cfgs:
24        crate, vals = cfg.split("=", 1)
25        crates_cfgs[crate] = vals.split()
26
27    return crates_cfgs
28
29class Dependency(TypedDict):
30    crate: int
31    name: str
32
33
34class Source(TypedDict):
35    include_dirs: List[str]
36    exclude_dirs: List[str]
37
38
39class Crate(TypedDict):
40    display_name: str
41    root_module: str
42    is_workspace_member: bool
43    deps: List[Dependency]
44    cfg: List[str]
45    edition: str
46    env: Dict[str, str]
47
48
49class ProcMacroCrate(Crate):
50    is_proc_macro: Literal[True]
51    proc_macro_dylib_path: str  # `pathlib.Path` is not JSON serializable.
52
53
54class CrateWithGenerated(Crate):
55    source: Source
56
57
58def generate_crates(
59    srctree: pathlib.Path,
60    objtree: pathlib.Path,
61    sysroot_src: pathlib.Path,
62    external_src: Optional[pathlib.Path],
63    cfgs: List[str],
64    core_edition: str,
65) -> List[Crate]:
66    # Generate the configuration list.
67    generated_cfg = []
68    with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
69        for line in fd:
70            line = line.replace("--cfg=", "")
71            line = line.replace("\n", "")
72            generated_cfg.append(line)
73
74    # Now fill the crates list.
75    crates: List[Crate] = []
76    crates_cfgs = args_crates_cfgs(cfgs)
77
78    def get_crate_name(path: pathlib.Path) -> str:
79        return invoke_rustc(["--print", "crate-name", str(path)])
80
81    def build_crate(
82        display_name: str,
83        root_module: pathlib.Path,
84        deps: List[Dependency],
85        *,
86        cfg: Optional[List[str]],
87        is_workspace_member: Optional[bool],
88        edition: Optional[str],
89    ) -> Crate:
90        cfg = cfg if cfg is not None else crates_cfgs.get(display_name, [])
91        is_workspace_member = (
92            is_workspace_member if is_workspace_member is not None else True
93        )
94        edition = edition if edition is not None else "2021"
95        return {
96            "display_name": display_name,
97            "root_module": str(root_module),
98            "is_workspace_member": is_workspace_member,
99            "deps": deps,
100            "cfg": cfg,
101            "edition": edition,
102            "env": {
103                "RUST_MODFILE": "This is only for rust-analyzer"
104            }
105        }
106
107    def append_proc_macro_crate(
108        display_name: str,
109        root_module: pathlib.Path,
110        deps: List[Dependency],
111        *,
112        cfg: Optional[List[str]] = None,
113        is_workspace_member: Optional[bool] = None,
114        edition: Optional[str] = None,
115    ) -> Dependency:
116        crate = build_crate(
117            display_name,
118            root_module,
119            deps,
120            cfg=cfg,
121            is_workspace_member=is_workspace_member,
122            edition=edition,
123        )
124        proc_macro_dylib_name = invoke_rustc([
125            "--print",
126            "file-names",
127            "--crate-name",
128            display_name,
129            "--crate-type",
130            "proc-macro",
131            "-",
132        ])
133        proc_macro_crate: ProcMacroCrate = {
134            **crate,
135            "is_proc_macro": True,
136            "proc_macro_dylib_path": str(objtree / "rust" / proc_macro_dylib_name),
137        }
138        return register_crate(proc_macro_crate)
139
140    def register_crate(crate: Crate) -> Dependency:
141        index = len(crates)
142        crates.append(crate)
143        return {"crate": index, "name": crate["display_name"]}
144
145    def append_crate(
146        display_name: str,
147        root_module: pathlib.Path,
148        deps: List[Dependency],
149        *,
150        cfg: Optional[List[str]] = None,
151        is_workspace_member: Optional[bool] = None,
152        edition: Optional[str] = None,
153    ) -> Dependency:
154        return register_crate(
155            build_crate(
156                display_name,
157                root_module,
158                deps,
159                cfg=cfg,
160                is_workspace_member=is_workspace_member,
161                edition=edition,
162            )
163        )
164
165    def append_sysroot_crate(
166        display_name: str,
167        deps: List[Dependency],
168        *,
169        cfg: Optional[List[str]] = None,
170    ) -> Dependency:
171        return append_crate(
172            display_name,
173            sysroot_src / display_name / "src" / "lib.rs",
174            deps,
175            cfg=cfg,
176            is_workspace_member=False,
177            # Miguel Ojeda writes:
178            #
179            # > ... in principle even the sysroot crates may have different
180            # > editions.
181            # >
182            # > For instance, in the move to 2024, it seems all happened at once
183            # > in 1.87.0 in these upstream commits:
184            # >
185            # >     0e071c2c6a58 ("Migrate core to Rust 2024")
186            # >     f505d4e8e380 ("Migrate alloc to Rust 2024")
187            # >     0b2489c226c3 ("Migrate proc_macro to Rust 2024")
188            # >     993359e70112 ("Migrate std to Rust 2024")
189            # >
190            # > But in the previous move to 2021, `std` moved in 1.59.0, while
191            # > the others in 1.60.0:
192            # >
193            # >     b656384d8398 ("Update stdlib to the 2021 edition")
194            # >     06a1c14d52a8 ("Switch all libraries to the 2021 edition")
195            #
196            # Link: https://lore.kernel.org/all/CANiq72kd9bHdKaAm=8xCUhSHMy2csyVed69bOc4dXyFAW4sfuw@mail.gmail.com/
197            #
198            # At the time of writing all rust versions we support build the
199            # sysroot crates with the same edition. We may need to relax this
200            # assumption if future edition moves span multiple rust versions.
201            edition=core_edition,
202        )
203
204    # NB: sysroot crates reexport items from one another so setting up our transitive dependencies
205    # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth
206    # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`.
207    core = append_sysroot_crate("core", [])
208    alloc = append_sysroot_crate("alloc", [core])
209    std = append_sysroot_crate("std", [alloc, core])
210    proc_macro = append_sysroot_crate("proc_macro", [core, std])
211
212    compiler_builtins = append_crate(
213        "compiler_builtins",
214        srctree / "rust" / "compiler_builtins.rs",
215        [core],
216    )
217
218    proc_macro2 = append_crate(
219        "proc_macro2",
220        srctree / "rust" / "proc-macro2" / "lib.rs",
221        [core, alloc, std, proc_macro],
222    )
223
224    quote = append_crate(
225        "quote",
226        srctree / "rust" / "quote" / "lib.rs",
227        [core, alloc, std, proc_macro, proc_macro2],
228        edition="2018",
229    )
230
231    syn = append_crate(
232        "syn",
233        srctree / "rust" / "syn" / "lib.rs",
234        [std, proc_macro, proc_macro2, quote],
235    )
236
237    macros = append_proc_macro_crate(
238        "macros",
239        srctree / "rust" / "macros" / "lib.rs",
240        [std, proc_macro, proc_macro2, quote, syn],
241    )
242
243    build_error = append_crate(
244        "build_error",
245        srctree / "rust" / "build_error.rs",
246        [core, compiler_builtins],
247    )
248
249    pin_init_internal = append_proc_macro_crate(
250        "pin_init_internal",
251        srctree / "rust" / "pin-init" / "internal" / "src" / "lib.rs",
252        [std, proc_macro, proc_macro2, quote, syn],
253    )
254
255    pin_init = append_crate(
256        "pin_init",
257        srctree / "rust" / "pin-init" / "src" / "lib.rs",
258        [core, compiler_builtins, pin_init_internal, macros],
259    )
260
261    ffi = append_crate(
262        "ffi",
263        srctree / "rust" / "ffi.rs",
264        [core, compiler_builtins],
265    )
266
267    def append_crate_with_generated(
268        display_name: str,
269        deps: List[Dependency],
270    ) -> Dependency:
271        crate = build_crate(
272            display_name,
273            srctree / "rust"/ display_name / "lib.rs",
274            deps,
275            cfg=generated_cfg,
276            is_workspace_member=True,
277            edition=None,
278        )
279        crate["env"]["OBJTREE"] = str(objtree.resolve(True))
280        crate_with_generated: CrateWithGenerated = {
281            **crate,
282            "source": {
283                "include_dirs": [
284                    str(srctree / "rust" / display_name),
285                    str(objtree / "rust"),
286                ],
287                "exclude_dirs": [],
288            },
289        }
290        return register_crate(crate_with_generated)
291
292    bindings = append_crate_with_generated("bindings", [core, ffi, pin_init])
293    uapi = append_crate_with_generated("uapi", [core, ffi, pin_init])
294    kernel = append_crate_with_generated(
295        "kernel", [core, macros, build_error, pin_init, ffi, bindings, uapi]
296    )
297
298    scripts = srctree / "scripts"
299    makefile = (scripts / "Makefile").read_text()
300    for path in scripts.glob("*.rs"):
301        name = path.stem
302        if f"{name}-rust" not in makefile:
303            continue
304        append_crate(
305            name,
306            path,
307            [std],
308        )
309
310    def is_root_crate(build_file: pathlib.Path, target: str) -> bool:
311        try:
312            contents = build_file.read_text()
313        except FileNotFoundError:
314            return False
315        return f"{target}.o" in contents
316
317    # Then, the rest outside of `rust/`.
318    #
319    # We explicitly mention the top-level folders we want to cover.
320    extra_dirs: Iterable[pathlib.Path] = (
321        srctree / dir for dir in ("samples", "drivers")
322    )
323    if external_src is not None:
324        extra_dirs = [external_src]
325    for folder in extra_dirs:
326        for path in folder.rglob("*.rs"):
327            logging.info("Checking %s", path)
328            file_name = path.stem
329
330            # Skip those that are not crate roots.
331            if not is_root_crate(path.parent / "Makefile", file_name) and \
332               not is_root_crate(path.parent / "Kbuild", file_name):
333                continue
334
335            crate_name = get_crate_name(path)
336            logging.info("Adding %s", crate_name)
337            append_crate(
338                crate_name,
339                path,
340                [core, kernel, pin_init],
341                cfg=generated_cfg,
342            )
343
344    return crates
345
346def main() -> None:
347    parser = argparse.ArgumentParser()
348    parser.add_argument('--verbose', '-v', action='store_true')
349    parser.add_argument('--cfgs', action='append', default=[])
350    parser.add_argument("core_edition")
351    parser.add_argument("srctree", type=pathlib.Path)
352    parser.add_argument("objtree", type=pathlib.Path)
353    parser.add_argument("sysroot", type=pathlib.Path)
354    parser.add_argument("sysroot_src", type=pathlib.Path)
355    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
356
357    class Args(argparse.Namespace):
358        verbose: bool
359        cfgs: List[str]
360        srctree: pathlib.Path
361        objtree: pathlib.Path
362        sysroot: pathlib.Path
363        sysroot_src: pathlib.Path
364        exttree: Optional[pathlib.Path]
365        core_edition: str
366
367    args = parser.parse_args(namespace=Args())
368
369    logging.basicConfig(
370        format="[%(asctime)s] [%(levelname)s] %(message)s",
371        level=logging.INFO if args.verbose else logging.WARNING
372    )
373
374    rust_project = {
375        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.core_edition),
376        "sysroot": str(args.sysroot),
377    }
378
379    json.dump(rust_project, sys.stdout, sort_keys=True, indent=4)
380
381if __name__ == "__main__":
382    main()
383