Merge pull request #18116 from github/redsun82/rust-perf-measures

Rust: add some performance diagnostics

MODULE.bazel

@@ -68,7 +68,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2.
 # deps for ruby+rust
 # keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
 tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
-use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__dunce-1.0.5", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
+use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__dunce-1.0.5", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.133", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
 
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "9.0.100")

misc/codegen/generators/dbschemegen.py

@@ -110,7 +110,8 @@ def cls_to_dbscheme(cls: schema.Class, lookup: typing.Dict[str, schema.Class], a
 
 def get_declarations(data: schema.Schema):
     add_or_none_except = data.root_class.name if data.null else None
-    declarations = [d for cls in data.classes.values() for d in cls_to_dbscheme(cls, data.classes, add_or_none_except)]
+    declarations = [d for cls in data.classes.values() if not cls.imported for d in cls_to_dbscheme(cls,
+                                                                                                    data.classes, add_or_none_except)]
     if data.null:
         property_classes = {
             prop.type for cls in data.classes.values() for prop in cls.properties

misc/codegen/generators/qlgen.py

@@ -105,8 +105,17 @@ def _get_doc(cls: schema.Class, prop: schema.Property, plural=None):
     return f"{prop_name} of this {class_name}"
 
 
-def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.Class],
+def _type_is_hideable(t: str, lookup: typing.Dict[str, schema.ClassBase]) -> bool:
+    if t in lookup:
+        match lookup[t]:
+            case schema.Class() as cls:
+                return "ql_hideable" in cls.pragmas
+    return False
+
+
+def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.ClassBase],
                     prev_child: str = "") -> ql.Property:
+
     args = dict(
         type=prop.type if not prop.is_predicate else "predicate",
         qltest_skip="qltest_skip" in prop.pragmas,
@@ -116,7 +125,8 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic
         is_unordered=prop.is_unordered,
         description=prop.description,
         synth=bool(cls.synth) or prop.synth,
-        type_is_hideable="ql_hideable" in lookup[prop.type].pragmas if prop.type in lookup else False,
+        type_is_hideable=_type_is_hideable(prop.type, lookup),
+        type_is_codegen_class=prop.type in lookup and not lookup[prop.type].imported,
         internal="ql_internal" in prop.pragmas,
     )
     ql_name = prop.pragmas.get("ql_name", prop.name)
@@ -155,7 +165,7 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic
     return ql.Property(**args)
 
 
-def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> ql.Class:
+def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.ClassBase]) -> ql.Class:
     if "ql_name" in cls.pragmas:
         raise Error("ql_name is not supported yet for classes, only for properties")
     prev_child = ""
@@ -392,14 +402,15 @@ def generate(opts, renderer):
 
     data = schemaloader.load_file(input)
 
-    classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()}
+    classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items() if not cls.imported}
     if not classes:
         raise NoClasses
     root = next(iter(classes.values()))
     if root.has_children:
         raise RootElementHasChildren(root)
 
-    imports = {}
+    pre_imports = {n: cls.module for n, cls in data.classes.items() if cls.imported}
+    imports = dict(pre_imports)
     imports_impl = {}
     classes_used_by = {}
     cfg_classes = []
@@ -411,7 +422,7 @@ def generate(opts, renderer):
                          force=opts.force) as renderer:
 
         db_classes = [cls for name, cls in classes.items() if not data.classes[name].synth]
-        renderer.render(ql.DbClasses(db_classes), out / "Raw.qll")
+        renderer.render(ql.DbClasses(classes=db_classes, imports=sorted(set(pre_imports.values()))), out / "Raw.qll")
 
         classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name))
         for c in classes_by_dir_and_name:
@@ -440,6 +451,8 @@ def generate(opts, renderer):
             renderer.render(cfg_classes_val, cfg_qll)
 
         for c in data.classes.values():
+            if c.imported:
+                continue
             path = _get_path(c)
             path_impl = _get_path_impl(c)
             stub_file = stub_out / path_impl
@@ -458,20 +471,23 @@ def generate(opts, renderer):
             renderer.render(class_public, class_public_file)
 
         # for example path/to/elements -> path/to/elements.qll
-        renderer.render(ql.ImportList([i for name, i in imports.items() if not classes[name].internal]),
+        renderer.render(ql.ImportList([i for name, i in imports.items() if name not in classes or not classes[name].internal]),
                         include_file)
 
         elements_module = get_import(include_file, opts.root_dir)
 
         renderer.render(
             ql.GetParentImplementation(
                 classes=list(classes.values()),
-                imports=[elements_module] + [i for name, i in imports.items() if classes[name].internal],
+                imports=[elements_module] + [i for name,
+                                             i in imports.items() if name in classes and classes[name].internal],
             ),
             out / 'ParentChild.qll')
 
         if test_out:
             for c in data.classes.values():
+                if c.imported:
+                    continue
                 if should_skip_qltest(c, data.classes):
                     continue
                 test_with_name = c.pragmas.get("qltest_test_with")
@@ -501,7 +517,8 @@ def generate(opts, renderer):
         constructor_imports = []
         synth_constructor_imports = []
         stubs = {}
-        for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)):
+        for cls in sorted((cls for cls in data.classes.values() if not cls.imported),
+                          key=lambda cls: (cls.group, cls.name)):
             synth_type = get_ql_synth_class(cls)
             if synth_type.is_final:
                 final_synth_types.append(synth_type)

misc/codegen/generators/rustgen.py

@@ -49,7 +49,7 @@ def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field:
 
 
 def _get_properties(
-    cls: schema.Class, lookup: dict[str, schema.Class],
+    cls: schema.Class, lookup: dict[str, schema.ClassBase],
 ) -> typing.Iterable[tuple[schema.Class, schema.Property]]:
     for b in cls.bases:
         yield from _get_properties(lookup[b], lookup)
@@ -58,20 +58,22 @@ def _get_properties(
 
 
 def _get_ancestors(
-    cls: schema.Class, lookup: dict[str, schema.Class]
+    cls: schema.Class, lookup: dict[str, schema.ClassBase]
 ) -> typing.Iterable[schema.Class]:
     for b in cls.bases:
         base = lookup[b]
-        yield base
-        yield from _get_ancestors(base, lookup)
+        if not base.imported:
+            base = typing.cast(schema.Class, base)
+            yield base
+            yield from _get_ancestors(base, lookup)
 
 
 class Processor:
     def __init__(self, data: schema.Schema):
         self._classmap = data.classes
 
     def _get_class(self, name: str) -> rust.Class:
-        cls = self._classmap[name]
+        cls = typing.cast(schema.Class, self._classmap[name])
         properties = [
             (c, p)
             for c, p in _get_properties(cls, self._classmap)
@@ -101,8 +103,10 @@ def _get_class(self, name: str) -> rust.Class:
     def get_classes(self):
         ret = {"": []}
         for k, cls in self._classmap.items():
-            if not cls.synth:
+            if not cls.imported and not cls.synth:
                 ret.setdefault(cls.group, []).append(self._get_class(cls.name))
+            elif cls.imported:
+                ret[""].append(rust.Class(name=cls.name))
         return ret
 
 

misc/codegen/generators/rusttestgen.py

@@ -56,6 +56,8 @@ def generate(opts, renderer):
                          registry=opts.ql_test_output / ".generated_tests.list",
                          force=opts.force) as renderer:
         for cls in schema.classes.values():
+            if cls.imported:
+                continue
             if (qlgen.should_skip_qltest(cls, schema.classes) or
                     "rust_skip_doc_test" in cls.pragmas):
                 continue

misc/codegen/lib/ql.py

@@ -44,6 +44,7 @@ class Property:
     doc_plural: Optional[str] = None
     synth: bool = False
     type_is_hideable: bool = False
+    type_is_codegen_class: bool = False
     internal: bool = False
     cfg: bool = False
 
@@ -66,10 +67,6 @@ def indefinite_getter(self):
             article = "An" if self.singular[0] in "AEIO" else "A"
             return f"get{article}{self.singular}"
 
-    @property
-    def type_is_class(self):
-        return bool(self.type) and self.type[0].isupper()
-
     @property
     def is_repeated(self):
         return bool(self.plural)
@@ -191,6 +188,7 @@ class DbClasses:
     template: ClassVar = 'ql_db'
 
     classes: List[Class] = field(default_factory=list)
+    imports: List[str] = field(default_factory=list)
 
 
 @dataclass