feat(native): Jinja - support fields reflection for PyObject (#7312)

packages/cubejs-backend-native/src/cross/clrepr_python.rs

@@ -9,7 +9,7 @@ use pyo3::{AsPyPointer, Py, PyAny, PyErr, PyObject, Python, ToPyObject};
 
 #[derive(Debug, Clone)]
 pub enum PythonRef {
-    PyObject(Py<PyAny>),
+    PyObject(PyObject),
     PyFunction(Py<PyFunction>),
     /// Special type to transfer functions through JavaScript
     /// In JS it's an external object. It's not the same as Function.

packages/cubejs-backend-native/src/template/entry.rs

@@ -1,70 +1,16 @@
 use crate::cross::*;
 use crate::template::mj_value::*;
+use crate::template::neon::NeonMiniJinjaContext;
 use crate::utils::bind_method;
+
 use log::trace;
 use minijinja as mj;
-use neon::context::Context;
 use neon::prelude::*;
 use std::cell::RefCell;
-use std::error::Error;
 
 #[cfg(feature = "python")]
 use pyo3::{exceptions::PyNotImplementedError, prelude::*, types::PyTuple, AsPyPointer};
 
-trait NeonMiniJinjaContext {
-    fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T>;
-}
-
-impl<'a> NeonMiniJinjaContext for FunctionContext<'a> {
-    fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T> {
-        let codeblock = if let Some(source) = err.template_source() {
-            let lines: Vec<_> = source.lines().enumerate().collect();
-            let idx = err.line().unwrap_or(1).saturating_sub(1);
-            let skip = idx.saturating_sub(3);
-
-            let pre = lines.iter().skip(skip).take(3.min(idx)).collect::<Vec<_>>();
-            let post = lines.iter().skip(idx + 1).take(3).collect::<Vec<_>>();
-
-            let mut content = "".to_string();
-
-            for (idx, line) in pre {
-                content += &format!("{:>4} | {}\r\n", idx + 1, line);
-            }
-
-            content += &format!("{:>4} > {}\r\n", idx + 1, lines[idx].1);
-
-            if let Some(_span) = err.range() {
-                // TODO(ovr): improve
-                content += &format!(
-                    "     i {}{} {}\r\n",
-                    " ".repeat(0),
-                    "^".repeat(24),
-                    err.kind(),
-                );
-            } else {
-                content += &format!("     | {}\r\n", "^".repeat(24));
-            }
-
-            for (idx, line) in post {
-                content += &format!("{:>4} | {}\r\n", idx + 1, line);
-            }
-
-            format!("{}\r\n{}\r\n{}", "-".repeat(79), content, "-".repeat(79))
-        } else {
-            "".to_string()
-        };
-
-        if let Some(next_err) = err.source() {
-            self.throw_error(format!(
-                "{} caused by: {:#}\r\n{}",
-                err, next_err, codeblock
-            ))
-        } else {
-            self.throw_error(format!("{}\r\n{}", err, codeblock))
-        }
-    }
-}
-
 struct JinjaEngine {
     inner: mj::Environment<'static>,
 }

packages/cubejs-backend-native/src/template/mj_value/python.rs

@@ -7,8 +7,8 @@ use minijinja as mj;
 use minijinja::value as mjv;
 use minijinja::value::{Object, ObjectKind, StructObject, Value};
 use pyo3::exceptions::PyNotImplementedError;
-use pyo3::types::{PyFunction, PyTuple};
-use pyo3::{AsPyPointer, Py, PyAny, PyErr, PyResult, Python};
+use pyo3::types::{PyDict, PyFunction, PyTuple};
+use pyo3::{AsPyPointer, Py, PyErr, PyObject, PyResult, Python};
 use std::convert::TryInto;
 use std::sync::Arc;
 
@@ -97,7 +97,7 @@ pub fn from_minijinja_value(from: &mjv::Value) -> Result<CLRepr, mj::Error> {
 }
 
 pub struct JinjaPythonObject {
-    pub(crate) inner: Py<PyAny>,
+    pub(crate) inner: PyObject,
 }
 
 impl std::fmt::Debug for JinjaPythonObject {
@@ -222,20 +222,25 @@ impl StructObject for JinjaPythonObject {
     }
 
     fn fields(&self) -> Vec<Arc<str>> {
-        // TODO(ovr): Should we enable it? dump fn?
-        // let obj_ref = &self.inner;
-        //
-        // Python::with_gil(|py| {
-        //     let mut fields = vec![];
-        //
-        //     for key in obj_ref.as_ref(py).keys() {
-        //         fields.push(key.to_string().into());
-        //     }
-        //
-        //     fields
-        // })
-
-        vec![]
+        let obj_ref = &self.inner as &PyObject;
+
+        Python::with_gil(|py| {
+            let mut fields: Vec<Arc<str>> = vec![];
+
+            match obj_ref.downcast::<PyDict>(py) {
+                Ok(dict_ref) => {
+                    for key in dict_ref.keys() {
+                        fields.push(key.to_string().into());
+                    }
+                }
+                Err(err) => {
+                    #[cfg(debug_assertions)]
+                    log::trace!("Unable to extract PyDict: {:?}", err)
+                }
+            }
+
+            fields
+        })
     }
 }
 

packages/cubejs-backend-native/src/template/mod.rs

@@ -1,4 +1,5 @@
 mod entry;
 mod mj_value;
+mod neon;
 
 pub use entry::template_register_module;

packages/cubejs-backend-native/src/template/neon.rs

@@ -0,0 +1,58 @@
+use minijinja as mj;
+use std::error::Error;
+
+use neon::prelude::*;
+
+pub(crate) trait NeonMiniJinjaContext<'a>: Context<'a> {
+    fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T> {
+        let codeblock = if let Some(source) = err.template_source() {
+            let lines: Vec<_> = source.lines().enumerate().collect();
+            let idx = err.line().unwrap_or(1).saturating_sub(1);
+            let skip = idx.saturating_sub(3);
+
+            let pre = lines.iter().skip(skip).take(3.min(idx)).collect::<Vec<_>>();
+            let post = lines.iter().skip(idx + 1).take(3).collect::<Vec<_>>();
+
+            let mut content = "".to_string();
+
+            for (idx, line) in pre {
+                content += &format!("{:>4} | {}\r\n", idx + 1, line);
+            }
+
+            content += &format!("{:>4} > {}\r\n", idx + 1, lines[idx].1);
+
+            if let Some(_span) = err.range() {
+                // TODO(ovr): improve
+                content += &format!(
+                    "     i {}{} {}\r\n",
+                    " ".repeat(0),
+                    "^".repeat(24),
+                    err.kind(),
+                );
+            } else {
+                content += &format!("     | {}\r\n", "^".repeat(24));
+            }
+
+            for (idx, line) in post {
+                content += &format!("{:>4} | {}\r\n", idx + 1, line);
+            }
+
+            format!("{}\r\n{}\r\n{}", "-".repeat(79), content, "-".repeat(79))
+        } else {
+            "".to_string()
+        };
+
+        if let Some(next_err) = err.source() {
+            self.throw_error(format!(
+                "{} caused by: {:#}\r\n{}",
+                err, next_err, codeblock
+            ))
+        } else {
+            self.throw_error(format!("{}\r\n{}", err, codeblock))
+        }
+    }
+}
+
+impl<'a> NeonMiniJinjaContext<'a> for FunctionContext<'a> {}
+
+impl<'a> NeonMiniJinjaContext<'a> for TaskContext<'a> {}

packages/cubejs-backend-native/test/__snapshots__/jinja.test.ts.snap

@@ -211,7 +211,8 @@ exports[`Jinja (new api) render data-model.yml.jinja: data-model.yml.jinja 1`] =
 `;
 
 exports[`Jinja (new api) render dump_context.yml.jinja: dump_context.yml.jinja 1`] = `
-"<pre></pre>
+"
+<pre></pre>
 
 print:
   bool_true: true
@@ -239,7 +240,10 @@ exports[`Jinja (new api) render filters.yml.jinja: filters.yml.jinja 1`] = `
 exports[`Jinja (new api) render python.yml: python.yml 1`] = `
 "test:
   unsafe_string: \\"\\"\\\\\\"unsafe string\\\\\\" <>\\"\\"
-  safe_string: \\"\\"safe string\\" <>\\""
+  safe_string: \\"\\"safe string\\" <>\\"
+
+dump:
+  dict_as_obj: \\"{\\\\n    \\\\\\"a_attr\\\\\\": String(\\\\n        \\\\\\"value for attribute a\\\\\\",\\\\n        Normal,\\\\n    ),\\\\n}\\""
 `;
 
 exports[`Jinja (new api) render template_error.jinja: template_error.jinja 1`] = `

packages/cubejs-backend-native/test/jinja.test.ts

@@ -83,6 +83,7 @@ suite('Python model', () => {
       new_int_tuple: expect.any(Object),
       new_str_tuple: expect.any(Object),
       new_safe_string: expect.any(Object),
+      new_object_from_dict: expect.any(Object),
       load_class_model: expect.any(Object),
     });
 
@@ -115,6 +116,7 @@ darwinSuite('Scope Python model', () => {
       new_int_tuple: expect.any(Object),
       new_str_tuple: expect.any(Object),
       new_safe_string: expect.any(Object),
+      new_object_from_dict: expect.any(Object),
       load_class_model: expect.any(Object),
     });
   });

packages/cubejs-backend-native/test/templates/dump_context.yml.jinja

@@ -1,3 +1,4 @@
+{# TODO: We need stable sort for dump #}
 <pre>{# debug() #}</pre>
 
 print:

packages/cubejs-backend-native/test/templates/jinja-instance.py

@@ -49,6 +49,16 @@ def new_str_tuple():
 def new_safe_string():
   return SafeString('"safe string" <>')
 
+class MyCustomObject(dict):
+  def __init__(self):
+    self['a_attr'] = "value for attribute a"
+# TODO: We need stable sort for dump
+#     self['b_attr'] = "value for attribute b"
+
+@template.function
+def new_object_from_dict():
+  return MyCustomObject()
+
 @template.function
 def load_data_sync():
   client = MyApiClient("google.com")

packages/cubejs-backend-native/test/templates/python.yml

@@ -1,6 +1,10 @@
 {%- set unsafe_string = '"unsafe string" <>' -%}
 {%- set safe_string = new_safe_string() -%}
+{%- set dict_as_obj = new_object_from_dict() -%}
 
 test:
   unsafe_string: "{{ unsafe_string }}"
   safe_string: "{{ safe_string }}"
+
+dump:
+  dict_as_obj: {{ debug(dict_as_obj) }}

packages/cubejs-backend-native/test/templates/scoped-utils.py

@@ -41,6 +41,16 @@ def new_str_tuple():
 def new_safe_string():
   return SafeString('"safe string" <>')
 
+class MyCustomObject(dict):
+  def __init__(self):
+    self['a_attr'] = "value for attribute a"
+# TODO: We need stable sort for dump
+#     self['b_attr'] = "value for attribute b"
+
+@context_func
+def new_object_from_dict():
+  return MyCustomObject()
+
 @context_func
 def load_data_sync():
    client = MyApiClient("google.com")

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -45,11 +45,6 @@ export class YamlCompiler {
   }
 
   public initFromPythonContext(ctx: PythonCtx) {
-    console.log({
-      debugInfo: getEnv('devMode'),
-      filters: ctx.filters,
-    });
-
     this.jinjaEngine = this.nativeInstance.newJinjaEngine({
       debugInfo: getEnv('devMode'),
       filters: ctx.filters,