use nanosecond timestamps in file metadata (datafusion-contrib#28)

polygon-io · Jan 8, 2025 · 7616021 · 7616021
1 parent 64eaabd
commit 7616021
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 8 deletions.
diff --git a/src/materialized/dependencies.rs b/src/materialized/dependencies.rs
@@ -200,9 +200,15 @@ impl TableFunctionImpl for StaleFilesUdtf {
                     col("expected_target").alias("target"),
                     col("target_last_modified"),
                     col("sources_last_modified"),
-                    nvl(col("target_last_modified"), lit(0))
-                        .lt(col("sources_last_modified"))
-                        .alias("is_stale"),
+                    nvl(
+                        col("target_last_modified"),
+                        lit(ScalarValue::TimestampNanosecond(
+                            Some(0),
+                            Some(Arc::from("UTC")),
+                        )),
+                    )
+                    .lt(col("sources_last_modified"))
+                    .alias("is_stale"),
                 ])?
                 .build()?;
 

diff --git a/src/materialized/file_metadata.rs b/src/materialized/file_metadata.rs
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow::array::{Int64Builder, StringBuilder, UInt64Builder};
+use arrow::array::{StringBuilder, TimestampNanosecondBuilder, UInt64Builder};
 use arrow::record_batch::RecordBatch;
-use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
 use async_trait::async_trait;
 use datafusion::catalog::SchemaProvider;
 use datafusion::catalog::{CatalogProvider, Session};
@@ -65,7 +65,11 @@ impl FileMetadata {
                 Field::new("table_schema", DataType::Utf8, false),
                 Field::new("table_name", DataType::Utf8, false),
                 Field::new("file_path", DataType::Utf8, false),
-                Field::new("last_modified", DataType::Int64, false),
+                Field::new(
+                    "last_modified",
+                    DataType::Timestamp(TimeUnit::Nanosecond, Some(Arc::from("UTC"))),
+                    false,
+                ),
                 Field::new("size", DataType::UInt64, false),
             ])),
             catalog_list,
@@ -444,7 +448,7 @@ struct FileMetadataBuilder {
     schema_names: StringBuilder,
     table_names: StringBuilder,
     file_paths: StringBuilder,
-    last_modified: Int64Builder,
+    last_modified: TimestampNanosecondBuilder,
     size: UInt64Builder,
 }
 
@@ -456,7 +460,7 @@ impl FileMetadataBuilder {
             schema_names: StringBuilder::new(),
             table_names: StringBuilder::new(),
             file_paths: StringBuilder::new(),
-            last_modified: Int64Builder::new(),
+            last_modified: TimestampNanosecondBuilder::new().with_timezone("UTC"),
             size: UInt64Builder::new(),
         }
     }