From 1ae3c5676d25727e6dd63d863d9b9896043d3aa4 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:44:50 +0200 Subject: [PATCH] move memory ownership to rust * Use more Vecs instead of malloc * Bump SZ3 to 3.2.2 * Bump to v0.3.0+SZ3-3.2.2 * bump SZ3 fixes windows build * bump cc * Build with C++17 --------- Co-authored-by: Robin Ole Heinemann --- Cargo.toml | 6 +- src/lib.rs | 285 ++++++++++++++++++++------------------------ sz3-sys/Cargo.toml | 8 +- sz3-sys/SZ3 | 2 +- sz3-sys/build.rs | 20 ++-- sz3-sys/wrapper.hpp | 25 ++-- 6 files changed, 158 insertions(+), 188 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 93441ba..69d321c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sz3" -version = "0.2.0+SZ3-3.2.1" +version = "0.3.0+SZ3-3.2.2" edition = "2021" authors = ["Robin Ole Heinemann "] description = "High level bindings to the SZ3 lossy floating point compressor." @@ -16,8 +16,8 @@ exclude = ["test_data/*"] openmp = ["sz3-sys/openmp"] [dependencies] -sz3-sys = { path = "sz3-sys", version = "0.2.0" } -thiserror = "1.0.61" +sz3-sys = { path = "sz3-sys", version = "0.3.0" } +thiserror = "2.0.0" [dev-dependencies] bytemuck = "1.16.0" diff --git a/src/lib.rs b/src/lib.rs index b3007c3..7ad202e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -296,36 +296,6 @@ pub struct Config { block_size: Option, } -#[derive(Debug)] -enum OwnedOrBorrowed<'a, A: ?Sized, B> { - Borrowed(&'a A), - Owned(B), -} - -impl<'a, A: ?Sized, B: Clone> Clone for OwnedOrBorrowed<'a, A, B> { - fn clone(&self) -> Self { - match self { - Self::Borrowed(a) => Self::Borrowed(a), - Self::Owned(b) => Self::Owned(b.clone()), - } - } -} - -impl<'a, A: ?Sized, B, T: 'a> std::ops::Deref for OwnedOrBorrowed<'a, A, B> -where - B: std::ops::Deref, - T: std::ops::Deref, -{ - type Target = A; - - fn deref(&self) -> &Self::Target { - match self { - Self::Owned(owned) => owned, - Self::Borrowed(borrowed) => borrowed, - } - } -} - pub trait SZ3Compressible: private::Sealed + std::ops::Sub + Sized {} impl SZ3Compressible for f32 {} impl SZ3Compressible for f64 {} @@ -334,107 +304,151 @@ impl SZ3Compressible for i64 {} mod private { pub trait Sealed { + unsafe fn compress_size_bound(config: sz3_sys::SZ3_Config) -> usize; + unsafe fn compress( config: sz3_sys::SZ3_Config, data: *const Self, - len: *mut usize, - ) -> *mut i8; + compressed_data: *mut u8, + compressed_capacity: usize, + ) -> usize; + + unsafe fn decompress_num( + compressed_data: *const u8, + compressed_len: usize, + ) -> usize; + unsafe fn decompress( - compressed_data: *const i8, + compressed_data: *const u8, compressed_len: usize, - uncompressed: *mut *mut Self, + decompressed_data: *mut Self, ) -> sz3_sys::SZ3_Config; - unsafe fn dealloc(data: *mut Self); } + impl Sealed for f32 { + unsafe fn compress_size_bound(config: sz3_sys::SZ3_Config) -> usize { + sz3_sys::compress_float_size_bound(config) + } + unsafe fn compress( config: sz3_sys::SZ3_Config, data: *const Self, - len: *mut usize, - ) -> *mut i8 { - sz3_sys::compress_float(config, data, len) as _ + compressed_data: *mut u8, + compressed_capacity: usize, + ) -> usize { + sz3_sys::compress_float(config, data, compressed_data.cast(), compressed_capacity) } - unsafe fn decompress( - compressed_data: *const i8, + unsafe fn decompress_num( + compressed_data: *const u8, compressed_len: usize, - uncompressed: *mut *mut Self, - ) -> sz3_sys::SZ3_Config { - sz3_sys::decompress_float(compressed_data as _, compressed_len, uncompressed) + ) -> usize { + sz3_sys::decompress_float_num(compressed_data.cast(), compressed_len) } - unsafe fn dealloc(data: *mut Self) { - sz3_sys::dealloc_result_float(data) + unsafe fn decompress( + compressed_data: *const u8, + compressed_len: usize, + decompressed_data: *mut Self, + ) -> sz3_sys::SZ3_Config { + sz3_sys::decompress_float(compressed_data.cast(), compressed_len, decompressed_data) } } + impl Sealed for f64 { + unsafe fn compress_size_bound(config: sz3_sys::SZ3_Config) -> usize { + sz3_sys::compress_double_size_bound(config) + } + unsafe fn compress( config: sz3_sys::SZ3_Config, data: *const Self, - len: *mut usize, - ) -> *mut i8 { - sz3_sys::compress_double(config, data, len) as _ + compressed_data: *mut u8, + compressed_capacity: usize, + ) -> usize { + sz3_sys::compress_double(config, data, compressed_data.cast(), compressed_capacity) } - unsafe fn decompress( - compressed_data: *const i8, + unsafe fn decompress_num( + compressed_data: *const u8, compressed_len: usize, - uncompressed: *mut *mut Self, - ) -> sz3_sys::SZ3_Config { - sz3_sys::decompress_double(compressed_data as _, compressed_len, uncompressed) + ) -> usize { + sz3_sys::decompress_double_num(compressed_data.cast(), compressed_len) } - unsafe fn dealloc(data: *mut Self) { - sz3_sys::dealloc_result_double(data) + unsafe fn decompress( + compressed_data: *const u8, + compressed_len: usize, + decompressed_data: *mut Self, + ) -> sz3_sys::SZ3_Config { + sz3_sys::decompress_double(compressed_data.cast(), compressed_len, decompressed_data) } } + impl Sealed for i32 { + unsafe fn compress_size_bound(config: sz3_sys::SZ3_Config) -> usize { + sz3_sys::compress_int32_t_size_bound(config) + } + unsafe fn compress( config: sz3_sys::SZ3_Config, data: *const Self, - len: *mut usize, - ) -> *mut i8 { - sz3_sys::compress_int32_t(config, data, len) as _ + compressed_data: *mut u8, + compressed_capacity: usize, + ) -> usize { + sz3_sys::compress_int32_t(config, data, compressed_data.cast(), compressed_capacity) } - unsafe fn decompress( - compressed_data: *const i8, + unsafe fn decompress_num( + compressed_data: *const u8, compressed_len: usize, - uncompressed: *mut *mut Self, - ) -> sz3_sys::SZ3_Config { - sz3_sys::decompress_int32_t(compressed_data as _, compressed_len, uncompressed) + ) -> usize { + sz3_sys::decompress_int32_t_num(compressed_data.cast(), compressed_len) } - unsafe fn dealloc(data: *mut Self) { - sz3_sys::dealloc_result_int32_t(data) + unsafe fn decompress( + compressed_data: *const u8, + compressed_len: usize, + decompressed_data: *mut Self, + ) -> sz3_sys::SZ3_Config { + sz3_sys::decompress_int32_t(compressed_data.cast(), compressed_len, decompressed_data) } } + impl Sealed for i64 { + unsafe fn compress_size_bound(config: sz3_sys::SZ3_Config) -> usize { + sz3_sys::compress_int64_t_size_bound(config) + } + unsafe fn compress( config: sz3_sys::SZ3_Config, data: *const Self, - len: *mut usize, - ) -> *mut i8 { - sz3_sys::compress_int64_t(config, data, len) as _ + compressed_data: *mut u8, + compressed_capacity: usize, + ) -> usize { + sz3_sys::compress_int64_t(config, data, compressed_data.cast(), compressed_capacity) } - unsafe fn decompress( - compressed_data: *const i8, + unsafe fn decompress_num( + compressed_data: *const u8, compressed_len: usize, - uncompressed: *mut *mut Self, - ) -> sz3_sys::SZ3_Config { - sz3_sys::decompress_int64_t(compressed_data as _, compressed_len, uncompressed) + ) -> usize { + sz3_sys::decompress_int64_t_num(compressed_data.cast(), compressed_len) } - unsafe fn dealloc(data: *mut Self) { - sz3_sys::dealloc_result_int64_t(data) + unsafe fn decompress( + compressed_data: *const u8, + compressed_len: usize, + decompressed_data: *mut Self, + ) -> sz3_sys::SZ3_Config { + sz3_sys::decompress_int64_t(compressed_data.cast(), compressed_len, decompressed_data) } } } #[derive(Clone, Debug)] -pub struct DimensionedData<'a, V: SZ3Compressible> { - data: OwnedOrBorrowed<'a, [V], SZ3DecompressionResult>, +pub struct DimensionedData> { + data: T, dims: Vec, } @@ -445,20 +459,8 @@ pub struct DimensionedDataBuilder<'a, V> { remainder: usize, } -impl DimensionedData<'static, V> { - fn from_raw(config: sz3_sys::SZ3_Config, data: SZ3DecompressionResult) -> Self { - let dims = (0..config.N) - .map(|i| unsafe { std::ptr::read(config.dims.add(i as _)) }) - .collect(); - Self { - data: OwnedOrBorrowed::Owned(data), - dims, - } - } -} - -impl<'a, V: SZ3Compressible> DimensionedData<'a, V> { - pub fn build>(data: &'a T) -> DimensionedDataBuilder<'a, V> { +impl> DimensionedData { + pub fn build<'a>(data: &'a T) -> DimensionedDataBuilder<'a, V> { DimensionedDataBuilder { data, dims: vec![], @@ -470,6 +472,10 @@ impl<'a, V: SZ3Compressible> DimensionedData<'a, V> { &self.data } + pub fn into_data(self) -> T { + self.data + } + pub fn dims(&self) -> &[usize] { &self.dims } @@ -544,12 +550,12 @@ impl<'a, V: SZ3Compressible> DimensionedDataBuilder<'a, V> { } } - pub fn remainder_dim(self) -> Result> { + pub fn remainder_dim(self) -> Result> { let remainder = self.remainder; self.dim(remainder)?.finish() } - pub fn finish(self) -> Result> { + pub fn finish(self) -> Result> { if self.remainder != 1 { Err(SZ3Error::UnderSpecifiedDimensions { dims: self.dims, @@ -558,7 +564,7 @@ impl<'a, V: SZ3Compressible> DimensionedDataBuilder<'a, V> { }) } else { Ok(DimensionedData { - data: OwnedOrBorrowed::Borrowed(self.data), + data: self.data, dims: self.dims, }) } @@ -644,38 +650,18 @@ impl Config { } } -#[derive(Debug)] -pub struct SZ3CompressionResult { - data: *mut u8, - len: usize, -} - -impl Drop for SZ3CompressionResult { - fn drop(&mut self) { - unsafe { sz3_sys::dealloc_result(self.data as _) }; - } -} - -impl std::ops::Deref for SZ3CompressionResult { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - unsafe { std::slice::from_raw_parts(self.data, self.len) } - } -} - -pub fn compress( - data: &DimensionedData<'_, V>, +pub fn compress>( + data: &DimensionedData, error_bound: ErrorBound, -) -> Result { +) -> Result> { let config = Config::new(error_bound); compress_with_config(data, &config) } -pub fn compress_with_config( - data: &DimensionedData<'_, V>, +pub fn compress_with_config>( + data: &DimensionedData, config: &Config, -) -> Result { +) -> Result> { if let Some(prediction_dimension) = config.compression_algorithm.prediction_dimension() { let data_dimensions = data.dims().len() as u32; if prediction_dimension == 0 { @@ -720,56 +706,37 @@ pub fn compress_with_config( stride: block_size as _, }; - let mut len: usize = 0; - let data = unsafe { V::compress(raw_config, data.as_ptr() as _, &mut len) }; + let capacity: usize = unsafe { V::compress_size_bound(raw_config) }; + let mut compressed_data = Vec::with_capacity(capacity); - Ok(SZ3CompressionResult { - data: data as _, - len: len as _, - }) -} - -type SZ3DecompressionResult = std::rc::Rc>; + let len = unsafe { V::compress(raw_config, data.as_ptr(), compressed_data.as_mut_ptr(), capacity) }; + unsafe { compressed_data.set_len(len) }; -#[derive(Debug)] -struct SZ3DecompressionResultInner { - data: *mut T, - len: usize, -} - -impl Drop for SZ3DecompressionResultInner { - fn drop(&mut self) { - unsafe { T::dealloc(self.data) }; - } -} - -impl std::ops::Deref for SZ3DecompressionResultInner { - type Target = [T]; - - fn deref(&self) -> &Self::Target { - unsafe { std::slice::from_raw_parts(self.data, self.len) } - } + Ok(compressed_data) } pub fn decompress>( compressed_data: T, -) -> (Config, DimensionedData<'static, V>) { - let mut destination_ptr = std::ptr::null_mut(); +) -> (Config, DimensionedData>) { + let len = unsafe { V::decompress_num(compressed_data.as_ptr(), compressed_data.len()) }; + let mut data = Vec::with_capacity(len); + let config = unsafe { V::decompress( - compressed_data.as_ptr() as _, - compressed_data.len() as _, - &mut destination_ptr, + compressed_data.as_ptr(), + compressed_data.len(), + data.as_mut_ptr(), ) }; - - let data = std::rc::Rc::new(SZ3DecompressionResultInner { - data: destination_ptr, - len: config.num as _, - }); + unsafe { data.set_len(len) }; let decoded = Config::from_decompressed(config); - let data = DimensionedData::from_raw(config, data); + + let dims = (0..config.N) + .map(|i| unsafe { std::ptr::read(config.dims.add(i as _)) }) + .collect(); + + let data = DimensionedData { data, dims }; unsafe { sz3_sys::dealloc_config_dims(config.dims); @@ -795,7 +762,7 @@ mod tests { } fn check_error_bound( - data: &'_ DimensionedData<'_, T>, + data: &DimensionedData, config: &Config, error_bound: ErrorBound, ) -> Result<()> diff --git a/sz3-sys/Cargo.toml b/sz3-sys/Cargo.toml index 2db989b..ae5360e 100644 --- a/sz3-sys/Cargo.toml +++ b/sz3-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sz3-sys" -version = "0.2.0+SZ3-3.2.1" +version = "0.3.0+SZ3-3.2.2" edition = "2021" license = "GPL-3.0-only" authors = ["Robin Ole Heinemann "] @@ -18,6 +18,6 @@ openmp-sys = { version = "1.2.3", optional = true } zstd-sys = { version = "2.0.10", default-features = false } [build-dependencies] -bindgen = "0.69.4" -cc = "1.0.98" -cmake = "0.1.50" +bindgen = "0.71.1" +cc = "1.2.5" +cmake = "0.1.52" diff --git a/sz3-sys/SZ3 b/sz3-sys/SZ3 index 29234fd..81c2818 160000 --- a/sz3-sys/SZ3 +++ b/sz3-sys/SZ3 @@ -1 +1 @@ -Subproject commit 29234fd919b87ab7f951d519314900f9066f9f07 +Subproject commit 81c2818ee2fb4353b53b85bc313bc0d4002c2ee3 diff --git a/sz3-sys/build.rs b/sz3-sys/build.rs index ed1a797..134be8f 100644 --- a/sz3-sys/build.rs +++ b/sz3-sys/build.rs @@ -62,20 +62,23 @@ fn main() { .allowlist_type("SZ3::EB") .allowlist_type("SZ3::ALGO") .allowlist_type("SZ3::INTERP_ALGO") + .allowlist_function("compress_float_size_bound") + .allowlist_function("compress_double_size_bound") + .allowlist_function("compress_int32_t_size_bound") + .allowlist_function("compress_int64_t_size_bound") .allowlist_function("compress_float") .allowlist_function("compress_double") .allowlist_function("compress_int32_t") .allowlist_function("compress_int64_t") + .allowlist_function("decompress_float_num") + .allowlist_function("decompress_double_num") + .allowlist_function("decompress_int32_t_num") + .allowlist_function("decompress_int64_t_num") .allowlist_function("decompress_float") .allowlist_function("decompress_double") .allowlist_function("decompress_int32_t") .allowlist_function("decompress_int64_t") - .allowlist_function("dealloc_result_float") - .allowlist_function("dealloc_result_double") - .allowlist_function("dealloc_result_int32_t") - .allowlist_function("dealloc_result_int64_t") .allowlist_function("dealloc_config_dims") - .allowlist_function("dealloc_result") .generate() .expect("Unable to generate bindings"); @@ -88,12 +91,11 @@ fn main() { build .cpp(true) - .warnings(false) - .flag_if_supported("-std=c++17") - .flag_if_supported("/std:c++17") + .std("c++17") .include(sz3_root.join("include")) .include(zstd_root.join("include")) - .file("lib.cpp"); + .file("lib.cpp") + .warnings(false); if cfg!(feature = "openmp") { env::var("DEP_OPENMP_FLAG") // set by openmp-sys diff --git a/sz3-sys/wrapper.hpp b/sz3-sys/wrapper.hpp index 407fddd..de5924f 100644 --- a/sz3-sys/wrapper.hpp +++ b/sz3-sys/wrapper.hpp @@ -77,17 +77,22 @@ struct SZ3_Config { #define func(ty) \ - char * compress_ ## ty(SZ3_Config config, const ty * data, size_t &outSize) { \ - return SZ_compress(config.into(), data, outSize); \ + size_t compress_ ## ty ## _size_bound(SZ3_Config config) { \ + return SZ3::SZ_compress_size_bound(config.into()); \ } \ - SZ3_Config decompress_ ## ty(const char * compressedData, size_t compressedSize, ty *&decompressedData) { \ + size_t compress_ ## ty(SZ3_Config config, const ty * data, char * compressedData, size_t compressedCapacity) { \ + return SZ_compress(config.into(), data, compressedData, compressedCapacity); \ + } \ + size_t decompress_ ## ty ## _num(const char * compressedData, size_t compressedSize) { \ auto conf = SZ3::Config{}; \ - auto d = std::vector(compressedData, compressedData + compressedSize); \ - SZ_decompress(conf, d.data(), d.size(), decompressedData); \ - return SZ3_Config(conf); \ + auto compressedConfig = reinterpret_cast(compressedData); \ + conf.load(compressedConfig); \ + return conf.num; \ } \ - void dealloc_result_ ## ty(ty * result) { \ - delete [] result; \ + SZ3_Config decompress_ ## ty(const char * compressedData, size_t compressedSize, ty * decompressedData) { \ + auto conf = SZ3::Config{}; \ + SZ_decompress(conf, compressedData, compressedSize, decompressedData); \ + return SZ3_Config(conf); \ } func(float) @@ -95,10 +100,6 @@ func(double) func(int32_t) func(int64_t) -void dealloc_result(char * data) { - delete[] data; -} - void dealloc_config_dims(size_t * data) { delete[] data; }