From 16e836a6f6840c0421d9970528319efbda69336a Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 23 Jan 2024 21:30:15 -0800 Subject: [PATCH 1/8] feat: raw_timestamps support for writer and defragment --- nptdms/test/data/raw_timestamps.tdms | Bin 0 -> 1290 bytes nptdms/test/test_example_files.py | 12 ++++++++++-- nptdms/timestamp.py | 6 +++++- nptdms/writer.py | 2 ++ 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 nptdms/test/data/raw_timestamps.tdms diff --git a/nptdms/test/data/raw_timestamps.tdms b/nptdms/test/data/raw_timestamps.tdms new file mode 100644 index 0000000000000000000000000000000000000000..71b7c3b81d87bb284fa4015bb384d518cc09dc39 GIT binary patch literal 1290 zcmY+BTS$~q5XZL?NzF@QiDcReX41uoE~cBKX4ykQQ1+zQn%~lf7#I^iq(vkN)~2OymRad;VrZdY=kxr(r3Qv`nK^T2{=dUnhb$pFoz5Jt z(|P>FMwmw~x+P*=Bs^(en7UEpR-;8(BUbg1%~+c6Fk2M6!)Ph72Z@`cgdD5G>?l?W z6VylIMz`|+>whvLL?lqSTG}aGHT8>N;iaa0QPt*HsGp@qRBNwVJ{$ekt!_hw`;tzY zOm@Yg*3xX5*od&%dd#L+6zf84u3@tGK*2#$^VFxJ>0-sMPU#e`S%fQiQi*M#4UZB# z>LZ^}it(~Fthdl7s-`%t-Uw`gTR1f)A_iggO)=M8_8&BQ5(np02P4%D3T=Xp0 z9T1i`K!$1m@mI=>Tv6og7^94esqEp0-{@{_NTTW1cS^HH?YUt4L2XZ+lWoc4 zo=*4M2z1~9~Ha Date: Tue, 23 Jan 2024 21:30:15 -0800 Subject: [PATCH 2/8] feat: raw_timestamps support for writer and defragment --- nptdms/test/data/raw_timestamps.tdms | Bin 0 -> 1290 bytes nptdms/test/test_example_files.py | 12 ++++++++++-- nptdms/timestamp.py | 6 +++++- nptdms/writer.py | 7 +++++-- 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 nptdms/test/data/raw_timestamps.tdms diff --git a/nptdms/test/data/raw_timestamps.tdms b/nptdms/test/data/raw_timestamps.tdms new file mode 100644 index 0000000000000000000000000000000000000000..71b7c3b81d87bb284fa4015bb384d518cc09dc39 GIT binary patch literal 1290 zcmY+BTS$~q5XZL?NzF@QiDcReX41uoE~cBKX4ykQQ1+zQn%~lf7#I^iq(vkN)~2OymRad;VrZdY=kxr(r3Qv`nK^T2{=dUnhb$pFoz5Jt z(|P>FMwmw~x+P*=Bs^(en7UEpR-;8(BUbg1%~+c6Fk2M6!)Ph72Z@`cgdD5G>?l?W z6VylIMz`|+>whvLL?lqSTG}aGHT8>N;iaa0QPt*HsGp@qRBNwVJ{$ekt!_hw`;tzY zOm@Yg*3xX5*od&%dd#L+6zf84u3@tGK*2#$^VFxJ>0-sMPU#e`S%fQiQi*M#4UZB# z>LZ^}it(~Fthdl7s-`%t-Uw`gTR1f)A_iggO)=M8_8&BQ5(np02P4%D3T=Xp0 z9T1i`K!$1m@mI=>Tv6og7^94esqEp0-{@{_NTTW1cS^HH?YUt4L2XZ+lWoc4 zo=*4M2z1~9~Ha Date: Wed, 24 Jan 2024 08:34:18 -0800 Subject: [PATCH 3/8] enum_value as a class attribute, bytes as a property --- nptdms/timestamp.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nptdms/timestamp.py b/nptdms/timestamp.py index 870de76..312658a 100644 --- a/nptdms/timestamp.py +++ b/nptdms/timestamp.py @@ -16,12 +16,11 @@ class TdmsTimestamp(object): :ivar ~.seconds: Seconds since the epoch as a signed integer :ivar ~.second_fractions: A positive number of 2^-64 fractions of a second """ + enum_value = 0x44 def __init__(self, seconds, second_fractions): self.seconds = seconds self.second_fractions = second_fractions - self.enum_value = 0x44 - self.bytes = _struct_pack(' Date: Wed, 24 Jan 2024 09:04:17 -0800 Subject: [PATCH 4/8] always reads raw_timestamps on defragment, add test for data round trip --- nptdms/test/test_example_files.py | 12 +++++++++++- nptdms/writer.py | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nptdms/test/test_example_files.py b/nptdms/test/test_example_files.py index 442570f..392528d 100644 --- a/nptdms/test/test_example_files.py +++ b/nptdms/test/test_example_files.py @@ -38,7 +38,17 @@ def test_defragment_raw_timestamps(): """Test defragmenting a file with raw timestamps""" test_file_path = DATA_DIR + '/raw_timestamps.tdms' output_file = BytesIO() - TdmsWriter.defragment(test_file_path, output_file, raw_timestamps=True) + + # verify we can defragment a file with raw timestamps + TdmsWriter.defragment(test_file_path, output_file) + + # verify that both files' channel data are the same + input_tdms = tdms.TdmsFile(test_file_path) + output_file.seek(0) + output_tdms = tdms.TdmsFile(output_file) + for group in input_tdms.groups(): + for channel in group.channels(): + np.testing.assert_equal(channel.data, output_tdms[group.name][channel.name].data) def test_big_endian_format(): diff --git a/nptdms/writer.py b/nptdms/writer.py index 1a905ae..4b2a5ce 100644 --- a/nptdms/writer.py +++ b/nptdms/writer.py @@ -22,7 +22,7 @@ class TdmsWriter(object): """ @classmethod - def defragment(cls, source, destination, version=4712, index_file=False, raw_timestamps=False): + def defragment(cls, source, destination, version=4712, index_file=False): """ Defragemnts an existing TdmsFile by loading and moving each Object to a separate channel to stream read one consecutive part of the file for faster access. @@ -39,7 +39,7 @@ def defragment(cls, source, destination, version=4712, index_file=False, raw_tim If ``destination`` is a readable object ``index_file`` can either be a redable object or ``False`` to store a ``.tdms_index`` file inside of the submitted object or not. """ - file = TdmsFile(source, raw_timestamps) + file = TdmsFile(source, raw_timestamps=True) with cls(destination, version=version, index_file=index_file) as new_file: new_file.write_segment([RootObject(file.properties)]) for group in file.groups(): From 8e0e9c549078dff31dd3a98038a27d228e3dd898 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Wed, 24 Jan 2024 09:08:06 -0800 Subject: [PATCH 5/8] add test for timestamps round trip --- nptdms/test/test_example_files.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nptdms/test/test_example_files.py b/nptdms/test/test_example_files.py index 392528d..e74bdef 100644 --- a/nptdms/test/test_example_files.py +++ b/nptdms/test/test_example_files.py @@ -48,7 +48,12 @@ def test_defragment_raw_timestamps(): output_tdms = tdms.TdmsFile(output_file) for group in input_tdms.groups(): for channel in group.channels(): + # verify that both files' channel data are the same np.testing.assert_equal(channel.data, output_tdms[group.name][channel.name].data) + # verify that both files' timestamps are the same + input_timestamps = channel.time_track() + output_timestamps = output_tdms[group.name][channel.name].time_track() + np.testing.assert_equal(input_timestamps, output_timestamps) def test_big_endian_format(): From 077a0091bf0cd13aac36986bfbf005b64950eb93 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Wed, 24 Jan 2024 12:46:47 -0800 Subject: [PATCH 6/8] refactor defragementation test - added tdms_files_assert_equal() support function that compares two TdmsFile instances - parametrized test_defragment() with multiple example files, for more test coverage --- nptdms/test/test_example_files.py | 52 +++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/nptdms/test/test_example_files.py b/nptdms/test/test_example_files.py index e74bdef..86daf00 100644 --- a/nptdms/test/test_example_files.py +++ b/nptdms/test/test_example_files.py @@ -1,11 +1,12 @@ -""" Test reading example TDMS files -""" +""" Test reading example TDMS files """ import os from io import BytesIO import numpy as np -from nptdms import tdms, TdmsWriter +import pytest + +from nptdms import tdms, TdmsWriter, TdmsFile DATA_DIR = os.path.dirname(os.path.realpath(__file__)) + '/data' @@ -34,26 +35,45 @@ def test_raw_format(): 0.2517777]) -def test_defragment_raw_timestamps(): - """Test defragmenting a file with raw timestamps""" - test_file_path = DATA_DIR + '/raw_timestamps.tdms' +def tdms_files_assert_equal(tdms1: TdmsFile, tdms2: TdmsFile): + """Assert that two TdmsFile instances are equal""" + # verify file properties + for p in tdms1.properties: + np.testing.assert_equal(tdms1.properties[p], tdms2.properties[p]) + # verify group + for group in tdms1.groups(): + # verify group properties + for p in group.properties: + np.testing.assert_equal(group.properties[p], tdms2[group.name].properties[p]) + # verify channels + for channel in group.channels(): + # verify channel data + np.testing.assert_equal(channel.data, tdms2[group.name][channel.name].data) + # verify channel properties + for p in channel.properties: + np.testing.assert_equal(channel.properties[p], tdms2[group.name][channel.name].properties[p]) + + +@pytest.mark.parametrize("tdms_file", [ + 'raw_timestamps.tdms', # Test defragmentation of a file with raw timestamps + # 'raw1.tdms', # <- cannot defragment this file (ValueError: Channel data must be a 1d array) + 'Digital_Input.tdms', + 'big_endian.tdms', +]) +def test_defragment(tdms_file): + """Test defragmentation round trip for a TDMS file""" + test_file_path = DATA_DIR + '/' + tdms_file output_file = BytesIO() # verify we can defragment a file with raw timestamps TdmsWriter.defragment(test_file_path, output_file) - # verify that both files' channel data are the same - input_tdms = tdms.TdmsFile(test_file_path) + # rewind output file BytesIO instance and read it back in as a TdmsFile output_file.seek(0) output_tdms = tdms.TdmsFile(output_file) - for group in input_tdms.groups(): - for channel in group.channels(): - # verify that both files' channel data are the same - np.testing.assert_equal(channel.data, output_tdms[group.name][channel.name].data) - # verify that both files' timestamps are the same - input_timestamps = channel.time_track() - output_timestamps = output_tdms[group.name][channel.name].time_track() - np.testing.assert_equal(input_timestamps, output_timestamps) + + # verify that both TdmsFile objects are the same + tdms_files_assert_equal(tdms.TdmsFile(test_file_path), output_tdms) def test_big_endian_format(): From 5cd13a98885896ab4d72880527cdfd39e0ad61ba Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Wed, 24 Jan 2024 13:07:03 -0800 Subject: [PATCH 7/8] implement __eq__ for TdmsTimestamp type --- nptdms/timestamp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nptdms/timestamp.py b/nptdms/timestamp.py index 312658a..f486d09 100644 --- a/nptdms/timestamp.py +++ b/nptdms/timestamp.py @@ -30,6 +30,9 @@ def __str__(self): fraction_string = "{0:.6f}".format(self.second_fractions * 2.0 ** -64).split('.')[1] return "{0}.{1}".format(dt, fraction_string) + def __eq__(self, other): + return self.seconds == other.seconds and self.second_fractions == other.second_fractions + def as_datetime64(self, resolution='us'): """ Convert this timestamp to a numpy datetime64 object From d2bbdddea6ebf0da204cdac391153ae2988afe33 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Wed, 24 Jan 2024 13:07:22 -0800 Subject: [PATCH 8/8] set raw_timestamps=True when reading files --- nptdms/test/test_example_files.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nptdms/test/test_example_files.py b/nptdms/test/test_example_files.py index 86daf00..9984a48 100644 --- a/nptdms/test/test_example_files.py +++ b/nptdms/test/test_example_files.py @@ -68,12 +68,14 @@ def test_defragment(tdms_file): # verify we can defragment a file with raw timestamps TdmsWriter.defragment(test_file_path, output_file) - # rewind output file BytesIO instance and read it back in as a TdmsFile + # rewind output file BytesIO instance, so it can read it back in as a TdmsFile output_file.seek(0) - output_tdms = tdms.TdmsFile(output_file) # verify that both TdmsFile objects are the same - tdms_files_assert_equal(tdms.TdmsFile(test_file_path), output_tdms) + tdms_files_assert_equal( + tdms.TdmsFile(test_file_path, raw_timestamps=True), + tdms.TdmsFile(output_file, raw_timestamps=True), + ) def test_big_endian_format():