From be904521c041ede7c4d4e90d87e8f089eea123fe Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:25:37 +0100 Subject: [PATCH 01/22] Add fix for sun_earth_distance_correction_factor The history of the sun earth distance correction is in pygac/satpy and pyorbital. With some combination of satpy and pyorbital (2020-2022) the correction where in practice applied twice. And since 2022 the attribute sun_earth_distance_correction_factor from satpy no longer includes the factor for correction but sqrt(the original correction). The history of the correction in pytroll: Original formula in pygac and pyorbital from 2014: corr_ = 1 - 0.0334 * np.cos(2 * np.pi * (jdays2000(utc_time) - 2) / year) Calulcated in pygac and applied like refl = refl * corr_. With attribute set in pygac: sun_earth_distance_correction_factor = corr_ 2020-06 Pygac correction depricated. 2020-09 Satpy reads corrrection from pyorbital but applies like refl = refl * corr_ * corr_ causing the correction to be applied twice. 2020-09 Attribute set in satpy: sun_earth_distance_correction_factor = corr_, same as before. 2022-06 Pyorbital correction in principle replaced with sqrt(corr_). This implies reflections now corrected with refl = refl * corr_ in satpy as expected. The sun_earth_distance_correction_factor attribute in principle set to np.sqrt(corr_) which means the attribute no longer contain the original sun_earth_distance_correction_factor attribute. --- level1c4pps/__init__.py | 16 +++++++++++ level1c4pps/seviri2pps_lib.py | 11 +++++++- level1c4pps/tests/test_seviri2pps.py | 42 +++++++++++++++------------- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 94fd99e..f4865a7 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -368,6 +368,21 @@ def adjust_lons_to_valid_range(scene): # scene['lon'] = centered_modulus(scene['lon']) # makes lon loose attrs satpy 0.24.0 scene['lon'].values = centered_modulus(scene['lon'].values) +def fix_sun_earth_distance_correction_factor(scene, band, start_time): + from pyorbital.astronomy import sun_earth_distance_correction + date_control = np.datetime64("2019-01-01T00:00:00") + sun_earth_distance_20190409 = sun_earth_distance_correction(date_control) + sun_earth_distance = sun_earth_distance_correction(start_time) + if (np.abs(sun_earth_distance_20190409 - 0.9833280675966011) < 0.00001 and + np.abs(sun_earth_distance - scene[band].attrs['sun_earth_distance_correction_factor']) < 0.00001): + logger.info("The sun earth distance correction attribute contain the sun earth distance, not the square.") + logger.info("Updating and adding sun earth distance correction attributes.") + current_factor = scene[band].attrs['sun_earth_distance_correction_factor'] + scene[band].attrs['satpy_sun_earth_distance_correction_factor'] = current_factor + scene[band].attrs['pps_sun_earth_distance_correction_factor'] = sun_earth_distance * sun_earth_distance + scene[band].attrs['sun_earth_distance'] = sun_earth_distance + scene[band].attrs['sun_earth_distance_correction_factor'] = sun_earth_distance * sun_earth_distance + def set_header_and_band_attrs_defaults(scene, BANDNAMES, PPS_TAGNAMES, REFL_BANDS, irch, orbit_n=0): """Add some default values for band attributes.""" @@ -422,6 +437,7 @@ def set_header_and_band_attrs_defaults(scene, BANDNAMES, PPS_TAGNAMES, REFL_BAND else: # Assume factor applied if available as attribute. scene[band].attrs['sun_earth_distance_correction_applied'] = 'True' + fix_sun_earth_distance_correction_factor(scene, band, irch.attrs['start_time']) scene[band].attrs['wavelength'] = scene[band].attrs['wavelength'][0:3] scene[band].attrs['sun_zenith_angle_correction_applied'] = 'False' if idtag in PPS_TAGNAMES_TO_IMAGE_NR: diff --git a/level1c4pps/seviri2pps_lib.py b/level1c4pps/seviri2pps_lib.py index 7ed0557..2423537 100644 --- a/level1c4pps/seviri2pps_lib.py +++ b/level1c4pps/seviri2pps_lib.py @@ -44,7 +44,11 @@ from pyorbital.orbital import get_observer_look from level1c4pps.calibration_coefs import get_calibration, CalibrationData -from level1c4pps import make_azidiff_angle, get_encoding, compose_filename, update_angle_attributes +from level1c4pps import (make_azidiff_angle, + get_encoding, + compose_filename, + update_angle_attributes, + fix_sun_earth_distance_correction_factor) try: @@ -274,10 +278,15 @@ def set_attrs(scene): scene[band].attrs['wavelength'] = [scene[band].attrs['wavelength'].min, scene[band].attrs['wavelength'].central, scene[band].attrs['wavelength'].max] + if 'sun_earth_distance_correction_factor' not in scene[band].attrs: scene[band].attrs['sun_earth_distance_correction_applied'] = False scene[band].attrs['sun_earth_distance_correction_factor'] = 1.0 + else: + fix_sun_earth_distance_correction_factor(scene, band, scene[band].attrs['start_time']) + scene[band].attrs['sun_zenith_angle_correction_applied'] = False + scene[band].attrs['name'] = "image{:d}".format(image_num) # Cosmetics diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index e55826e..307e5a6 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -81,11 +81,6 @@ def test_load_and_calibrate(self, mocked_scene): ) # Compare results and expectations - vis006_exp_pyspectral_1_7_1 = xr.DataArray( - [[1.07025268, 2.14050537], - [3.21075805, 4.28101074]], - dims=('y', 'x') - ) vis006_exp = xr.DataArray( [[1.034205, 2.06841], [3.102615, 4.13682]], @@ -96,11 +91,8 @@ def test_load_and_calibrate(self, mocked_scene): [7, 8]], dims=('y', 'x') ) - if res['VIS006'][0, 0] > 1.04: - # pyspectral 1.7.1 and older - xr.testing.assert_allclose(res['VIS006'], vis006_exp_pyspectral_1_7_1) - else: - xr.testing.assert_allclose(res['VIS006'], vis006_exp) + res['VIS006'].sun_earth_distance_correction_factor + xr.testing.assert_allclose(res['VIS006'], vis006_exp) xr.testing.assert_equal(res['IR_108'], ir_108_exp) self.assertFalse( res['VIS006'].attrs['sun_earth_distance_correction_applied'], @@ -211,15 +203,14 @@ def get_observer_look_patched(lon, lat, alt, *args): def test_set_attrs(self): """Test setting scene attributes.""" seviri2pps.BANDNAMES = ['VIS006', 'IR_108'] - vis006 = mock.MagicMock(attrs={'wavelength': WavelengthRange(0.56, 0.635, 0.71)}) - ir108 = mock.MagicMock(attrs={'platform_name': 'myplatform', - 'wavelength': WavelengthRange(9.8, 10.8, 11.8), - 'orbital_parameters': {'orb_a': 1, - 'orb_b': 2}, - 'georef_offset_corrected': True}) - scene_dict = {'VIS006': vis006, 'IR_108': ir108} - scene = mock.MagicMock(attrs={}) - scene.__getitem__.side_effect = scene_dict.__getitem__ + scene = get_fake_scene() + scene['VIS006'].attrs['sun_earth_distance_correction_applied'] = True + scene['VIS006'].attrs['start_time'] = dt.datetime(2020, 1, 1, 12) + scene['VIS006'].attrs['sun_earth_distance_correction_factor'] = 0.9833241577909706 + scene['IR_108'].attrs['orbital_parameters'] = {'orb_a': 1, + 'orb_b': 2} + scene['IR_108'].attrs['georef_offset_corrected'] = True + scene['IR_108'].attrs['platform_name'] = "my_platform_name" seviri2pps.set_attrs(scene) self.assertEqual(scene['VIS006'].attrs['name'], 'image0') @@ -229,7 +220,18 @@ def test_set_attrs(self): self.assertNotIn('orb_a', scene.attrs) self.assertNotIn('orbital_parameters', scene.attrs) self.assertNotIn('georef_offset_corrected', scene.attrs) - + exp_sun_earth_distance_correction_factor = 0.9669263992953216 + sun_earth_distance = 0.9833241577909706 + self.assertAlmostEqual(scene['VIS006'].sun_earth_distance_correction_factor, + exp_sun_earth_distance_correction_factor, places=7) + self.assertAlmostEqual(scene['VIS006'].pps_sun_earth_distance_correction_factor, + exp_sun_earth_distance_correction_factor, places=7) + self.assertAlmostEqual(scene['VIS006'].satpy_sun_earth_distance_correction_factor, + sun_earth_distance, places=7) + self.assertAlmostEqual(scene['VIS006'].sun_earth_distance, + sun_earth_distance, places=7) + + def test_get_mean_acq_time(self): """Test computation of mean scanline acquisition time.""" seviri2pps.BANDNAMES = ['VIS006', 'IR_108'] From a2dd7429c6d8304604cdcd2323abcca89d1edd1b Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:32:41 +0100 Subject: [PATCH 02/22] Remove option to undo sun-earth-distance-correction --- bin/seviri2pps.py | 3 --- level1c4pps/seviri2pps_lib.py | 26 +------------------------- level1c4pps/tests/test_seviri2pps.py | 9 +++------ 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/bin/seviri2pps.py b/bin/seviri2pps.py index 965acf1..9bf1b48 100755 --- a/bin/seviri2pps.py +++ b/bin/seviri2pps.py @@ -56,9 +56,6 @@ help="Engine for saving netcdf files netcdf4 or h5netcdf (default).") parser.add_argument('--use-nominal-time-in-filename', action='store_true', help='Use nominal scan timestamps in output filename.') - parser.add_argument('--no-sun-earth-distance-correction', - action='store_true', - help='Do not apply sun earth distance correction.') options = parser.parse_args() process_one_scan( options.files, diff --git a/level1c4pps/seviri2pps_lib.py b/level1c4pps/seviri2pps_lib.py index 2423537..8d402eb 100644 --- a/level1c4pps/seviri2pps_lib.py +++ b/level1c4pps/seviri2pps_lib.py @@ -38,7 +38,6 @@ from datetime import datetime, timedelta from satpy.scene import Scene import satpy.utils -from satpy.readers.utils import remove_earthsun_distance_correction from trollsift.parser import globify, Parser from pyorbital.astronomy import get_alt_az, sun_zenith_angle from pyorbital.orbital import get_observer_look @@ -93,7 +92,7 @@ class UnexpectedSatpyVersion(Exception): # MSG4-SEVI-MSG15-1234-NA-20190409121243.927000000Z -def load_and_calibrate(filenames, apply_sun_earth_distance_correction, rotate, +def load_and_calibrate(filenames, rotate, clip_calib): """Load and calibrate data. @@ -101,8 +100,6 @@ def load_and_calibrate(filenames, apply_sun_earth_distance_correction, rotate, Args: filenames: List of data files - apply_sun_earth_distance_correction: If True, apply sun-earth-distance- - correction to visible channels. rotate: Rotate image so that pixel (0, 0) is N-W. clip_calib: If True, do not extrapolate calibration coefficients beyond the time coverage of the calibration dataset. Instead, clip at the @@ -125,9 +122,6 @@ def load_and_calibrate(filenames, apply_sun_earth_distance_correction, rotate, _load_bands(scn_, rotate) _update_scene_attrs(scn_, {'image_rotated': rotate}) - if not apply_sun_earth_distance_correction: - remove_sun_earth_distance_correction(scn_) - return scn_ @@ -164,22 +158,6 @@ def _update_scene_attrs(scene, attrs): scene.attrs.update(attrs) -def remove_sun_earth_distance_correction(scene): - """Remove sun-earth-distance correction from visible channels. - - This is required because a downstream CLAAS module (CPP) does not recognize - the "sun_earth_distance_correction_applied" attribute and applies the - correction anyway. - """ - for band in BANDNAMES: - is_vis = scene[band].attrs['calibration'] == 'reflectance' - correction_applied = scene[band].attrs.get( - 'sun_earth_distance_correction_applied', False - ) - if is_vis and correction_applied: - scene[band] = remove_earthsun_distance_correction(scene[band]) - - def get_lonlats(dataset): """Get lat/lon coordinates.""" lons, lats = dataset.attrs['area'].get_lonlats() @@ -590,7 +568,6 @@ def _postproc_hrit(self, parsed): def process_one_scan(tslot_files, out_path, rotate=True, engine='h5netcdf', use_nominal_time_in_filename=False, - apply_sun_earth_distance_correction=True, clip_calib=False, save_azimuth_angles=False): """Make level 1c files in PPS-format.""" @@ -601,7 +578,6 @@ def process_one_scan(tslot_files, out_path, rotate=True, engine='h5netcdf', tic = time.time() scn_ = load_and_calibrate( tslot_files, - apply_sun_earth_distance_correction=apply_sun_earth_distance_correction, rotate=rotate, clip_calib=clip_calib ) diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index 307e5a6..5629330 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -75,15 +75,14 @@ def test_load_and_calibrate(self, mocked_scene): filenames = ['MSG4-SEVI-MSG15-1234-NA-20190409121243.927000000Z'] res = seviri2pps.load_and_calibrate( filenames, - apply_sun_earth_distance_correction=False, rotate=False, clip_calib=False ) # Compare results and expectations vis006_exp = xr.DataArray( - [[1.034205, 2.06841], - [3.102615, 4.13682]], + [[1.0, 2.0], + [3.0, 4.0]], dims=('y', 'x') ) ir_108_exp = xr.DataArray( @@ -91,10 +90,9 @@ def test_load_and_calibrate(self, mocked_scene): [7, 8]], dims=('y', 'x') ) - res['VIS006'].sun_earth_distance_correction_factor xr.testing.assert_allclose(res['VIS006'], vis006_exp) xr.testing.assert_equal(res['IR_108'], ir_108_exp) - self.assertFalse( + self.assertTrue( res['VIS006'].attrs['sun_earth_distance_correction_applied'], ) @@ -106,7 +104,6 @@ def test_load_and_calibrate_with_rotation(self, mocked_scene): filenames = ['MSG4-SEVI-MSG15-1234-NA-20190409121243.927000000Z'] res = seviri2pps.load_and_calibrate( filenames, - apply_sun_earth_distance_correction=False, rotate=True, clip_calib=False ) From 29a44c00ef5ef46fffb9f85e90a122728316e5c3 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:53:25 +0100 Subject: [PATCH 03/22] Lets keep the check of the sun-earth-distance-correction This test makes sure satpy is applying and removing the correction correctly. --- level1c4pps/tests/test_seviri2pps.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index 5629330..337903f 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -81,8 +81,8 @@ def test_load_and_calibrate(self, mocked_scene): # Compare results and expectations vis006_exp = xr.DataArray( - [[1.0, 2.0], - [3.0, 4.0]], + [[1.034205, 2.06841], + [3.102615, 4.13682]], dims=('y', 'x') ) ir_108_exp = xr.DataArray( @@ -90,9 +90,12 @@ def test_load_and_calibrate(self, mocked_scene): [7, 8]], dims=('y', 'x') ) + from satpy.readers.utils import remove_earthsun_distance_correction + res['VIS006'] = remove_earthsun_distance_correction(res['VIS006']) xr.testing.assert_allclose(res['VIS006'], vis006_exp) xr.testing.assert_equal(res['IR_108'], ir_108_exp) - self.assertTrue( + + self.assertFalse( res['VIS006'].attrs['sun_earth_distance_correction_applied'], ) From 60733b2c3b0da6cf3644a40b2f2226849766ad49 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:30:59 +0100 Subject: [PATCH 04/22] small fix --- bin/seviri2pps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/seviri2pps.py b/bin/seviri2pps.py index 9bf1b48..849a054 100755 --- a/bin/seviri2pps.py +++ b/bin/seviri2pps.py @@ -63,6 +63,5 @@ rotate=not options.no_rotation, engine=options.nc_engine, use_nominal_time_in_filename=options.use_nominal_time_in_filename, - apply_sun_earth_distance_correction=not options.no_sun_earth_distance_correction, save_azimuth_angles=options.azimuth_angles, ) From bbb7539c1988dabf57635b9ddb30afed6024485d Mon Sep 17 00:00:00 2001 From: Salomon Eliasson Date: Fri, 6 Dec 2024 13:08:12 +0100 Subject: [PATCH 05/22] added NN and linear v7 and v8 --- level1c4pps/vgac2pps_lib.py | 138 +++++++++++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index d1634ca..7ffe0de 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -323,6 +323,136 @@ "slope": 1.002, "offset": -0.69, "comment": "Based on collocation data for VZA < 3 and SZA 0-180", + } + }, + "v7": { + "r06": { + "viirs_channel": "M05", + "slope": 0.8534, + "offset":1.8517, + "comment": "Based on collocation data for VZA < 15", + }, + "r09": { + "viirs_channel": "M07", + "slope":0.8507, + "offset":1.1157, + "comment": "Based on collocation data for VZA < 15", + }, + "tb37": { + "viirs_channel": "M12", + "slope":0.9734, + "offset":6.1707, + "comment": "Based on collocation data for VZA < 15", + }, + "tb11": { + "viirs_channel": "M15", + "slope": 1.0006, + "offset": -0.0378, + "comment": "Based on collocation data for VZA < 15", + }, + "tb12": { + "viirs_channel": "M16", + "slope": 0.9906, + "offset": 2.1505, + "comment": "Based on collocation data for VZA < 15", + } + }, + "v8": { + "r06_day": { + "viirs_channel": "M05", + "slope": 0.8960, + "offset": 1.7118, + "max_sunzenith": 80, + "comment": "Based on collocation data for VZA < 15", + }, + "r09_day": { + "viirs_channel": "M07", + "slope": 0.8907, + "offset": 0.5547, + "max_sunzenith": 80, + "comment": "Based on collocation data for VZA < 15", + }, + "tb37_day": { + "viirs_channel": "M12", + "slope": 0.9817, + "offset": 3.0744, + "max_sunzenith": 80, + "comment": "Based on collocations for VZA < 15 and SZA < 80", + }, + "tb11_day": { + "viirs_channel": "M15", + "slope": 0.9926, + "offset": 2.0934, + "max_sunzenith": 80, + "comment": "Based on collocation data for VZA < 15 and SZA < 80", + }, + "tb12_day": { + "viirs_channel": "M16", + "slope": 0.9774, + "offset": 5.6555, + "max_sunzenith": 80, + "comment": "Based on collocation data for VZA < 15 and SZA < 80", + }, + "r06_twilight": { + "viirs_channel": "M05", + "slope": 0.6710, + "offset": 4.5075, + "min_sunzenith": 80, + "max_sunzenith": 90, + "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", + }, + "r09_twilight": { + "viirs_channel": "M07", + "slope": 0.7120, + "offset": 3.7473, + "min_sunzenith": 80, + "max_sunzenith": 90, + "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", + }, + "tb37_twilight": { + "viirs_channel": "M12", + "slope": 0.9711, + "offset": 6.77, + "min_sunzenith": 80, + "max_sunzenith": 90, + "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", + }, + "tb11_twilight": { + "viirs_channel": "M15", + "slope": 1.0024, + "offset": -0.5164, + "min_sunzenith": 80, + "max_sunzenith": 90, + "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", + }, + "tb12_twilight": { + "viirs_channel": "M16", + "slope": 0.9973, + "offset": -0.7479, + "min_sunzenith": 80, + "max_sunzenith": 90, + "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", + }, + "tb37_night": { + "viirs_channel": "M12", + "slope": 0.9948, + "offset": 1.3254, + "min_sunzenith": 90, + "comment": "Based on collocations for VZA < 15 and SZA >90", + }, + "tb11_night": { + "viirs_channel": "M15", + "slope": 1.0042, + "offset": -0.9186, + "min_sunzenith": 90, + "comment": "Based on collocations for VZA < 15 and SZA >90", + }, + "tb12_night": { + "viirs_channel": "M16", + "slope": 0.9962, + "offset": 0.7373, + "min_sunzenith": 90, + "comment": "Based on collocation data for VZA < 15 and SZA > 90", }, }, "KNMI": { @@ -438,8 +568,14 @@ "cfg_file_night": "ch4_satz_max_15_SUNZ_90_180_tdiff_120_sec_20241120.yaml", "cfg_file_twilight": "ch7_satz_max_15_SUNZ_80_89_tdiff_120_sec_20241120.yaml", "comment": "NN based on AVHRR and VGAC matchups using all AVHRR heritage channels" - } + }, + "NN_v4": { + "cfg_file_day": "ch7_satz_max_15_SUNZ_0_80_tdiff_120_sec_20241204.yaml", + "cfg_file_night": "ch4_satz_max_15_SUNZ_90_180_tdiff_120_sec_20241204.yaml", + "cfg_file_twilight": "ch7_satz_max_15_SUNZ_80_89_tdiff_120_sec_20241204.yaml", + "comment": "NN based on AVHRR and VGAC matchups using all AVHRR heritage channels" } +} def convert_to_noaa19_neural_network(scene, sbaf_version): From 0efe8b4a90aca0456e69e8bca677e7aefcfeb99d Mon Sep 17 00:00:00 2001 From: Salomon Eliasson Date: Tue, 10 Dec 2024 17:10:16 +0100 Subject: [PATCH 06/22] added NN_v4 to if statement --- level1c4pps/vgac2pps_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 7ffe0de..5e32f52 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -581,7 +581,7 @@ def convert_to_noaa19_neural_network(scene, sbaf_version): """Applies AVHRR SBAF to VGAC channels using NN approach""" - if sbaf_version in ["NN_v1", "NN_v2", "NN_v3"]: + if sbaf_version in ["NN_v1", "NN_v2", "NN_v3", "NN_v4"]: day_cfg_file = SBAF[sbaf_version]['cfg_file_day'] night_cfg_file = SBAF[sbaf_version]['cfg_file_night'] twilight_cfg_file = SBAF[sbaf_version]['cfg_file_twilight'] From 0966f56eabae5eceabc39bdd303e7469abe821ff Mon Sep 17 00:00:00 2001 From: Salomon Eliasson <46442045+salomoneliassonSMHI@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:54:07 +0100 Subject: [PATCH 07/22] Update level1c4pps/vgac2pps_lib.py Co-authored-by: Nina <6160529+ninahakansson@users.noreply.github.com> --- level1c4pps/vgac2pps_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 5e32f52..6d49a5e 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -412,7 +412,7 @@ "tb37_twilight": { "viirs_channel": "M12", "slope": 0.9711, - "offset": 6.77, + "offset": -0.7479, "min_sunzenith": 80, "max_sunzenith": 90, "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", From 75603cb702c93f7263ef0ed9b95fb86d9c5b3d5d Mon Sep 17 00:00:00 2001 From: Salomon Eliasson <46442045+salomoneliassonSMHI@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:54:15 +0100 Subject: [PATCH 08/22] Update level1c4pps/vgac2pps_lib.py Co-authored-by: Nina <6160529+ninahakansson@users.noreply.github.com> --- level1c4pps/vgac2pps_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 6d49a5e..3168891 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -411,7 +411,7 @@ }, "tb37_twilight": { "viirs_channel": "M12", - "slope": 0.9711, + "slope": 0.9973, "offset": -0.7479, "min_sunzenith": 80, "max_sunzenith": 90, From 30d62c712c7422432340f91e89fe8db74cd81856 Mon Sep 17 00:00:00 2001 From: Salomon Eliasson <46442045+salomoneliassonSMHI@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:54:51 +0100 Subject: [PATCH 09/22] Update level1c4pps/vgac2pps_lib.py Co-authored-by: Nina <6160529+ninahakansson@users.noreply.github.com> --- level1c4pps/vgac2pps_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 3168891..e26f439 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -427,7 +427,7 @@ }, "tb12_twilight": { "viirs_channel": "M16", - "slope": 0.9973, + "slope": 0.9967, "offset": -0.7479, "min_sunzenith": 80, "max_sunzenith": 90, From 35df83d2c3650cb74f6681b204d191e0df1f8092 Mon Sep 17 00:00:00 2001 From: Salomon Eliasson <46442045+salomoneliassonSMHI@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:55:05 +0100 Subject: [PATCH 10/22] Update level1c4pps/vgac2pps_lib.py Co-authored-by: Nina <6160529+ninahakansson@users.noreply.github.com> --- level1c4pps/vgac2pps_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index e26f439..5ea6f1b 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -428,7 +428,7 @@ "tb12_twilight": { "viirs_channel": "M16", "slope": 0.9967, - "offset": -0.7479, + "offset": 0.6742, "min_sunzenith": 80, "max_sunzenith": 90, "comment": "Based on collocation data for VZA < 15 and 80<= SZA <=90", From 84356b20d9dc2a035a1764cfd7793b7153385cf7 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:14:23 +0100 Subject: [PATCH 11/22] Original VGAC files provide normalized reflectances AND Earth-Sun distance corrected data --- level1c4pps/vgac2pps_lib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index d1634ca..fa6c12b 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -563,8 +563,9 @@ def set_header_and_band_attrs(scene, orbit_n=0): for band in REFL_BANDS: if band not in scene: continue - # For VIIRS data sun_zenith_angle_correction_applied is applied always! + # Original VGAC files provide normalized reflectances AND Earth-Sun distance corrected data scene[band].attrs["sun_zenith_angle_correction_applied"] = "True" + scene[band].attrs['sun_earth_distance_correction_applied'] = "True" return nimg From 59c69f0f13ab76802b7bd2bd1eebea02d4c7ed62 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:40:40 +0100 Subject: [PATCH 12/22] Updated CHANGELOG.md for v0.2.35 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89a3220..8ffaa9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +## Version <0.2.35> (2024/12/16) + +### Issues Closed + +* [Issue 94](https://github.com/foua-pps/level1c4pps/issues/94) - VGAC sun-earth distance correction attribute is set wrong ([PR 102](https://github.com/foua-pps/level1c4pps/pull/102) by [@ninahakansson](https://github.com/ninahakansson)) +* [Issue 93](https://github.com/foua-pps/level1c4pps/issues/93) - VGAC time unit is in microseconds not milliseconds ([PR 96](https://github.com/foua-pps/level1c4pps/pull/96) by [@BengtRydberg](https://github.com/BengtRydberg)) +* [Issue 79](https://github.com/foua-pps/level1c4pps/issues/79) - Satpy doesn't support Python 3.8 anymore + +In this release 3 issues were closed. + +### Pull Requests Merged + +* [PR 102](https://github.com/foua-pps/level1c4pps/pull/102) - Fix VGAC sun earth distance applied attribute, by [@ninahakansson](https://github.com/ninahakansson) ([94](https://github.com/foua-pps/level1c4pps/issues/94)) +* [PR 100](https://github.com/foua-pps/level1c4pps/pull/100) - Sbaf n nv4, by [@salomoneliassonSMHI](https://github.com/salomoneliassonSMHI) +* [PR 99](https://github.com/foua-pps/level1c4pps/pull/99) - Add fix for sun_earth_distance_correction_factor, by [@ninahakansson](https://github.com/ninahakansson) +* [PR 97](https://github.com/foua-pps/level1c4pps/pull/97) - Sbaf nn, by [@salomoneliassonSMHI](https://github.com/salomoneliassonSMHI) +* [PR 96](https://github.com/foua-pps/level1c4pps/pull/96) - vgac time units, by [@BengtRydberg](https://github.com/BengtRydberg) ([93](https://github.com/foua-pps/level1c4pps/issues/93)) +* [PR 95](https://github.com/foua-pps/level1c4pps/pull/95) - vgac cli bugfix, by [@BengtRydberg](https://github.com/BengtRydberg) +* [PR 92](https://github.com/foua-pps/level1c4pps/pull/92) - removed check for pygac and numpy2.0 gac2pps_lib.py, by [@ikaur17](https://github.com/ikaur17) +* [PR 91](https://github.com/foua-pps/level1c4pps/pull/91) - adding support for handling mersi3 data, by [@BengtRydberg](https://github.com/BengtRydberg) +* [PR 90](https://github.com/foua-pps/level1c4pps/pull/90) - Adding v3 SBAFs, by [@salomoneliassonSMHI](https://github.com/salomoneliassonSMHI) + +In this release 9 pull requests were closed. + ## Version <0.2.34> (2024/08/29) ### Issues Closed From 5767d8e5b6a7bf440515c5c0bd1ea1b1eef47407 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:32:17 +0100 Subject: [PATCH 13/22] Fix ci --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 852ab17..ce77b7a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ jobs: python-version: ["3.9", "3.11", "3.12"] experimental: [false] include: - - python-version: "3.12" + - python-version: "3.13" os: "ubuntu-latest" experimental: true @@ -28,15 +28,15 @@ jobs: uses: actions/checkout@v3 - name: Setup Conda Environment - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true python-version: ${{ matrix.python-version }} environment-file: continuous_integration/environment.yaml activate-environment: test-environment - auto-update-conda: true + channels: conda-forge + conda-remove-defaults: true + channel-priority: strict - name: Install unstable dependencies if: matrix.experimental == true From 672657759f30bbb387d60638a7219f756845e46c Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:07:23 +0100 Subject: [PATCH 14/22] Do not test VGAC with NN-SBAFs on github --- continuous_integration/environment.yaml | 6 +----- level1c4pps/vgac2pps_lib.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index 3e0d840..795b326 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -2,15 +2,12 @@ name: test-environment channels: - conda-forge dependencies: - - tensorflow - - scikit-learn - sphinx - scipy - h5py - python-geotiepoints - - matplotlib - mock - - numpy<2.0.0 + - numpy - satpy>0.41.1 - pyspectral - h5netcdf @@ -21,4 +18,3 @@ dependencies: - pip: - trollsift - pygac - - git+https://github.com/foua-pps/sbafs_ann@main \ No newline at end of file diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 5f74a80..071dba0 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -28,7 +28,6 @@ import os import time from satpy.scene import Scene -from sbafs_ann.convert_vgac import convert_to_vgac_with_nn from level1c4pps import (get_encoding, compose_filename, set_header_and_band_attrs_defaults, rename_latitude_longitude, @@ -581,6 +580,7 @@ def convert_to_noaa19_neural_network(scene, sbaf_version): """Applies AVHRR SBAF to VGAC channels using NN approach""" + from sbafs_ann.convert_vgac import convert_to_vgac_with_nn if sbaf_version in ["NN_v1", "NN_v2", "NN_v3", "NN_v4"]: day_cfg_file = SBAF[sbaf_version]['cfg_file_day'] night_cfg_file = SBAF[sbaf_version]['cfg_file_night'] From 5bbfbd975b9f23963508d7adc75aafcf3d3b837d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:36:43 +0100 Subject: [PATCH 15/22] Move authors from headers to specific file AUTHORS.md --- AUTHORS.md | 12 ++++++++++++ bin/avhrr2pps.py | 5 ----- bin/eumgacfdr2pps.py | 6 ------ bin/gac2pps.py | 6 ------ bin/mersi2pps.py | 5 ----- bin/metimage2pps.py | 5 ----- bin/modis2pps.py | 5 ----- bin/seviri2pps.py | 6 ------ bin/slstr2pps.py | 5 ----- bin/vgac2pps.py | 6 ------ bin/viirs2pps.py | 5 ----- level1c4pps/__init__.py | 4 ---- level1c4pps/avhrr2pps_lib.py | 5 ----- level1c4pps/calibration_coefs.py | 4 ---- level1c4pps/eumgacfdr2pps_lib.py | 5 ----- level1c4pps/gac2pps_lib.py | 5 ----- level1c4pps/mersi2pps_lib.py | 6 ------ level1c4pps/metimage2pps_lib.py | 5 ----- level1c4pps/modis2pps_lib.py | 5 ----- level1c4pps/seviri2pps_lib.py | 6 ------ level1c4pps/slstr2pps_lib.py | 5 ----- level1c4pps/tests/__init__.py | 3 --- level1c4pps/tests/test_angles.py | 10 +++------- level1c4pps/tests/test_avhrr2pps.py | 4 ---- level1c4pps/tests/test_eumgacfdr2pps.py | 4 ---- level1c4pps/tests/test_gac2pps.py | 4 ---- level1c4pps/tests/test_mersi2pps.py | 4 ---- level1c4pps/tests/test_modis2pps.py | 4 ---- level1c4pps/tests/test_seviri2pps.py | 3 --- level1c4pps/tests/test_slstr2pps.py | 4 ---- level1c4pps/tests/test_vgac2pps.py | 3 --- level1c4pps/tests/test_viirs2pps.py | 4 ---- level1c4pps/vgac2pps_lib.py | 6 ------ level1c4pps/viirs2pps_lib.py | 5 ----- setup.py | 12 ++++-------- 35 files changed, 19 insertions(+), 167 deletions(-) create mode 100644 AUTHORS.md diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..9dd651f --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,12 @@ +# Project Contributors + +- [Nina Håkansson] (https://github.com/ninahakansson) +- [Bengt Rydberg] (BengtRydberg)](https://github.com/BengtRydberg) +- [Sara Hörnquist] (shornqui)](https://github.com/shornqui) +- [Inderpreet Kaur] (https://github.com/ikaur17) +- [Salomon Eliasson] (https://github.com/salomoneliassonSMHI) +- [Erik Johansson] (https://github.com/smhi-erik) +- [Stephan Finkensieper] (https://github.com/sfinkens) +- [Adam Dybbroe (adybbroe)] (https://github.com/adybbroe) +- [Martin Raspaud (mraspaud)] (https://github.com/mraspaud) +- [Panu Lahtinen (pnuu)] (https://github.com/pnuu) diff --git a/bin/avhrr2pps.py b/bin/avhrr2pps.py index ae284c0..666e831 100644 --- a/bin/avhrr2pps.py +++ b/bin/avhrr2pps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Script to convert AVHRR level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/bin/eumgacfdr2pps.py b/bin/eumgacfdr2pps.py index 5054446..f6fd767 100644 --- a/bin/eumgacfdr2pps.py +++ b/bin/eumgacfdr2pps.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe - """Script to make EUMETSAT GAC level1c in PPS-format with pytroll.""" diff --git a/bin/gac2pps.py b/bin/gac2pps.py index d1c8ca3..5697eb5 100755 --- a/bin/gac2pps.py +++ b/bin/gac2pps.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe - """Script to make seviri level1c in PPS-format with pytroll.""" diff --git a/bin/mersi2pps.py b/bin/mersi2pps.py index 4cc14b2..b9b4644 100755 --- a/bin/mersi2pps.py +++ b/bin/mersi2pps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Script to convert MERSI-2 level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/bin/metimage2pps.py b/bin/metimage2pps.py index 97e7693..21b3c34 100644 --- a/bin/metimage2pps.py +++ b/bin/metimage2pps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Script to convert METIMAGE level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/bin/modis2pps.py b/bin/modis2pps.py index 65c471e..982e5a3 100644 --- a/bin/modis2pps.py +++ b/bin/modis2pps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Script to convert MODIS level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/bin/seviri2pps.py b/bin/seviri2pps.py index 849a054..2034886 100755 --- a/bin/seviri2pps.py +++ b/bin/seviri2pps.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe -# Stephan Finkensieper # This program was developed by CMSAF to be used for the processing of # CLAAS3. diff --git a/bin/slstr2pps.py b/bin/slstr2pps.py index da3ab7b..e41bd10 100644 --- a/bin/slstr2pps.py +++ b/bin/slstr2pps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Script to convert SLSTR level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/bin/vgac2pps.py b/bin/vgac2pps.py index ecb9b88..90c351f 100644 --- a/bin/vgac2pps.py +++ b/bin/vgac2pps.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe -# Salomon.Eliasson """Script to convert VIIRS level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/bin/viirs2pps.py b/bin/viirs2pps.py index 96c108a..a1cc188 100755 --- a/bin/viirs2pps.py +++ b/bin/viirs2pps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Script to convert VIIRS level-1 to PPS level-1c format using Pytroll/Satpy.""" diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index f4865a7..15b392d 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Adam.Dybbroe -# Nina Hakansson """Package Initializer for level1c4pps.""" from importlib.metadata import version diff --git a/level1c4pps/avhrr2pps_lib.py b/level1c4pps/avhrr2pps_lib.py index 272d694..6b15c86 100644 --- a/level1c4pps/avhrr2pps_lib.py +++ b/level1c4pps/avhrr2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Functions to convert AVHRR AAPP or EPS l1b data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" diff --git a/level1c4pps/calibration_coefs.py b/level1c4pps/calibration_coefs.py index 3643623..a24d55f 100644 --- a/level1c4pps/calibration_coefs.py +++ b/level1c4pps/calibration_coefs.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# -*- coding: utf-8 -*- -# Author(s): - -# Nina.Hakansson """Module with calibration coefficients for SEVIRI.""" diff --git a/level1c4pps/eumgacfdr2pps_lib.py b/level1c4pps/eumgacfdr2pps_lib.py index 448d860..f33f32b 100644 --- a/level1c4pps/eumgacfdr2pps_lib.py +++ b/level1c4pps/eumgacfdr2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Utilities to convert AVHRR GAC formattet data to PPS level-1c format.""" diff --git a/level1c4pps/gac2pps_lib.py b/level1c4pps/gac2pps_lib.py index 8da5490..4209c7b 100644 --- a/level1c4pps/gac2pps_lib.py +++ b/level1c4pps/gac2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Utilities to convert AVHRR GAC formattet data to PPS level-1c format.""" diff --git a/level1c4pps/mersi2pps_lib.py b/level1c4pps/mersi2pps_lib.py index 366ac53..70f037b 100644 --- a/level1c4pps/mersi2pps_lib.py +++ b/level1c4pps/mersi2pps_lib.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe -# Bengt.Rydberg >bengt.rydberg@smhi.se> """Functions to convert MERSI-2/3 level-1 data to a NWCSAF/PPS level-1c formated netCDF/CF file.""" diff --git a/level1c4pps/metimage2pps_lib.py b/level1c4pps/metimage2pps_lib.py index c1de6b9..672bae5 100644 --- a/level1c4pps/metimage2pps_lib.py +++ b/level1c4pps/metimage2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Functions to convert EPS-SG MetImage level-1 data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" diff --git a/level1c4pps/modis2pps_lib.py b/level1c4pps/modis2pps_lib.py index 774f052..d064ea2 100644 --- a/level1c4pps/modis2pps_lib.py +++ b/level1c4pps/modis2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Functions to convert MODIS level-1 data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" diff --git a/level1c4pps/seviri2pps_lib.py b/level1c4pps/seviri2pps_lib.py index 8d402eb..c649427 100644 --- a/level1c4pps/seviri2pps_lib.py +++ b/level1c4pps/seviri2pps_lib.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe -# Stephan Finkensieper # This program was developed by CMSAF to be used for the processing of # CLAAS3. diff --git a/level1c4pps/slstr2pps_lib.py b/level1c4pps/slstr2pps_lib.py index bdd0f41..566f0c3 100644 --- a/level1c4pps/slstr2pps_lib.py +++ b/level1c4pps/slstr2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Functions to convert MERSI-2 level-1 data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" diff --git a/level1c4pps/tests/__init__.py b/level1c4pps/tests/__init__.py index 1de273b..e9747d7 100644 --- a/level1c4pps/tests/__init__.py +++ b/level1c4pps/tests/__init__.py @@ -16,9 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Nina Hakansson """Test Initializer for level1c4pps.""" diff --git a/level1c4pps/tests/test_angles.py b/level1c4pps/tests/test_angles.py index bb13cac..fbf079a 100644 --- a/level1c4pps/tests/test_angles.py +++ b/level1c4pps/tests/test_angles.py @@ -1,12 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -# Copyright (c) 2019 Adam.Dybbroe - -# Author(s): - -# Adam.Dybbroe - +# Copyright (c) 2019 level1c4pps developers +# +# This file is part of level1c4pps # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/level1c4pps/tests/test_avhrr2pps.py b/level1c4pps/tests/test_avhrr2pps.py index e036cd8..da8a161 100644 --- a/level1c4pps/tests/test_avhrr2pps.py +++ b/level1c4pps/tests/test_avhrr2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the avhrr2pps_lib module.""" diff --git a/level1c4pps/tests/test_eumgacfdr2pps.py b/level1c4pps/tests/test_eumgacfdr2pps.py index fd519df..fe51bf9 100644 --- a/level1c4pps/tests/test_eumgacfdr2pps.py +++ b/level1c4pps/tests/test_eumgacfdr2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the eumgacfdr2pps_lib module.""" diff --git a/level1c4pps/tests/test_gac2pps.py b/level1c4pps/tests/test_gac2pps.py index e2ed1eb..e0c1a07 100644 --- a/level1c4pps/tests/test_gac2pps.py +++ b/level1c4pps/tests/test_gac2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the gac2pps_lib module.""" diff --git a/level1c4pps/tests/test_mersi2pps.py b/level1c4pps/tests/test_mersi2pps.py index d526143..613abdf 100644 --- a/level1c4pps/tests/test_mersi2pps.py +++ b/level1c4pps/tests/test_mersi2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the merci2pps_lib module.""" diff --git a/level1c4pps/tests/test_modis2pps.py b/level1c4pps/tests/test_modis2pps.py index c9be9e4..68a16a1 100644 --- a/level1c4pps/tests/test_modis2pps.py +++ b/level1c4pps/tests/test_modis2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the gac2pps_lib module.""" diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index 337903f..6279679 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -16,9 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper """Unit tests for the seviri2pps_lib module.""" diff --git a/level1c4pps/tests/test_slstr2pps.py b/level1c4pps/tests/test_slstr2pps.py index 95da6a5..1373ef3 100644 --- a/level1c4pps/tests/test_slstr2pps.py +++ b/level1c4pps/tests/test_slstr2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the gac2pps_lib module.""" diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index aa9ecfd..3a40ef3 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -16,9 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Nina Hakansson """Unit tests for the vgac2pps_lib module.""" diff --git a/level1c4pps/tests/test_viirs2pps.py b/level1c4pps/tests/test_viirs2pps.py index c7202ff..adab6d0 100644 --- a/level1c4pps/tests/test_viirs2pps.py +++ b/level1c4pps/tests/test_viirs2pps.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Stephan Finkensieper -# Nina Hakansson """Unit tests for the gac2pps_lib module.""" diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 071dba0..ce0eeac 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -16,12 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe -# Salomom Eliasson """Functions to convert VGAC level-1c data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" diff --git a/level1c4pps/viirs2pps_lib.py b/level1c4pps/viirs2pps_lib.py index fa9f6d4..e5a67ad 100644 --- a/level1c4pps/viirs2pps_lib.py +++ b/level1c4pps/viirs2pps_lib.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with level1c4pps. If not, see . -# Author(s): - -# Martin Raspaud -# Nina Hakansson -# Adam.Dybbroe """Functions to convert VIIRS level-1 data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" diff --git a/setup.py b/setup.py index 4764bb1..a5ab7ad 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2019, 2021 Pytroll +# Copyright (c) 2019 level1c4pps developers -# Author(s): - -# Nina Håkansson -# Erik Johansson -# Adam Dybbroe +# Author(s): AUTHORS.md # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -46,8 +42,8 @@ # version=version.__version__, description='Tools to convert various satellite level-1 data formats to NWCSAF/PPS level-1c format', long_description=README, - author='Nina Hakansson', - author_email='nina.hakansson@smhi.se', + author='NWCSAF PPS Team et al.', + author_email='6160529+ninahakansson@users.noreply.github.com', classifiers=["Development Status :: 3 - Alpha", "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU General Public License v3 " + From c6a138d6cec7d1e76b597117a8b67af61c5f261f Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:38:29 +0100 Subject: [PATCH 16/22] update gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4541ca2..e50f684 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ build +dist *py~ *pyc *~ -*.egg* \ No newline at end of file +*.egg* +*#* \ No newline at end of file From 9ed48cbdfabff18e06a65ac26e24ff194e67dc75 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:02:49 +0100 Subject: [PATCH 17/22] Test SBAF-NN for vgac if sbaf_ann is installed --- level1c4pps/tests/test_vgac2pps.py | 50 +++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 3a40ef3..92b98c8 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -30,7 +30,16 @@ import level1c4pps.vgac2pps_lib as vgac2pps import numpy as np +import pytest +try: + import sbafs_ann + no_sbaf_module = False +except ModuleNotFoundError: + no_sbaf_module = True + + + class TestVgac2PPS(unittest.TestCase): """Test vgac2pps_lib.""" @@ -93,7 +102,6 @@ def test_set_header_and_band_attrs(self): def test_process_one_scene(self): """Test process one scene for one example file.""" - vgac2pps.process_one_scene( ['./level1c4pps/tests/VGAC_VJ102MOD_A2018305_1042_n004946_K005.nc'], out_path='./level1c4pps/tests/' @@ -120,6 +128,7 @@ def test_process_one_scene(self): np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor, 1.0, decimal=4) + self.assertTrue(pps_nc.variables['image1'].sun_earth_distance_correction_applied) assert ( pps_nc.variables["scanline_timestamps"].units == "milliseconds since 1970-01-01" and pps_nc.variables["scanline_timestamps"].dtype == "int" @@ -161,6 +170,45 @@ def test_process_one_scene_n19(self): np.testing.assert_equal(pps_nc.__dict__["platform"], "vgac20") self.assertTrue(np.abs(pps_nc.variables['image1'][0, 0, 0] - pps_nc_viirs.variables['image1'][0, 0, 0]) > 0.01) + @unittest.skipIf(no_sbaf_module, "Install sbafs_ann to test NN-SBAFS.") + def test_process_one_scene_n19_nn(self): + """Test process one scene for one example file.""" + import sbafs_ann + + vgac2pps.process_one_scene( + ['./level1c4pps/tests/VGAC_VJ102MOD_A2018305_1042_n004946_K005.nc'], + out_path='./level1c4pps/tests/', + noaa19_sbaf_version='NN_v4' + ) + filename = './level1c4pps/tests/S_NWC_avhrr_vgac20_00000_20181101T1042080Z_20181101T1224090Z.nc' + filename_viirs = './level1c4pps/tests/S_NWC_viirs_noaa20_00000_20181101T1042080Z_20181101T1224090Z.nc' + # written with hfnetcdf read with NETCDF4 ensure compatability + pps_nc = netCDF4.Dataset(filename, 'r', format='NETCDF4') # Check compatability implicitly + pps_nc_viirs = netCDF4.Dataset(filename_viirs, 'r', format='NETCDF4') # Check compatability implicitly + + for key in ['start_time', 'end_time', 'history', 'instrument', + 'orbit_number', 'platform', + 'sensor', 'source']: + if key not in pps_nc.__dict__.keys(): + print("Missing in attributes:", key) + self.assertTrue(key in pps_nc.__dict__.keys()) + + expected_vars = ['satzenith', 'azimuthdiff', + 'satazimuth', 'sunazimuth', 'sunzenith', + 'lon', 'lat', + 'image1', 'image2', 'image3', 'image4', 'image5', + 'scanline_timestamps', 'time', 'time_bnds'] + + for var in expected_vars: + self.assertTrue(var in pps_nc.variables.keys()) + + np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor, + 1.0, decimal=4) + + np.testing.assert_equal(pps_nc.__dict__["platform"], "vgac20") + self.assertTrue(np.abs(pps_nc.variables['image1'][0, 0, 400] - pps_nc_viirs.variables['image1'][0, 0, 400]) > 0.01) + + def test_process_one_scene_midnight(self): """Test process one scene for one example file.""" From 9ecde69176c3bf79aa641c2ee3f946cff46f300d Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:12:58 +0100 Subject: [PATCH 18/22] Fix depricated utcnow --- level1c4pps/__init__.py | 2 +- level1c4pps/seviri2pps_lib.py | 8 ++++---- level1c4pps/tests/test_eumgacfdr2pps.py | 6 +++--- level1c4pps/tests/test_gac2pps.py | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 15b392d..48d2449 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -409,7 +409,7 @@ def set_header_and_band_attrs_defaults(scene, BANDNAMES, PPS_TAGNAMES, REFL_BAND sensor_name = (fix_too_great_attributes(sensor_name)).upper() scene.attrs['sensor'] = sensor_name.upper() scene.attrs['instrument'] = sensor_name.upper() - nowutc = datetime.utcnow() + nowutc = datetime.now(timezone.utc) scene.attrs['orbit_number'] = int(orbit_n) scene.attrs['date_created'] = nowutc.strftime("%Y-%m-%dT%H:%M:%SZ") scene.attrs['version_level1c4pps_satpy'] = satpy.__version__ diff --git a/level1c4pps/seviri2pps_lib.py b/level1c4pps/seviri2pps_lib.py index c649427..867c23a 100644 --- a/level1c4pps/seviri2pps_lib.py +++ b/level1c4pps/seviri2pps_lib.py @@ -29,7 +29,7 @@ import dask.array as da from glob import glob import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from satpy.scene import Scene import satpy.utils from trollsift.parser import globify, Parser @@ -206,10 +206,10 @@ def get_satellite_angles(dataset, lons, lats): # => There have been updates to SatPy and this script # need to be modified. if not (get_observer_look(0, 0, 36000*1000, - datetime.utcnow(), np.array([16]), + datetime.now(timezone.utc), np.array([16]), np.array([58]), np.array([0]))[1] > 30 and get_observer_look(0, 0, 36000, - datetime.utcnow(), np.array([16]), + datetime.now(timezone.utc), np.array([16]), np.array([58]), np.array([0]))[1] < 23 and sat_alt > 38000): raise UnexpectedSatpyVersion( @@ -239,7 +239,7 @@ def set_attrs(scene): scene.attrs['instrument'] = 'SEVIRI' scene.attrs['source'] = "seviri2pps.py" scene.attrs['orbit_number'] = 99999 - nowutc = datetime.utcnow() + nowutc = datetime.now(timezone.utc) scene.attrs['date_created'] = nowutc.strftime("%Y-%m-%dT%H:%M:%SZ") # For each band diff --git a/level1c4pps/tests/test_eumgacfdr2pps.py b/level1c4pps/tests/test_eumgacfdr2pps.py index fe51bf9..4c3f31b 100644 --- a/level1c4pps/tests/test_eumgacfdr2pps.py +++ b/level1c4pps/tests/test_eumgacfdr2pps.py @@ -21,7 +21,7 @@ import netCDF4 import unittest -from datetime import datetime +from datetime import datetime, timezone try: from unittest import mock except ImportError: @@ -43,8 +43,8 @@ def setUp(self): ir_108 = mock.MagicMock(attrs={'name': 'image1', 'id_tag': 'ch_tb11', 'wavelength': [1, 2, 3, 'um'], - 'start_time': datetime.utcnow(), - 'end_time': datetime.utcnow(), + 'start_time': datetime.now(timezone.utc), + 'end_time': datetime.now(timezone.utc), 'history': 'dummy', 'platform_name': 'tirosn', 'orbit_number': 99999}) diff --git a/level1c4pps/tests/test_gac2pps.py b/level1c4pps/tests/test_gac2pps.py index e0c1a07..e41b585 100644 --- a/level1c4pps/tests/test_gac2pps.py +++ b/level1c4pps/tests/test_gac2pps.py @@ -22,7 +22,7 @@ import datetime as dt import netCDF4 import unittest -from datetime import datetime +from datetime import datetime, timezone try: from unittest import mock except ImportError: @@ -44,8 +44,8 @@ def setUp(self): ir_108 = mock.MagicMock(attrs={'name': 'image1', 'id_tag': 'ch_tb11', 'wavelength': [1, 2, 3, 'um'], - 'start_time': datetime.utcnow(), - 'end_time': datetime.utcnow(), + 'start_time': datetime.now(timezone.utc), + 'end_time': datetime.now(timezone.utc), 'platform_name': 'tirosn', 'orbit_number': 99999}) qual_f = mock.MagicMock(attrs={'name': 'qual_flags', From 25cd69a2ca008a7f57875794586dc8fafae2cf04 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:17:01 +0100 Subject: [PATCH 19/22] Update pull request tempate --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6c9db04..de4ebe9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,7 @@ - - [ ] Closes #xxxx + - [ ] Closes #xxxx - [ ] Tests passed: Passes ``pytest level1c4pps`` - [ ] Passes ``flake8`` - - [ ] Fully documented + - [ ] Add your name to `AUTHORS.md` if not there already \ No newline at end of file From 1c1baed0b3810c7971f9700bbd0a1d10cb6f0533 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:19:48 +0100 Subject: [PATCH 20/22] autopep8 --- level1c4pps/__init__.py | 9 +++++---- level1c4pps/gac2pps_lib.py | 1 - level1c4pps/mersi2pps_lib.py | 2 +- level1c4pps/seviri2pps_lib.py | 4 ++-- level1c4pps/tests/test_seviri2pps.py | 7 +++---- level1c4pps/tests/test_vgac2pps.py | 6 ++---- level1c4pps/vgac2pps_lib.py | 26 +++++++++++++------------- 7 files changed, 26 insertions(+), 29 deletions(-) diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 48d2449..779887a 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -207,7 +207,7 @@ def convert_angles(scene, delete_azimuth=False): 'standard_name': "longitude", 'units': 'degrees_east', 'valid_range': np.array([-180, 180], dtype='float32')} - } +} def make_azidiff_angle(sata, suna, divisor=360): @@ -364,20 +364,21 @@ def adjust_lons_to_valid_range(scene): # scene['lon'] = centered_modulus(scene['lon']) # makes lon loose attrs satpy 0.24.0 scene['lon'].values = centered_modulus(scene['lon'].values) + def fix_sun_earth_distance_correction_factor(scene, band, start_time): from pyorbital.astronomy import sun_earth_distance_correction date_control = np.datetime64("2019-01-01T00:00:00") sun_earth_distance_20190409 = sun_earth_distance_correction(date_control) sun_earth_distance = sun_earth_distance_correction(start_time) if (np.abs(sun_earth_distance_20190409 - 0.9833280675966011) < 0.00001 and - np.abs(sun_earth_distance - scene[band].attrs['sun_earth_distance_correction_factor']) < 0.00001): + np.abs(sun_earth_distance - scene[band].attrs['sun_earth_distance_correction_factor']) < 0.00001): logger.info("The sun earth distance correction attribute contain the sun earth distance, not the square.") logger.info("Updating and adding sun earth distance correction attributes.") current_factor = scene[band].attrs['sun_earth_distance_correction_factor'] - scene[band].attrs['satpy_sun_earth_distance_correction_factor'] = current_factor + scene[band].attrs['satpy_sun_earth_distance_correction_factor'] = current_factor scene[band].attrs['pps_sun_earth_distance_correction_factor'] = sun_earth_distance * sun_earth_distance scene[band].attrs['sun_earth_distance'] = sun_earth_distance - scene[band].attrs['sun_earth_distance_correction_factor'] = sun_earth_distance * sun_earth_distance + scene[band].attrs['sun_earth_distance_correction_factor'] = sun_earth_distance * sun_earth_distance def set_header_and_band_attrs_defaults(scene, BANDNAMES, PPS_TAGNAMES, REFL_BANDS, irch, orbit_n=0): diff --git a/level1c4pps/gac2pps_lib.py b/level1c4pps/gac2pps_lib.py index 4209c7b..88ff10d 100644 --- a/level1c4pps/gac2pps_lib.py +++ b/level1c4pps/gac2pps_lib.py @@ -38,7 +38,6 @@ logger = logging.getLogger('gac2pps') - BANDNAMES = ['1', '2', '3', '3a', '3b', '4', '5'] REFL_BANDS = ['1', '2', '3a'] diff --git a/level1c4pps/mersi2pps_lib.py b/level1c4pps/mersi2pps_lib.py index 70f037b..90ceca4 100644 --- a/level1c4pps/mersi2pps_lib.py +++ b/level1c4pps/mersi2pps_lib.py @@ -119,7 +119,7 @@ def process_one_scene(scene_files, out_path, engine='h5netcdf', orbit_n=0): update_angle_attributes(scene, band) for angle in ['sunzenith', 'satzenith', 'azimuthdiff']: scene[angle].attrs['file_key'] = ANGLE_ATTRIBUTES['mersi_file_key'][angle] - filename=compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=band) + filename = compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=band) scene.save_datasets( writer='cf', filename=filename, diff --git a/level1c4pps/seviri2pps_lib.py b/level1c4pps/seviri2pps_lib.py index 867c23a..0b09f59 100644 --- a/level1c4pps/seviri2pps_lib.py +++ b/level1c4pps/seviri2pps_lib.py @@ -256,9 +256,9 @@ def set_attrs(scene): scene[band].attrs['sun_earth_distance_correction_factor'] = 1.0 else: fix_sun_earth_distance_correction_factor(scene, band, scene[band].attrs['start_time']) - + scene[band].attrs['sun_zenith_angle_correction_applied'] = False - + scene[band].attrs['name'] = "image{:d}".format(image_num) # Cosmetics diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index 6279679..d328d53 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -91,7 +91,7 @@ def test_load_and_calibrate(self, mocked_scene): res['VIS006'] = remove_earthsun_distance_correction(res['VIS006']) xr.testing.assert_allclose(res['VIS006'], vis006_exp) xr.testing.assert_equal(res['IR_108'], ir_108_exp) - + self.assertFalse( res['VIS006'].attrs['sun_earth_distance_correction_applied'], ) @@ -206,7 +206,7 @@ def test_set_attrs(self): scene['VIS006'].attrs['sun_earth_distance_correction_factor'] = 0.9833241577909706 scene['IR_108'].attrs['orbital_parameters'] = {'orb_a': 1, 'orb_b': 2} - scene['IR_108'].attrs['georef_offset_corrected'] = True + scene['IR_108'].attrs['georef_offset_corrected'] = True scene['IR_108'].attrs['platform_name'] = "my_platform_name" seviri2pps.set_attrs(scene) @@ -228,7 +228,6 @@ def test_set_attrs(self): self.assertAlmostEqual(scene['VIS006'].sun_earth_distance, sun_earth_distance, places=7) - def test_get_mean_acq_time(self): """Test computation of mean scanline acquisition time.""" seviri2pps.BANDNAMES = ['VIS006', 'IR_108'] @@ -438,7 +437,7 @@ def test_get_encoding(self): encoding = seviri2pps.get_encoding_seviri(scene) for key in encoding_exp: print(key) - print(encoding[key],encoding_exp[key]) + print(encoding[key], encoding_exp[key]) self.assertDictEqual(encoding[key], encoding_exp[key]) diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 92b98c8..578005c 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -39,8 +39,6 @@ no_sbaf_module = True - - class TestVgac2PPS(unittest.TestCase): """Test vgac2pps_lib.""" @@ -206,8 +204,8 @@ def test_process_one_scene_n19_nn(self): 1.0, decimal=4) np.testing.assert_equal(pps_nc.__dict__["platform"], "vgac20") - self.assertTrue(np.abs(pps_nc.variables['image1'][0, 0, 400] - pps_nc_viirs.variables['image1'][0, 0, 400]) > 0.01) - + self.assertTrue(np.abs(pps_nc.variables['image1'][0, 0, 400] - + pps_nc_viirs.variables['image1'][0, 0, 400]) > 0.01) def test_process_one_scene_midnight(self): """Test process one scene for one example file.""" diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index ce0eeac..a597a76 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -172,7 +172,7 @@ "slope": 1.003, "offset": -0.84, "comment": "based on nadir collocation data for SZA 0-180", - }, + }, }, "v4": { "r06": { @@ -322,19 +322,19 @@ "r06": { "viirs_channel": "M05", "slope": 0.8534, - "offset":1.8517, + "offset": 1.8517, "comment": "Based on collocation data for VZA < 15", }, "r09": { "viirs_channel": "M07", - "slope":0.8507, - "offset":1.1157, + "slope": 0.8507, + "offset": 1.1157, "comment": "Based on collocation data for VZA < 15", }, "tb37": { "viirs_channel": "M12", - "slope":0.9734, - "offset":6.1707, + "slope": 0.9734, + "offset": 6.1707, "comment": "Based on collocation data for VZA < 15", }, "tb11": { @@ -549,19 +549,19 @@ "cfg_file_night": "ch4_SATZ_less_25_SUNZ_90_180_TD_5_min.yaml", "cfg_file_twilight": None, "comment": "NN based on AVHRR and VGAC matchups using all AVHRR heritage channels" - }, + }, "NN_v2": { "cfg_file_day": "ch7_satz_max_25_SUNZ_0_80_tdiff_300_sec_20241031.yaml", "cfg_file_night": "ch4_satz_max_25_SUNZ_90_180_tdiff_300_sec_20241031.yaml", "cfg_file_twilight": "ch7_satz_max_25_SUNZ_80_89_tdiff_300_sec_20241031.yaml", "comment": "NN based on AVHRR and VGAC matchups using all AVHRR heritage channels" - }, + }, "NN_v3": { "cfg_file_day": "ch7_satz_max_15_SUNZ_0_80_tdiff_120_sec_20241120.yaml", "cfg_file_night": "ch4_satz_max_15_SUNZ_90_180_tdiff_120_sec_20241120.yaml", "cfg_file_twilight": "ch7_satz_max_15_SUNZ_80_89_tdiff_120_sec_20241120.yaml", "comment": "NN based on AVHRR and VGAC matchups using all AVHRR heritage channels" - }, + }, "NN_v4": { "cfg_file_day": "ch7_satz_max_15_SUNZ_0_80_tdiff_120_sec_20241204.yaml", "cfg_file_night": "ch4_satz_max_15_SUNZ_90_180_tdiff_120_sec_20241204.yaml", @@ -574,7 +574,7 @@ def convert_to_noaa19_neural_network(scene, sbaf_version): """Applies AVHRR SBAF to VGAC channels using NN approach""" - from sbafs_ann.convert_vgac import convert_to_vgac_with_nn + from sbafs_ann.convert_vgac import convert_to_vgac_with_nn if sbaf_version in ["NN_v1", "NN_v2", "NN_v3", "NN_v4"]: day_cfg_file = SBAF[sbaf_version]['cfg_file_day'] night_cfg_file = SBAF[sbaf_version]['cfg_file_night'] @@ -628,7 +628,7 @@ def convert_to_noaa19_KNMI_v2(scene, sbaf_version): filt, slope * scene[viirs_channel].values + offset, scene[viirs_channel].values - ) + ) logger.info(f"{avhrr_chan:<13} = {slope:<6}*{viirs_channel:<3}+{offset:<5} ({comment})") else: if avhrr_chan == "tb37_twilight": @@ -847,7 +847,7 @@ def process_one_scene(scene_files, out_path, engine="h5netcdf", flatten_attrs=True, encoding=encoding) logger.info("Saved file {:s} after {:3.1f} seconds".format( - os.path.basename(filename), - time.time()-tic)) + os.path.basename(filename), + time.time()-tic)) filenames.append(filename) return filenames From 884fbc8e229a9d229e4c0b627d8603ea074b8b12 Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:24:17 +0100 Subject: [PATCH 21/22] autopep8 --- bin/eumgacfdr2pps.py | 2 +- bin/viirs2pps.py | 2 +- level1c4pps/__init__.py | 6 ++-- level1c4pps/avhrr2pps_lib.py | 8 ++--- level1c4pps/eumgacfdr2pps_lib.py | 2 +- level1c4pps/gac2pps_lib.py | 2 +- level1c4pps/metimage2pps_lib.py | 2 +- level1c4pps/modis2pps_lib.py | 20 +++++------ level1c4pps/seviri2pps_lib.py | 4 +-- level1c4pps/slstr2pps_lib.py | 4 +-- level1c4pps/tests/__init__.py | 2 +- level1c4pps/tests/test_angles.py | 46 ++++++++++++------------- level1c4pps/tests/test_eumgacfdr2pps.py | 4 +-- level1c4pps/tests/test_gac2pps.py | 4 +-- level1c4pps/tests/test_seviri2pps.py | 4 +-- level1c4pps/tests/test_vgac2pps.py | 4 +-- level1c4pps/vgac2pps_lib.py | 15 ++++---- level1c4pps/viirs2pps_lib.py | 12 +++---- 18 files changed, 72 insertions(+), 71 deletions(-) diff --git a/bin/eumgacfdr2pps.py b/bin/eumgacfdr2pps.py index f6fd767..ef586dc 100644 --- a/bin/eumgacfdr2pps.py +++ b/bin/eumgacfdr2pps.py @@ -52,7 +52,7 @@ required=False, default=99999, help="Orbit number (default is 99999).") options = parser.parse_args() - process_one_file(options.file, options.out_dir, start_line=options.start_line, + process_one_file(options.file, options.out_dir, start_line=options.start_line, end_line=options.end_line, engine=options.nc_engine, remove_broken=not options.no_remove_bad, orbit_n=options.orbit_number) diff --git a/bin/viirs2pps.py b/bin/viirs2pps.py index a1cc188..98f77cf 100755 --- a/bin/viirs2pps.py +++ b/bin/viirs2pps.py @@ -36,7 +36,7 @@ help="Output directory where to store the level1c file") parser.add_argument('--iband', action='store_true', help="Iband resolution include I01-I03, M15-M16 and optional M09, M14") - parser.add_argument('--reader', type=str, nargs='?', + parser.add_argument('--reader', type=str, nargs='?', required=False, default="viirs_sdr", help="VIIRS reader default: viirs_sdr") parser.add_argument('-ne', '--nc_engine', type=str, nargs='?', diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 779887a..20e7eee 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -180,12 +180,12 @@ def convert_angles(scene, delete_azimuth=False): 'sunazimuth': np.array([-18000, 18000], dtype='int16'), 'satazimuth': np.array([-18000, 18000], dtype='int16'), }, - 'mersi_file_key': { + 'mersi_file_key': { 'sunzenith': 'Geolocation/SolarZenithAngle', 'satzenith': 'Geolocation/SensorZenithAngle', 'azimuthdiff': 'Geolocation/SensorSolarAzimuthDifference', }, - 'standard_name': { + 'standard_name': { 'sunzenith': 'solar_zenith_angle', 'satzenith': 'sensor_zenith_angle', # platform in ppsv2018 'azimuthdiff': 'absolute_angle_of_rotation_from_solar_azimuth_to_platform_azimuth', @@ -212,7 +212,7 @@ def convert_angles(scene, delete_azimuth=False): def make_azidiff_angle(sata, suna, divisor=360): """Calculate azimuth difference angle.""" - daz = abs(sata-suna) + daz = abs(sata - suna) half_divisor = divisor / 2.0 daz = daz % divisor if isinstance(daz, np.ndarray): diff --git a/level1c4pps/avhrr2pps_lib.py b/level1c4pps/avhrr2pps_lib.py index 6b15c86..11be61b 100644 --- a/level1c4pps/avhrr2pps_lib.py +++ b/level1c4pps/avhrr2pps_lib.py @@ -45,9 +45,9 @@ REFL_BANDS = ['1', '2', '3a'] -PPS_TAGNAMES = {'1': 'ch_r06', - '2': 'ch_r09', - '3a': 'ch_r16', +PPS_TAGNAMES = {'1': 'ch_r06', + '2': 'ch_r09', + '3a': 'ch_r16', '3b': 'ch_tb37', '4': 'ch_tb11', '5': 'ch_tb12'} @@ -128,5 +128,5 @@ def process_one_scene(scene_files, out_path, engine='h5netcdf', orbit_n=0): encoding=get_encoding_avhrr(scn_)) print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) return filename diff --git a/level1c4pps/eumgacfdr2pps_lib.py b/level1c4pps/eumgacfdr2pps_lib.py index f33f32b..31b9508 100644 --- a/level1c4pps/eumgacfdr2pps_lib.py +++ b/level1c4pps/eumgacfdr2pps_lib.py @@ -244,5 +244,5 @@ def process_one_file(eumgacfdr_file, out_path='.', reader_kwargs=None, logger.info("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) return filename diff --git a/level1c4pps/gac2pps_lib.py b/level1c4pps/gac2pps_lib.py index 88ff10d..6cf5780 100644 --- a/level1c4pps/gac2pps_lib.py +++ b/level1c4pps/gac2pps_lib.py @@ -165,5 +165,5 @@ def process_one_file(gac_file, out_path='.', reader_kwargs=None, engine='h5netcd print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) return filename diff --git a/level1c4pps/metimage2pps_lib.py b/level1c4pps/metimage2pps_lib.py index 672bae5..4fb1daa 100644 --- a/level1c4pps/metimage2pps_lib.py +++ b/level1c4pps/metimage2pps_lib.py @@ -180,5 +180,5 @@ def process_one_scene(scene_files, out_path, encoding=get_encoding_metimage(scn_)) print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) return filename diff --git a/level1c4pps/modis2pps_lib.py b/level1c4pps/modis2pps_lib.py index d064ea2..5e9867f 100644 --- a/level1c4pps/modis2pps_lib.py +++ b/level1c4pps/modis2pps_lib.py @@ -53,24 +53,24 @@ ANGLE_NAMES = ['satellite_zenith_angle', 'solar_zenith_angle', 'satellite_azimuth_angle', 'solar_azimuth_angle'] -PPS_TAGNAMES = {'1': 'ch_r06', - '2': 'ch_r09', +PPS_TAGNAMES = {'1': 'ch_r06', + '2': 'ch_r09', '26': 'ch_r13', - '6': 'ch_r16', + '6': 'ch_r16', '20': 'ch_tb37', '29': 'ch_tb85', '31': 'ch_tb11', '32': 'ch_tb12', # Not used yet: - '7': 'ch_r21', + '7': 'ch_r21', '27': 'ch_tb67', '28': 'ch_tb73', '33': 'ch_tb133', - '3': 'ch_rxx', - '4': 'ch_rxx', - '5': 'ch_rxx', - '8': 'ch_rxx', - '9': 'ch_rxx', + '3': 'ch_rxx', + '4': 'ch_rxx', + '5': 'ch_rxx', + '8': 'ch_rxx', + '9': 'ch_rxx', '10': 'ch_rxx', '11': 'ch_rxx', '12': 'ch_rxx', @@ -152,5 +152,5 @@ def process_one_scene(scene_files, out_path, engine='h5netcdf', all_channels=Fal encoding=get_encoding_modis(scn_)) print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) return filename diff --git a/level1c4pps/seviri2pps_lib.py b/level1c4pps/seviri2pps_lib.py index 0b09f59..9bcdff8 100644 --- a/level1c4pps/seviri2pps_lib.py +++ b/level1c4pps/seviri2pps_lib.py @@ -205,7 +205,7 @@ def get_satellite_angles(dataset, lons, lats): # else: # => There have been updates to SatPy and this script # need to be modified. - if not (get_observer_look(0, 0, 36000*1000, + if not (get_observer_look(0, 0, 36000 * 1000, datetime.now(timezone.utc), np.array([16]), np.array([58]), np.array([0]))[1] > 30 and get_observer_look(0, 0, 36000, @@ -624,7 +624,7 @@ def process_one_scan(tslot_files, out_path, rotate=True, engine='h5netcdf', exclude_attrs=['raw_metadata']) print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) # About 40 seconds + time.time() - tic)) # About 40 seconds return filename diff --git a/level1c4pps/slstr2pps_lib.py b/level1c4pps/slstr2pps_lib.py index 566f0c3..0b39436 100644 --- a/level1c4pps/slstr2pps_lib.py +++ b/level1c4pps/slstr2pps_lib.py @@ -76,7 +76,7 @@ 'F2': 'ch_tbxx'} BANDNAMES_PPS = ['S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8', 'S9'] -BANDNAMES_DEFAULT = ['S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8', 'S9'] +BANDNAMES_DEFAULT = ['S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8', 'S9'] def get_encoding_slstr(scene): @@ -137,4 +137,4 @@ def process_one_scene(scene_files, out_path, engine='h5netcdf', encoding=get_encoding_slstr(scn_)) print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) diff --git a/level1c4pps/tests/__init__.py b/level1c4pps/tests/__init__.py index e9747d7..1260ddb 100644 --- a/level1c4pps/tests/__init__.py +++ b/level1c4pps/tests/__init__.py @@ -22,7 +22,7 @@ import unittest from level1c4pps.tests import (test_angles, test_seviri2pps, test_gac2pps, - test_mersi2pps, test_modis2pps, test_slstr2pps, + test_mersi2pps, test_modis2pps, test_slstr2pps, test_viirs2pps, test_eumgacfdr2pps, test_avhrr2pps, test_init) diff --git a/level1c4pps/tests/test_angles.py b/level1c4pps/tests/test_angles.py index fbf079a..18b4ab4 100644 --- a/level1c4pps/tests/test_angles.py +++ b/level1c4pps/tests/test_angles.py @@ -33,38 +33,38 @@ else: import unittest -SAT_AZ = np.ma.array([[48.0, 56.0, 64.0, 72.0], - [80.0, 88.0, 96.0, 104.0], - [-80.0, -88.0, -96.0, -104.0], - [-180.0, -188.0, -196.0, -204.0], - [261.88999414630234, 266.1, 271.1, 276.1]], mask=False) -XSAT_AZ = xr.DataArray([[48.0, 56.0, 64.0, 72.0], - [80.0, 88.0, 96.0, 104.0], - [-80.0, -88.0, -96.0, -104.0], - [-180.0, -188.0, -196.0, -204.0], +SAT_AZ = np.ma.array([[48.0, 56.0, 64.0, 72.0], + [80.0, 88.0, 96.0, 104.0], + [-80.0, -88.0, -96.0, -104.0], + [-180.0, -188.0, -196.0, -204.0], + [261.88999414630234, 266.1, 271.1, 276.1]], mask=False) +XSAT_AZ = xr.DataArray([[48.0, 56.0, 64.0, 72.0], + [80.0, 88.0, 96.0, 104.0], + [-80.0, -88.0, -96.0, -104.0], + [-180.0, -188.0, -196.0, -204.0], [261.88999414630234, 266.1, 271.1, 276.1]]) -SUN_AZ = np.ma.array([[148.0, 156.0, 164.0, 172.0], - [180.0, 188.0, 196.0, 204.0], - [180.0, 188.0, 196.0, 204.0], - [185.0, 193.0, 201.0, 209.0], +SUN_AZ = np.ma.array([[148.0, 156.0, 164.0, 172.0], + [180.0, 188.0, 196.0, 204.0], + [180.0, 188.0, 196.0, 204.0], + [185.0, 193.0, 201.0, 209.0], [196.77999560162425, 201.1, 206.1, 211.1]], mask=False) -XSUN_AZ = xr.DataArray([[148.0, 156.0, 164.0, 172.0], - [180.0, 188.0, 196.0, 204.0], - [180.0, 188.0, 196.0, 204.0], - [185.0, 193.0, 201.0, 209.0], +XSUN_AZ = xr.DataArray([[148.0, 156.0, 164.0, 172.0], + [180.0, 188.0, 196.0, 204.0], + [180.0, 188.0, 196.0, 204.0], + [185.0, 193.0, 201.0, 209.0], [196.77999560162425, 201.1, 206.1, 211.1]]) RES = np.ma.array([[100., 100., 100., 100.], [100., 100., 100., 100.], - [100., 84., 68., 52.], - [5., 21., 37., 53.], + [100., 84., 68., 52.], + [5., 21., 37., 53.], [65.10999854, 65., 65., 65.]], mask=False) XRES = xr.DataArray([[100., 100., 100., 100.], [100., 100., 100., 100.], - [100., 84., 68., 52.], - [5., 21., 37., 53.], + [100., 84., 68., 52.], + [5., 21., 37., 53.], [65.10999854, 65., 65., 65.]]) @@ -86,8 +86,8 @@ class TestUpdateAnglesAttribute(unittest.TestCase): def test_update_angle_attributes(self): """Test setting of attributes for angles.""" - band = mock.MagicMock(attrs={'start_time': dt.datetime(2009, 7, 1, 12, 15), - 'end_time': dt.datetime(2009, 7, 1, 12, 16)}) + band = mock.MagicMock(attrs={'start_time': dt.datetime(2009, 7, 1, 12, 15), + 'end_time': dt.datetime(2009, 7, 1, 12, 16)}) class AngleObj(object): def __init__(self): diff --git a/level1c4pps/tests/test_eumgacfdr2pps.py b/level1c4pps/tests/test_eumgacfdr2pps.py index 4c3f31b..0488c14 100644 --- a/level1c4pps/tests/test_eumgacfdr2pps.py +++ b/level1c4pps/tests/test_eumgacfdr2pps.py @@ -75,8 +75,8 @@ def test_get_encoding(self): 'zlib': True, 'complevel': 4, 'add_offset': 273.15}, - 'qual_flags': {'dtype': 'int16', 'zlib': True, - 'complevel': 4, '_FillValue': -32001.0}, + 'qual_flags': {'dtype': 'int16', 'zlib': True, + 'complevel': 4, '_FillValue': -32001.0}, 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, diff --git a/level1c4pps/tests/test_gac2pps.py b/level1c4pps/tests/test_gac2pps.py index e41b585..eb87c90 100644 --- a/level1c4pps/tests/test_gac2pps.py +++ b/level1c4pps/tests/test_gac2pps.py @@ -74,8 +74,8 @@ def test_get_encoding(self): 'zlib': True, 'complevel': 4, 'add_offset': 273.15}, - 'qual_flags': {'dtype': 'int16', 'zlib': True, - 'complevel': 4, '_FillValue': -32001.0}, + 'qual_flags': {'dtype': 'int16', 'zlib': True, + 'complevel': 4, '_FillValue': -32001.0}, 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', 'complevel': 4, '_FillValue': -1.0}, diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index d328d53..913abc8 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -169,7 +169,7 @@ def alt_az_patched(time, lon, lat): def test_get_satellite_angles(self, get_satpos, get_observer_look): """Test getting satellite angles.""" def get_observer_look_patched(lon, lat, alt, *args): - if alt == 36000*1000: + if alt == 36000 * 1000: return None, 31 # > 30 elif alt == 36000: return None, 22 # < 20 @@ -386,7 +386,7 @@ def test_get_encoding(self): scene = Scene() scene.attrs = {'start_time': dt.datetime(2009, 7, 1, 12, 15)} scene_dict = {'VIS006': vis006, 'IR_108': ir_108, 'lat': lat, 'lon': lon, - 'sunzenith': sunzenith, 'satzenith': satzenith, 'azimuthdiff': azimuthdiff} + 'sunzenith': sunzenith, 'satzenith': satzenith, 'azimuthdiff': azimuthdiff} for key in scene_dict: pps_name = scene_dict[key].attrs['name'] scene[key] = scene_dict[key] diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 578005c..1cde6d9 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -82,8 +82,8 @@ def test_get_encoding(self): 'zlib': True, 'complevel': 4, 'add_offset': 273.15}, - 'qual_flags': {'dtype': 'int16', 'zlib': True, - 'complevel': 4, '_FillValue': -32001.0}, + 'qual_flags': {'dtype': 'int16', 'zlib': True, + 'complevel': 4, '_FillValue': -32001.0}, 'scanline_timestamps': {'dtype': 'int64', 'zlib': True, 'units': 'milliseconds since 1970-01-01', diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index a597a76..5da41d7 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -36,7 +36,7 @@ # Order of BANDNAMES decides order of channels in file. Not important # but nice to have the same order for I- and M-bands -BANDNAMES = ["M01", "M02", "M03", "M04", +BANDNAMES = ["M01", "M02", "M03", "M04", "M05", "M06", "M07", # 0.6, 0.7, 0.9 M-band "I01", "I02", # 0.6, 0.9 I-band "M08", "M09", # 1.2, 1.3 M-band @@ -61,7 +61,7 @@ # "M10", "M14" are not AVHRR channels, but needed for NN SABAF MBAND_AVHRR = ["M05", "M07", "M12", "M15", "M16", "M10", "M14"] -MBAND_DEFAULT = ["M05", "M07", "M09", "M10", "M11", "M12", "M14", "M15", "M16"] +MBAND_DEFAULT = ["M05", "M07", "M09", "M10", "M11", "M12", "M14", "M15", "M16"] ANGLE_NAMES = ["vza", "sza", "azn", "azi"] @@ -122,7 +122,7 @@ "tb12": { "viirs_channel": "M16", "slope": 0.9934, - "offset": 1.52, + "offset": 1.52, "comment": "based on nadir collocation data for SZA 0-180", }, }, @@ -634,7 +634,7 @@ def convert_to_noaa19_KNMI_v2(scene, sbaf_version): if avhrr_chan == "tb37_twilight": # 70 < SZA < 85: BT = (1-f)*BT(day) + f*BT(night), f=(SZA-70)/15 - f = (scene["sunzenith"].values - scaling["min_sunzenith"])/15 + f = (scene["sunzenith"].values - scaling["min_sunzenith"]) / 15 tb37_day_slope = scaling["tb37_day"]["slope"] tb37_day_offset = scaling["tb37_day"]["offset"] tb37_night_slope = scaling["tb37_night"]["slope"] @@ -644,13 +644,14 @@ def convert_to_noaa19_KNMI_v2(scene, sbaf_version): scene[viirs_channel].values = np.where( filt, - (1-f)*tb37_day + f*tb37_night, + (1 - f) * tb37_day + f * tb37_night, scene[viirs_channel].values ) elif avhrr_chan == "tb12": # BT(5) = BT(ch4)-1.1646*(BT(M15)-BT(M16))-0.235 - scene[viirs_channel].values = scene["M15"].values-1.1646*(tb11_original-scene["M16"].values)-0.235 + scene[viirs_channel].values = scene["M15"].values - \ + 1.1646 * (tb11_original - scene["M16"].values) - 0.235 else: logger.exception(f'Unknown channel, {avhrr_chan}, or missing slope parameter') logger.info(f"{avhrr_chan:<13}: ({comment})") @@ -848,6 +849,6 @@ def process_one_scene(scene_files, out_path, engine="h5netcdf", encoding=encoding) logger.info("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) filenames.append(filename) return filenames diff --git a/level1c4pps/viirs2pps_lib.py b/level1c4pps/viirs2pps_lib.py index e5a67ad..92dd642 100644 --- a/level1c4pps/viirs2pps_lib.py +++ b/level1c4pps/viirs2pps_lib.py @@ -36,7 +36,7 @@ # Order of BANDNAMES decides order of channels in file. Not important # but nice to have the same order for I- and M-bands -BANDNAMES = ["M01", "M02", "M03", "M04", +BANDNAMES = ["M01", "M02", "M03", "M04", "M05", "M06", "M07", # 0.6, 0.7, 0.9 M-band "I01", "I02", # 0.6, 0.9 I-band "M08", "M09", # 1.2, 1.3 M-band @@ -61,9 +61,9 @@ IBAND_PPS_I = ["I01", "I02", "I03", "I04"] IBAND_PPS_M = ["M09", "M14", "M15", "M16"] -MBAND_DEFAULT = ["M05", "M07", "M09", "M10", "M11", "M12", "M14", "M15", "M16"] +MBAND_DEFAULT = ["M05", "M07", "M09", "M10", "M11", "M12", "M14", "M15", "M16"] IBAND_DEFAULT_I = ["I01", "I02", "I03", "I04"] -IBAND_DEFAULT_M = ["M09", "M11", "M14", "M15", "M16"] +IBAND_DEFAULT_M = ["M09", "M11", "M14", "M15", "M16"] ANGLE_NAMES = ['satellite_zenith_angle', 'solar_zenith_angle', 'satellite_azimuth_angle', 'solar_azimuth_angle'] @@ -107,10 +107,10 @@ def set_header_and_band_attrs(scene, orbit_n=0): scene.attrs['source'] = "viirs2pps.py" if 'I04' in scene: # If highresolution we should have I04, - scene.attrs['number_of_scans'] = scene['I04'].values.shape[0]/scene['I04'].attrs['rows_per_scan'] + scene.attrs['number_of_scans'] = scene['I04'].values.shape[0] / scene['I04'].attrs['rows_per_scan'] else: # else use 11um. - scene.attrs['number_of_scans'] = scene['M15'].values.shape[0]/scene['M15'].attrs['rows_per_scan'] + scene.attrs['number_of_scans'] = scene['M15'].values.shape[0] / scene['M15'].attrs['rows_per_scan'] for band in REFL_BANDS: if band not in scene: continue @@ -172,5 +172,5 @@ def process_one_scene(scene_files, out_path, use_iband_res=False, reader='viirs_ encoding=get_encoding_viirs(scn_)) print("Saved file {:s} after {:3.1f} seconds".format( os.path.basename(filename), - time.time()-tic)) + time.time() - tic)) return filename From 177aed18e520520e6e769dcc593342e9b2e1b0ac Mon Sep 17 00:00:00 2001 From: "Nina.Hakansson" <6160529+ninahakansson@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:34:18 +0100 Subject: [PATCH 22/22] flake8 --- level1c4pps/gac2pps_lib.py | 1 - level1c4pps/tests/test_seviri2pps.py | 1 - level1c4pps/tests/test_vgac2pps.py | 7 +------ level1c4pps/vgac2pps_lib.py | 12 ++++-------- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/level1c4pps/gac2pps_lib.py b/level1c4pps/gac2pps_lib.py index 6cf5780..cc7b034 100644 --- a/level1c4pps/gac2pps_lib.py +++ b/level1c4pps/gac2pps_lib.py @@ -34,7 +34,6 @@ get_header_attrs, convert_angles) import logging -from packaging.version import Version logger = logging.getLogger('gac2pps') diff --git a/level1c4pps/tests/test_seviri2pps.py b/level1c4pps/tests/test_seviri2pps.py index 913abc8..3d69f5a 100644 --- a/level1c4pps/tests/test_seviri2pps.py +++ b/level1c4pps/tests/test_seviri2pps.py @@ -65,7 +65,6 @@ class TestSeviri2PPS(unittest.TestCase): @mock.patch('level1c4pps.seviri2pps_lib.Scene') def test_load_and_calibrate(self, mocked_scene): """Test loading and calibrating the data.""" - mocked_scene.return_value = get_fake_scene() # Load and calibrate diff --git a/level1c4pps/tests/test_vgac2pps.py b/level1c4pps/tests/test_vgac2pps.py index 1cde6d9..d97edcd 100644 --- a/level1c4pps/tests/test_vgac2pps.py +++ b/level1c4pps/tests/test_vgac2pps.py @@ -30,10 +30,9 @@ import level1c4pps.vgac2pps_lib as vgac2pps import numpy as np -import pytest try: - import sbafs_ann + import sbafs_ann # noqa: F401 no_sbaf_module = False except ModuleNotFoundError: no_sbaf_module = True @@ -134,7 +133,6 @@ def test_process_one_scene(self): def test_process_one_scene_n19(self): """Test process one scene for one example file.""" - vgac2pps.process_one_scene( ['./level1c4pps/tests/VGAC_VJ102MOD_A2018305_1042_n004946_K005.nc'], out_path='./level1c4pps/tests/', @@ -171,8 +169,6 @@ def test_process_one_scene_n19(self): @unittest.skipIf(no_sbaf_module, "Install sbafs_ann to test NN-SBAFS.") def test_process_one_scene_n19_nn(self): """Test process one scene for one example file.""" - import sbafs_ann - vgac2pps.process_one_scene( ['./level1c4pps/tests/VGAC_VJ102MOD_A2018305_1042_n004946_K005.nc'], out_path='./level1c4pps/tests/', @@ -209,7 +205,6 @@ def test_process_one_scene_n19_nn(self): def test_process_one_scene_midnight(self): """Test process one scene for one example file.""" - vgac2pps.process_one_scene( ['./level1c4pps/tests/VGAC_VNPP02MOD_A2012365_2304_n06095_K005.nc'], out_path='./level1c4pps/tests/' diff --git a/level1c4pps/vgac2pps_lib.py b/level1c4pps/vgac2pps_lib.py index 5da41d7..7564d4c 100644 --- a/level1c4pps/vgac2pps_lib.py +++ b/level1c4pps/vgac2pps_lib.py @@ -572,8 +572,7 @@ def convert_to_noaa19_neural_network(scene, sbaf_version): - """Applies AVHRR SBAF to VGAC channels using NN approach""" - + """Apply AVHRR SBAF to VGAC channels using NN approach.""" from sbafs_ann.convert_vgac import convert_to_vgac_with_nn if sbaf_version in ["NN_v1", "NN_v2", "NN_v3", "NN_v4"]: day_cfg_file = SBAF[sbaf_version]['cfg_file_day'] @@ -587,8 +586,7 @@ def convert_to_noaa19_neural_network(scene, sbaf_version): def convert_to_noaa19_linear(scene, SBAF): - """ Apply linear regression""" - + """Apply linear regression.""" for avhhr_chan, scaling in SBAF.items(): viirs_channel = scaling["viirs_channel"] offset = scaling["offset"] @@ -607,8 +605,7 @@ def convert_to_noaa19_linear(scene, SBAF): def convert_to_noaa19_KNMI_v2(scene, sbaf_version): - """ Apply 1 channel linear regression SBAF for KNMI version 2""" - + """Apply 1 channel linear regression SBAF for KNMI version 2.""" # I need to save the t11 values before the SBAF adjustment as they are needed for the tb12 SBAF tb11_original = scene["M15"].values.copy() for avhrr_chan, scaling in SBAF[sbaf_version].items(): @@ -658,8 +655,7 @@ def convert_to_noaa19_KNMI_v2(scene, sbaf_version): def convert_to_noaa19(scene, sbaf_version): - """ Applies AVHRR SBAF to VGAC channels""" - + """Apply AVHRR SBAF to VGAC channels.""" logger.info(f"Using SBAF_{sbaf_version}") if "NN" in sbaf_version: