From 55b341fd8a8ef375414e4a558b7c9ed624c1c906 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 18:10:52 -0500 Subject: [PATCH 1/9] chore: replace print with logging in A02c --- .../analysis_db/A02c_IOWAGA_thredds_prior.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py index 03ee59e..b6f5f3d 100644 --- a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py +++ b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import datetime +import logging from pathlib import Path import h5py @@ -22,16 +23,16 @@ from icesat2_tracks.config.IceSAT2_startup import font_for_print from icesat2_tracks.clitools import ( - echo, validate_batch_key, validate_output_dir, - suppress_stdout, update_paths_mconfig, report_input_parameters, validate_track_name_steps_gt_1, makeapp, ) +_logger = logging.getLogger(__name__) + def get_iowaga(data_url, dataset_key): ## load WW3 data @@ -447,7 +448,7 @@ def test_nan_frac(imask): return ((~imask).sum() / imask.size).data < 0.3 while test_nan_frac(ice_mask_prior): - print(lat_range_prior) + _logger.debug("%s", lat_range_prior) lat_range_prior = lat_range_prior[0] + 0.5, lat_range_prior[1] + 0.5 G_prior = sel_data(G_beam, lon_range, lat_range_prior) ice_mask_prior = ice_mask.sel(latitude=G_prior.latitude) @@ -629,8 +630,8 @@ def plot_prior(Prior, axx): F.save_pup(path=plot_path, name=plot_name + "_hindcast_prior") except Exception as e: - print(e) - echo("print 2nd figure failed", "red") + _logger.debug("%s", e) + _logger.warning("print 2nd figure failed") MT.json_save( target_name, @@ -638,7 +639,7 @@ def plot_prior(Prior, axx): str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")), ) - echo("done") + _logger.info("done") make_iowaga_threads_prior_app = makeapp( @@ -646,4 +647,5 @@ def plot_prior(Prior, axx): ) if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) make_iowaga_threads_prior_app() From 3383f116764c75da6c695235132934743272c5df Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 18:11:39 -0500 Subject: [PATCH 2/9] chore: replace print with logging in B02 --- .../analysis_db/B02_make_spectra_gFT.py | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py b/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py index 794c8c2..e0a4cf8 100644 --- a/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py +++ b/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py @@ -6,6 +6,8 @@ import copy import datetime +import logging + import h5py from pathlib import Path from functools import partial @@ -29,16 +31,16 @@ from icesat2_tracks.config.IceSAT2_startup import mconfig from icesat2_tracks.clitools import ( - echo, validate_batch_key, validate_output_dir, - suppress_stdout, update_paths_mconfig, report_input_parameters, validate_track_name_steps_gt_1, makeapp, ) +_logger = logging.getLogger(__name__) + # import tracemalloc # removing this for now. CP @@ -101,7 +103,7 @@ def run_B02_make_spectra_gFT( all_beams = mconfig["beams"]["all_beams"] N_process = 4 - print("N_process=", N_process) + _logger.debug("N_process= %s", N_process) Gd = h5py.File(Path(load_path) / (track_name + "_B01_binned.h5"), "r") @@ -122,11 +124,11 @@ def run_B02_make_spectra_gFT( Ib = Gd[group[1]] ratio = Ia["x"][:].size / Ib["x"][:].size if (ratio > 10) | (ratio < 0.1): - print("bad data ratio ", ratio, 1 / ratio) + _logger.debug("bad data ratio %s %s", ratio, 1 / ratio) bad_ratio_flag = True if (np.array(nan_fraction).mean() > 0.95) | bad_ratio_flag: - print( + _logger.critical( "nan fraction > 95%, or bad ratio of data, pass this track, add to bad tracks" ) MT.json_save( @@ -137,11 +139,11 @@ def run_B02_make_spectra_gFT( "date": str(datetime.date.today()), }, ) - print("exit.") + _logger.critical("exit.") exit() # test LS with an even grid where missing values are set to 0 - print(Gd.keys()) + _logger.debug("%s", Gd.keys()) Gi = Gd[list(Gd.keys())[0]] # to select a test beam dist = io.get_beam_var_hdf_store(Gd[list(Gd.keys())[0]], "x") # make dataframe form hdf5 @@ -156,9 +158,9 @@ def run_B02_make_spectra_gFT( Lpoints = int(np.round(min_datapoint) * 10) Lmeters = Lpoints * dx - print("L number of gridpoint:", Lpoints) - print("L length in km:", Lmeters / 1e3) - print("approx number windows", 2 * dist.iloc[-1] / Lmeters - 1) + _logger.debug("L number of gridpoint: %s", Lpoints) + _logger.debug("L length in km: %s", Lmeters / 1e3) + _logger.debug("approx number windows %s", 2 * dist.iloc[-1] / Lmeters - 1) T_min = 6 lambda_min = 9.81 * T_min**2 / (2 * np.pi) @@ -168,14 +170,14 @@ def run_B02_make_spectra_gFT( kk = np.arange(0, 1 / lambda_min, 1 / dlambda) * 2 * np.pi kk = kk[k_0 <= kk] - print("2 M = ", kk.size * 2) + _logger.debug("2 M = %s", kk.size * 2) - print("define global xlims") + _logger.debug("define global xlims") dist_list = np.array([np.nan, np.nan]) for k in all_beams: - print(k) + _logger.debug(k) x = Gd[k + "/x"][:] - print(x[0], x[-1]) + _logger.debug("%s %s",x[0], x[-1]) dist_list = np.vstack([dist_list, [x[0], x[-1]]]) xlims = np.nanmin(dist_list[:, 0]) - dx, np.nanmin(dist_list[:, 1]) @@ -183,16 +185,16 @@ def run_B02_make_spectra_gFT( for k in all_beams: dist_i = io.get_beam_var_hdf_store(Gd[k], "x") x_mask = (dist_i > xlims[0]) & (dist_i < xlims[1]) - print(k, sum(x_mask["x"]) / (xlims[1] - xlims[0])) + _logger.debug("%s %s", k, sum(x_mask["x"]) / (xlims[1] - xlims[0])) - print("-reduced frequency resolution") + _logger.debug("-reduced frequency resolution") kk = kk[::2] - print("set xlims: ", xlims) + _logger.debug("set xlims: %s", xlims) # Commented out for now. CP - # print( - # "Loop start: ", + # _logger.debug( + # "Loop start: %s %s", # tracemalloc.get_traced_memory()[0] / 1e6, # tracemalloc.get_traced_memory()[1] / 1e6, # ) @@ -211,7 +213,7 @@ def run_B02_make_spectra_gFT( Gi = io.get_beam_hdf_store(Gd[k]) x_mask = (Gi["x"] > xlims[0]) & (Gi["x"] < xlims[1]) if sum(x_mask) / (xlims[1] - xlims[0]) < 0.005: - print("------------------- no data in beam found; skip") + _logger.debug("------------------- no data in beam found; skip") Gd_cut = Gi[x_mask] x = Gd_cut["x"] @@ -235,7 +237,7 @@ def run_B02_make_spectra_gFT( x_no_nans = x[~dd_nans] dd_error_no_nans = dd_error[~dd_nans] - print("gFT") + _logger.debug("gFT") with threadpool_limits(limits=N_process, user_api="blas"): pprint(threadpool_info()) @@ -254,8 +256,8 @@ def run_B02_make_spectra_gFT( ) # Commented out for now. CP - # print( - # "after ", + # _logger.debug( + # "after %s %s %s", # k, # tracemalloc.get_traced_memory()[0] / 1e6, # tracemalloc.get_traced_memory()[1] / 1e6, @@ -454,7 +456,7 @@ def _get_stancil_nans(stancil, Gd_cut=Gd_cut): plt.plot(GG.k, np.nanmean(S.G, 1), "k", label="mean gFT power model") # standard FFT - print("FFT") + _logger.debug("FFT") dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) @@ -503,7 +505,7 @@ def get_stancil_nans(stancil): plt.legend() plt.show() except Exception as e: - print(e, "An error occurred. Nothing to plot.") + _logger.debug("%s: An error occurred. Nothing to plot.", e) del Gd_cut Gd.close() @@ -568,10 +570,11 @@ def make_dummy_beam(GG, beam): G_fft_DS.attrs["name"] = "FFT_power_spectra" G_fft_DS.to_netcdf(savepathname + "_FFT.nc") - echo("saved and done") + _logger.info("saved and done") make_spectra_app = makeapp(run_B02_make_spectra_gFT, name="makespectra") if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) make_spectra_app() From 53924c9064455c7c176c62f8907d1e0695bd0982 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 18:12:26 -0500 Subject: [PATCH 3/9] chore: replace print with logging in B03 --- .../analysis_db/B03_plot_spectra_ov.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py index 4f32d30..1dd677d 100644 --- a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py +++ b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py @@ -3,6 +3,7 @@ This file open a ICEsat2 track applied filters and corrections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ +import logging from ast import comprehension from pathlib import Path import matplotlib @@ -23,16 +24,16 @@ ) from icesat2_tracks.clitools import ( - echo, validate_batch_key, validate_output_dir, - suppress_stdout, update_paths_mconfig, report_input_parameters, validate_track_name_steps_gt_1, makeapp, ) +_logger = logging.getLogger(__name__) + def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): if Gi.k[0] == 0: @@ -352,11 +353,11 @@ def run_B03_plot_spectra_ov( Gk.sel(beam=k).gFT_PSD_data.plot() if "y_data" in Gx.sel(beam="gt3r").keys(): - print("ydata is ", ("y_data" in Gx.sel(beam="gt3r").keys())) + _logger.debug("ydata is %s", ("y_data" in Gx.sel(beam="gt3r").keys())) else: - print("ydata is ", ("y_data" in Gx.sel(beam="gt3r").keys())) + _logger.warning("ydata is %s", ("y_data" in Gx.sel(beam="gt3r").keys())) MT.json_save("B03_fail", plot_path, {"reason": "no y_data"}) - echo("failed, exit", "red") + _logger.warning("failed, exit") exit() fltostr, _ = MT.float_to_str, MT.num_to_str @@ -581,10 +582,11 @@ def run_B03_plot_spectra_ov( {"time": "time.asctime( time.localtime(time.time()) )"}, ) - echo("success", "green") + _logger.info("success") plot_spectra = makeapp(run_B03_plot_spectra_ov, name="plotspectra") if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) plot_spectra() From 899dc0ee5e51a4a2725109f1ac3ec8aff6194a22 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 18:13:45 -0500 Subject: [PATCH 4/9] chore: replace print with logging in B04 --- src/icesat2_tracks/analysis_db/B04_angle.py | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B04_angle.py b/src/icesat2_tracks/analysis_db/B04_angle.py index d7464c6..0daf7d7 100644 --- a/src/icesat2_tracks/analysis_db/B04_angle.py +++ b/src/icesat2_tracks/analysis_db/B04_angle.py @@ -5,6 +5,7 @@ """ import itertools +import logging from icesat2_tracks.config.IceSAT2_startup import ( mconfig, @@ -41,13 +42,14 @@ from icesat2_tracks.clitools import ( validate_batch_key, validate_output_dir, - suppress_stdout, update_paths_mconfig, report_input_parameters, validate_track_name_steps_gt_1, makeapp, ) +_logger = logging.getLogger(__name__) + def run_B04_angle( track_name: str = Option(..., callback=validate_track_name_steps_gt_1), @@ -115,7 +117,7 @@ def run_B04_angle( "priors_hindcast" ] except FileNotFoundError: - print("Prior not found. exit") + _logger.critical("Prior not found. exit") MT.json_save( "B04_fail", plot_path, @@ -127,7 +129,7 @@ def run_B04_angle( exit() if np.isnan(Prior["mean"]["dir"]): - print("Prior failed, entries are nan. exit.") + _logger.critical("Prior failed, entries are nan. exit.") MT.json_save( "B04_fail", plot_path, @@ -173,7 +175,7 @@ def run_B04_angle( dir_best = np.array(dir_best[1:]) if len(Pperiod) == 0: - print("constant peak wave number") + _logger.debug("constant peak wave number") kk = Gk.k Pwavenumber = kk * 0 + (2 * np.pi / (1 / Prior.loc["fp"]["mean"])) ** 2 / g dir_best = kk * 0 + Prior.loc["dp"]["mean"] @@ -263,11 +265,8 @@ def run_B04_angle( prior_angle = Prior_smth.Prior_direction * 180 / np.pi if (abs(prior_angle) > 80).all(): - print( - "Prior angle is ", - prior_angle.min().data, - prior_angle.max().data, - ". quit.", + _logger.critical( + "Prior angle is %s %s. Quit.", prior_angle.min().data, prior_angle.max().data, ) dd_save = { "time": time.asctime(time.localtime(time.time())), @@ -280,7 +279,7 @@ def run_B04_angle( ), } MT.json_save("B04_fail", plot_path, dd_save) - print("exit()") + _logger.critical("exit()") exit() # define parameter range @@ -556,12 +555,12 @@ def plot_instance( ggg, xxx = np.meshgrid(group_number, x_list.data) for gi in zip(ggg.flatten(), xxx.flatten()): - print(gi) + _logger.debug(gi) group, xi = beam_groups[gi[0]], gi[1] if bool(x_list_flag.sel(x=xi).isel(beam_group=gi[0]).data) is False: - print("no data, fill with dummy") + _logger.debug("no data, fill with dummy") ikey = str(xi) + "_" + "_".join(group) Marginals[ikey] = make_fake_data(xi, group) continue @@ -594,7 +593,7 @@ def plot_instance( mean_dist = (nu_2d.isel(beam=0) - nu_2d.isel(beam=1)).mean().data k_upper_lim = 2 * np.pi / (mean_dist * 1) - print("k_upper_lim ", k_upper_lim) + _logger.debug("k_upper_lim %s", k_upper_lim) # variance method amp_data = np.sqrt(GGk.gFT_cos_coeff**2 + GGk.gFT_sin_coeff**2) @@ -607,7 +606,7 @@ def plot_instance( ) if len(k[mask]) == 0: - print("no good k found, fill with dummy") + _logger.debug("no good k found, fill with dummy") ikey = str(xi) + "_" + "_".join(group) Marginals[ikey] = make_fake_data(xi, group) continue @@ -804,9 +803,9 @@ def get_instance(k_pair): return k_prime_max, rdict k_list, weight_list = k[mask], weights[mask] - print("# of wavenumber: ", len(k_list)) + _logger.debug("# of wavenumber: %s", len(k_list)) if len(k_list) > max_wavenumbers: - print("cut wavenumber list to 20") + _logger.debug("cut wavenumber list to 20") k_list = k_list[0:max_wavenumbers] weight_list = weight_list[0:max_wavenumbers] @@ -842,7 +841,7 @@ def get_instance(k_pair): L_optimize = L_optimize.T.sort_values("K_prime") L_brute = L_brute.T.sort_values("K_prime") - print("done with ", group, xi / 1e3) + _logger.info("done with %s %s", group, xi / 1e3) # collect ikey = str(xi) + "_" + "_".join(group) @@ -880,7 +879,7 @@ def get_instance(k_pair): {"L_sample": LL}, save_name + "_res_table", str(save_path) ) # TODO: clean up save_pandas_table to use pathlib except Exception as e: - print(f"This is a warning: {e}") + _logger.warning(f"This is a warning: %s", e) else: # plotting with LL font_for_print() @@ -1015,4 +1014,5 @@ def get_instance(k_pair): make_b04_angle_app = makeapp(run_B04_angle, name="B04_angle") if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) make_b04_angle_app() From 2c96d3b45f2b93145169f1d9990e74dd0dd5cc4e Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 18:14:13 -0500 Subject: [PATCH 5/9] chore: replace print with logging in B05 --- .../analysis_db/B05_define_angle.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B05_define_angle.py b/src/icesat2_tracks/analysis_db/B05_define_angle.py index 81bf006..d8b1b91 100644 --- a/src/icesat2_tracks/analysis_db/B05_define_angle.py +++ b/src/icesat2_tracks/analysis_db/B05_define_angle.py @@ -3,7 +3,7 @@ This file open a ICEsat2 track applied filters and corrections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ - +import logging from pathlib import Path import matplotlib from matplotlib import pyplot as plt @@ -31,7 +31,6 @@ from icesat2_tracks.clitools import ( validate_batch_key, validate_output_dir, - suppress_stdout, report_input_parameters, validate_track_name_steps_gt_1, makeapp, @@ -40,6 +39,8 @@ matplotlib.use("Agg") color_schemes.colormaps2(21) +_logger = logging.getLogger(__name__) + def derive_weights(weights): """ @@ -119,8 +120,8 @@ def __init__( self.min = np.round(np.nanmin(data[freq_sel_bool, :]), 2) self.max = np.round(np.nanmax(data[freq_sel_bool, :]), 2) - if verbose: - print(str(self.min), str(self.max)) + + _logger.info("%s %s", str(self.min), str(self.max)) self.klabels = np.linspace(self.min, self.max, 5) @@ -370,7 +371,7 @@ def define_angle( ax_list[key].tick_params(labelbottom=True) ax_list[key].set_xlabel("Angle (rad)") else: - print(f"Key {key} not found in ax_list") + _logger.debug("Key %s not found in ax_list", key) ax_final = F.fig.add_subplot(gs[-1, :]) plt.title("Final angle PDF", loc="left") @@ -418,7 +419,7 @@ def define_angle( Gpdf = xr.merge([M_final, M_final_smth]) if len(Gpdf.x) < 2: - print("not enough x data, exit") + _logger.critical("not enough x data, exit") MT.json_save( "B05_fail", plot_path.parent, @@ -427,7 +428,7 @@ def define_angle( "reason": "not enough x segments", }, ) - print("exit()") + _logger.critical("exit()") exit() font_for_print() @@ -570,4 +571,5 @@ def define_angle( define_angle_app = makeapp(define_angle, name="B04_angle") if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) define_angle_app() From 82452695a5ae0dbc5a78654d211ce7bf2145cad5 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 18:14:50 -0500 Subject: [PATCH 6/9] chore: replace print with logging in B05 --- .../analysis_db/B06_correct_separate_var.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py b/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py index 84f3f62..0872824 100644 --- a/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py +++ b/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py @@ -2,7 +2,7 @@ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ - +import logging import os @@ -32,16 +32,16 @@ ) from icesat2_tracks.clitools import ( - echo, validate_batch_key, validate_output_dir, - suppress_stdout, update_paths_mconfig, report_input_parameters, validate_track_name_steps_gt_1, makeapp, ) +_logger = logging.getLogger(__name__) + def get_correct_breakpoint(pw_results): br_points = [i for i in pw_results.keys() if "breakpoint" in i] @@ -77,18 +77,18 @@ def get_correct_breakpoint(pw_results): alphas_v2_sorted = pd.Series(index=alphas_sorted, data=alphas_v2) br_points_sorted[f"breakpoint{br_points_sorted.size + 1}"] = "end" - echo("all alphas") - echo(alphas_v2_sorted) + _logger.debug("all alphas") + _logger.debug("%s", alphas_v2_sorted) slope_mask = alphas_v2_sorted < 0 if sum(slope_mask) == 0: - echo("no negative slope found, set to lowest") + _logger.debug("no negative slope found, set to lowest") breakpoint = "start" else: # take steepest slope alpah_v2_sub = alphas_v2_sorted[slope_mask] - echo(alpah_v2_sub) - echo(alpah_v2_sub.argmin()) + _logger.debug("%s", alpah_v2_sub) + _logger.debug("%s", alpah_v2_sub.argmin()) break_point_name = alpah_v2_sub.index[alpah_v2_sub.argmin()].replace( "alpha", "breakpoint" ) @@ -105,7 +105,9 @@ def get_breakingpoints(xx, dd): n_breakpoints = 3 while convergence_flag: pw_fit = piecewise_regression.Fit(xx, dd, n_breakpoints=n_breakpoints) - print("n_breakpoints", n_breakpoints, pw_fit.get_results()["converged"]) + _logger.debug( + "n_breakpoints %s %s", n_breakpoints, pw_fit.get_results()["converged"] + ) convergence_flag = not pw_fit.get_results()["converged"] n_breakpoints += 1 if n_breakpoints >= 4: @@ -134,10 +136,10 @@ def define_noise_wavenumber_piecewise(data_xr, plot_flag=False): pw_fit, breakpoint_log = get_breakingpoints(k_log, data_log.data) if breakpoint_log == "start": - echo("no decay, set to lowerst wavenumber") + _logger.debug("no decay, set to lowerst wavenumber") breakpoint_log = k_log[0] if (breakpoint_log == "end") | (breakpoint_log is False): - echo("higest wavenumner") + _logger.debug("higest wavenumner") breakpoint_log = k_log[-1] breakpoint_pos = abs(k_log - breakpoint_log).argmin() @@ -165,12 +167,12 @@ def weighted_mean(data, weights, additional_data=None): def calculate_k_end(x, k, k_end_previous, G_gFT_smth): - echo(x) + _logger.debug("%s", x) k_end, _ = define_noise_wavenumber_piecewise( G_gFT_smth.sel(x=x) / k, plot_flag=False ) k_save = k_end_previous if k_end == k[0] else k_end - echo("--------------------------") + _logger.debug("--------------------------") return k_save @@ -249,7 +251,11 @@ def save_table(data, tablename, save_path): io.save_pandas_table(data, tablename, save_path) except Exception as e: tabletoremove = save_path + tablename + ".h5" - echo(e, f"Failed to save table. Removing {tabletoremove} and re-trying..") + _logger.warning( + "Failed to save table with error %s. Removing %s and re-trying..", + e, + tabletoremove, + ) os.remove(tabletoremove) io.save_pandas_table(data, tablename, save_path) @@ -657,7 +663,7 @@ def reconstruct_displacement(Gx_1, Gk_1, T3, k_thresh): Gx_v2, B2_v2, B3_v2 = dict(), dict(), dict() for bb in Gx.beam.data: - echo(bb) + _logger.debug("%s", bb) Gx_k = Gx.sel(beam=bb) Gh = Gx["height_model"].sel(beam=bb).T Gh_err = Gx_k["model_error_x"].T @@ -697,7 +703,7 @@ def reconstruct_displacement(Gx_1, Gk_1, T3, k_thresh): ) except ValueError as e: - echo(f"{e} no angle data found, skip angle corretion") + _logger.warning("%s no angle data found, skip angle corretion", e) theta = 0 theta_flag = False else: @@ -773,11 +779,12 @@ def reconstruct_displacement(Gx_1, Gk_1, T3, k_thresh): (plot_path / "../"), {"time": time.asctime(time.localtime(time.time()))}, ) - echo("done. saved target at " + str(plot_path) + "../B06_success") - echo("Done B06_correct_separate_var") + _logger.info("done. saved target at %s../B06_success", str(plot_path)) + _logger.info("Done B06_correct_separate_var") correct_separate_app = makeapp(run_B06_correct_separate_var, name="correct-separate") if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) correct_separate_app() From 6f1c86bfb267a28c8418044e8e3ceec6fc394a01 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 5 Mar 2024 15:39:15 -0500 Subject: [PATCH 7/9] Update src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py Co-authored-by: Carlos Paniagua --- src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py index b6f5f3d..6ef90f0 100644 --- a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py +++ b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py @@ -448,7 +448,7 @@ def test_nan_frac(imask): return ((~imask).sum() / imask.size).data < 0.3 while test_nan_frac(ice_mask_prior): - _logger.debug("%s", lat_range_prior) + _logger.debug("lat range prior: %s", lat_range_prior) lat_range_prior = lat_range_prior[0] + 0.5, lat_range_prior[1] + 0.5 G_prior = sel_data(G_beam, lon_range, lat_range_prior) ice_mask_prior = ice_mask.sel(latitude=G_prior.latitude) From 416940c24eb8c14047a0f0da71ffefd3a5f0b707 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 5 Mar 2024 15:41:56 -0500 Subject: [PATCH 8/9] Apply suggestions from code review Co-authored-by: Carlos Paniagua --- .../analysis_db/B02_make_spectra_gFT.py | 8 ++++---- src/icesat2_tracks/analysis_db/B04_angle.py | 8 ++++---- src/icesat2_tracks/analysis_db/B05_define_angle.py | 4 ++-- .../analysis_db/B06_correct_separate_var.py | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py b/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py index e0a4cf8..e2f7143 100644 --- a/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py +++ b/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py @@ -143,7 +143,7 @@ def run_B02_make_spectra_gFT( exit() # test LS with an even grid where missing values are set to 0 - _logger.debug("%s", Gd.keys()) + _logger.debug("Gd.keys: %s", Gd.keys()) Gi = Gd[list(Gd.keys())[0]] # to select a test beam dist = io.get_beam_var_hdf_store(Gd[list(Gd.keys())[0]], "x") # make dataframe form hdf5 @@ -175,9 +175,9 @@ def run_B02_make_spectra_gFT( _logger.debug("define global xlims") dist_list = np.array([np.nan, np.nan]) for k in all_beams: - _logger.debug(k) + _logger.debug("k: %s", k) x = Gd[k + "/x"][:] - _logger.debug("%s %s",x[0], x[-1]) + _logger.debug("x first element: %s, last element: %s", x[0], x[-1]) dist_list = np.vstack([dist_list, [x[0], x[-1]]]) xlims = np.nanmin(dist_list[:, 0]) - dx, np.nanmin(dist_list[:, 1]) @@ -185,7 +185,7 @@ def run_B02_make_spectra_gFT( for k in all_beams: dist_i = io.get_beam_var_hdf_store(Gd[k], "x") x_mask = (dist_i > xlims[0]) & (dist_i < xlims[1]) - _logger.debug("%s %s", k, sum(x_mask["x"]) / (xlims[1] - xlims[0])) + _logger.debug("k: %s, sum/range: %s", k, sum(x_mask["x"]) / (xlims[1] - xlims[0])) _logger.debug("-reduced frequency resolution") kk = kk[::2] diff --git a/src/icesat2_tracks/analysis_db/B04_angle.py b/src/icesat2_tracks/analysis_db/B04_angle.py index 0daf7d7..04783db 100644 --- a/src/icesat2_tracks/analysis_db/B04_angle.py +++ b/src/icesat2_tracks/analysis_db/B04_angle.py @@ -279,7 +279,7 @@ def run_B04_angle( ), } MT.json_save("B04_fail", plot_path, dd_save) - _logger.critical("exit()") + _logger.critical("terminating") exit() # define parameter range @@ -555,7 +555,7 @@ def plot_instance( ggg, xxx = np.meshgrid(group_number, x_list.data) for gi in zip(ggg.flatten(), xxx.flatten()): - _logger.debug(gi) + _logger.debug("gi = %s", gi) group, xi = beam_groups[gi[0]], gi[1] @@ -841,7 +841,7 @@ def get_instance(k_pair): L_optimize = L_optimize.T.sort_values("K_prime") L_brute = L_brute.T.sort_values("K_prime") - _logger.info("done with %s %s", group, xi / 1e3) + _logger.info("done with group %s, xi = %s", group, xi / 1e3) # collect ikey = str(xi) + "_" + "_".join(group) @@ -879,7 +879,7 @@ def get_instance(k_pair): {"L_sample": LL}, save_name + "_res_table", str(save_path) ) # TODO: clean up save_pandas_table to use pathlib except Exception as e: - _logger.warning(f"This is a warning: %s", e) + _logger.warning("This is a warning: %s", e) else: # plotting with LL font_for_print() diff --git a/src/icesat2_tracks/analysis_db/B05_define_angle.py b/src/icesat2_tracks/analysis_db/B05_define_angle.py index d8b1b91..0c6f787 100644 --- a/src/icesat2_tracks/analysis_db/B05_define_angle.py +++ b/src/icesat2_tracks/analysis_db/B05_define_angle.py @@ -121,7 +121,7 @@ def __init__( self.min = np.round(np.nanmin(data[freq_sel_bool, :]), 2) self.max = np.round(np.nanmax(data[freq_sel_bool, :]), 2) - _logger.info("%s %s", str(self.min), str(self.max)) + _logger.info("min %s, max %s", self.min, self.max) self.klabels = np.linspace(self.min, self.max, 5) @@ -428,7 +428,7 @@ def define_angle( "reason": "not enough x segments", }, ) - _logger.critical("exit()") + _logger.critical("exiting") exit() font_for_print() diff --git a/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py b/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py index 0872824..7061704 100644 --- a/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py +++ b/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py @@ -87,8 +87,8 @@ def get_correct_breakpoint(pw_results): else: # take steepest slope alpah_v2_sub = alphas_v2_sorted[slope_mask] - _logger.debug("%s", alpah_v2_sub) - _logger.debug("%s", alpah_v2_sub.argmin()) + _logger.debug("alpah_v2_sub: %s", alpah_v2_sub) + _logger.debug("alpah_v2_sub.argmin: %s", alpah_v2_sub.argmin()) break_point_name = alpah_v2_sub.index[alpah_v2_sub.argmin()].replace( "alpha", "breakpoint" ) @@ -106,7 +106,7 @@ def get_breakingpoints(xx, dd): while convergence_flag: pw_fit = piecewise_regression.Fit(xx, dd, n_breakpoints=n_breakpoints) _logger.debug( - "n_breakpoints %s %s", n_breakpoints, pw_fit.get_results()["converged"] + "n_breakpoints %s\n%s", n_breakpoints, pw_fit.get_results()["converged"] ) convergence_flag = not pw_fit.get_results()["converged"] n_breakpoints += 1 @@ -167,7 +167,7 @@ def weighted_mean(data, weights, additional_data=None): def calculate_k_end(x, k, k_end_previous, G_gFT_smth): - _logger.debug("%s", x) + _logger.debug("From calculate_k_end -- x: %s", x) k_end, _ = define_noise_wavenumber_piecewise( G_gFT_smth.sel(x=x) / k, plot_flag=False ) @@ -663,7 +663,7 @@ def reconstruct_displacement(Gx_1, Gk_1, T3, k_thresh): Gx_v2, B2_v2, B3_v2 = dict(), dict(), dict() for bb in Gx.beam.data: - _logger.debug("%s", bb) + _logger.debug("bb: %s", bb) Gx_k = Gx.sel(beam=bb) Gh = Gx["height_model"].sel(beam=bb).T Gh_err = Gx_k["model_error_x"].T @@ -779,7 +779,7 @@ def reconstruct_displacement(Gx_1, Gk_1, T3, k_thresh): (plot_path / "../"), {"time": time.asctime(time.localtime(time.time()))}, ) - _logger.info("done. saved target at %s../B06_success", str(plot_path)) + _logger.info("done. saved target at %s../B06_success", plot_path) _logger.info("Done B06_correct_separate_var") From 3b23cab904a7e7d81530ba50f1c62a66a20920b0 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 19 Mar 2024 14:46:09 +0100 Subject: [PATCH 9/9] Update src/icesat2waves/analysis_db/B06_correct_separate_var.py Co-authored-by: Camilo Diaz --- src/icesat2waves/analysis_db/B06_correct_separate_var.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/icesat2waves/analysis_db/B06_correct_separate_var.py b/src/icesat2waves/analysis_db/B06_correct_separate_var.py index 7ae7262..c3ecbf0 100644 --- a/src/icesat2waves/analysis_db/B06_correct_separate_var.py +++ b/src/icesat2waves/analysis_db/B06_correct_separate_var.py @@ -172,7 +172,6 @@ def calculate_k_end(x, k, k_end_previous, G_gFT_smth): G_gFT_smth.sel(x=x) / k, plot_flag=False ) k_save = k_end_previous if k_end == k[0] else k_end - _logger.debug("--------------------------") return k_save