diff --git a/examples/benchmarks/cifar-10/cnn_cifar10.py b/examples/benchmarks/cifar-10/cnn_cifar10.py index 0a06d061..70efa219 100644 --- a/examples/benchmarks/cifar-10/cnn_cifar10.py +++ b/examples/benchmarks/cifar-10/cnn_cifar10.py @@ -41,11 +41,16 @@ class Cifar10Datamodule(BaseDataModule): num_workers : int Number of workers to use for loading data. """ - - def __init__(self, dataset_path: str, batch_size: int, num_workers: int, **kwargs): + def __init__(self, + dataset_path: str, + train_batch_size: int, + inference_batch_size: int, + num_workers: int, + **kwargs): super().__init__() self.dataset_path = dataset_path - self.batch_size = batch_size + self.train_batch_size = train_batch_size + self.inference_batch_size = inference_batch_size self.num_workers = num_workers self.__dict__.update(kwargs) self.cifar10_train = None @@ -104,15 +109,19 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = Cifar10Datamodule(**cfg.data, **cfg.task) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( @@ -126,9 +135,10 @@ def main(cfg: DictConfig) -> None: ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) + # Run if cfg.run.predict: model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, + model=classif_system.model, hparams_preprocess=False) # Option 1: Predict on the entire test dataset (Pytorch Lightning) @@ -149,8 +159,8 @@ def main(cfg: DictConfig) -> None: df.to_csv(os.path.join(log_dir, 'scores_test_dataset.csv'), index=False) print('Test dataset prediction (extract) : ', predictions[:1]) else: - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) - trainer.validate(model, datamodule=datamodule) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/benchmarks/cifar-10/config/cnn_cifar10.yaml b/examples/benchmarks/cifar-10/config/cnn_cifar10.yaml index 85e28615..2d35d4ae 100644 --- a/examples/benchmarks/cifar-10/config/cnn_cifar10.yaml +++ b/examples/benchmarks/cifar-10/config/cnn_cifar10.yaml @@ -3,8 +3,8 @@ hydra: dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S} run: - predict: false - checkpoint_path: # "outputs/cnn_cifar10/30_epochs_no_pretrained/last.ckpt" + predict: true + checkpoint_path: "outputs/cnn_cifar10/30_epochs_pretrained/last.ckpt" data: num_classes: &num_classes 10 diff --git a/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_patches.py b/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_patches.py index 687ff69b..bb59e58a 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_patches.py +++ b/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_patches.py @@ -10,6 +10,7 @@ from pathlib import Path import hydra +import numpy as np import pytorch_lightning as pl from omegaconf import DictConfig from pytorch_lightning.callbacks import ModelCheckpoint @@ -43,11 +44,13 @@ def __init__( inference_batch_size: int = 256, num_workers: int = 8, download: bool = False, + task: str = 'classification_multiclass', ): super().__init__(train_batch_size, inference_batch_size, num_workers) self.dataset_path = dataset_path self.minigeolifeclef = minigeolifeclef self.download = download + self.task = task @property def train_transform(self): @@ -96,17 +99,19 @@ def get_dataset(self, split, transform, **kwargs): @hydra.main(version_base="1.3", config_path="config", config_name="mono_modal_3_channels_model") def main(cfg: DictConfig) -> None: - + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(Path(log_dir)/Path(cfg.loggers.log_dir_name), name=cfg.loggers.exp_name, version="") logger_tb.log_hyperparams(cfg) - datamodule = GeoLifeCLEF2022DataModule(**cfg.data) - - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + # Datamodule & Model + datamodule = GeoLifeCLEF2022DataModule(**cfg.data, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( @@ -119,9 +124,23 @@ def main(cfg: DictConfig) -> None: ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - trainer.fit(model, datamodule=datamodule) - trainer.validate(model, datamodule=datamodule) + # Run + if cfg.run.predict: + model_loaded = ClassificationSystem.load_from_checkpoint(classif_system.checkpoint_path, + model=classif_system.model, + hparams_preprocess=False) + + # Option 1: Predict on the entire test dataset (Pytorch Lightning) + predictions = model_loaded.predict(datamodule, trainer) + preds, probas = datamodule.predict_logits_to_class(predictions, + np.arange(datamodule.get_test_dataset().n_classes)) + datamodule.export_predict_csv(preds, probas, + out_dir=log_dir, out_name='predictions_test_dataset', top_k=3, return_csv=True) + print('Test dataset prediction (extract) : ', predictions[:1]) + else: + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=classif_system.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_temperature_patches.py b/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_temperature_patches.py index 8a59e864..43527340 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_temperature_patches.py +++ b/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_rgb_temperature_patches.py @@ -10,6 +10,7 @@ from pathlib import Path import hydra +import numpy as np import pytorch_lightning as pl import torch from omegaconf import DictConfig @@ -22,7 +23,8 @@ MiniGeoLifeCLEF2022Dataset) from malpolon.data.environmental_raster import PatchExtractor from malpolon.logging import Summary -from malpolon.models.multi_modal import HomogeneousMultiModalModel +from malpolon.models.custom_models.multi_modal import \ + HomogeneousMultiModalModel from malpolon.models.standard_prediction_systems import ClassificationSystem @@ -45,10 +47,12 @@ def __init__( train_batch_size: int = 32, inference_batch_size: int = 256, num_workers: int = 8, + task: str = 'classification_multiclass', ): super().__init__(train_batch_size, inference_batch_size, num_workers) self.dataset_path = dataset_path self.minigeolifeclef = minigeolifeclef + self.task = task @property def train_transform(self): @@ -117,6 +121,7 @@ def __init__( num_outputs: int, cfg_optimizer: DictConfig, cfg_task: DictConfig, + checkpoint_path: str = None, ): model = HomogeneousMultiModalModel( ["rgb", "temperature"], @@ -124,22 +129,24 @@ def __init__( torch.nn.LazyLinear(num_outputs), ) - super().__init__(model, **cfg_optimizer, **cfg_task) + super().__init__(model, **cfg_optimizer, **cfg_task, checkpoint_path=checkpoint_path) @hydra.main(version_base="1.3", config_path="config", config_name="homogeneous_multi_modal_model") def main(cfg: DictConfig) -> None: - + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(Path(log_dir)/Path(cfg.loggers.log_dir_name), name=cfg.loggers.exp_name, version="") logger_tb.log_hyperparams(cfg) - datamodule = GeoLifeCLEF2022DataModule(**cfg.data) - - model = CustomClassificationSystem(**cfg.model, cfg_optimizer=cfg.optimizer, cfg_task=cfg.task) + # Datamodule & Model + datamodule = GeoLifeCLEF2022DataModule(**cfg.data, **cfg.task) + classif_system = CustomClassificationSystem(**cfg.model, cfg_optimizer=cfg.optimizer, cfg_task=cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( @@ -152,9 +159,22 @@ def main(cfg: DictConfig) -> None: ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - trainer.fit(model, datamodule=datamodule) - trainer.validate(model, datamodule=datamodule) + if cfg.run.predict: + model_loaded = CustomClassificationSystem.load_from_checkpoint(classif_system.checkpoint_path, + model=classif_system.model, + hparams_preprocess=False) + + # Option 1: Predict on the entire test dataset (Pytorch Lightning) + predictions = model_loaded.predict(datamodule, trainer) + preds, probas = datamodule.predict_logits_to_class(predictions, + np.arange(datamodule.get_test_dataset().n_classes)) + datamodule.export_predict_csv(preds, probas, + out_dir=log_dir, out_name='predictions_test_dataset', top_k=3, return_csv=True) + print('Test dataset prediction (extract) : ', predictions[:1]) + else: + trainer.fit(classif_system, datamodule=datamodule) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_temperature_patches.py b/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_temperature_patches.py index 4535c2c6..0cb0af11 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_temperature_patches.py +++ b/examples/benchmarks/geolifeclef/geolifeclef2022/cnn_on_temperature_patches.py @@ -10,6 +10,7 @@ from pathlib import Path import hydra +import numpy as np import pytorch_lightning as pl from cnn_on_rgb_patches import ClassificationSystem from omegaconf import DictConfig @@ -43,10 +44,12 @@ def __init__( train_batch_size: int = 32, inference_batch_size: int = 256, num_workers: int = 8, + task: str = 'classification_multiclass', ): super().__init__(train_batch_size, inference_batch_size, num_workers) self.dataset_path = dataset_path self.minigeolifeclef = minigeolifeclef + self.task = task @property def train_transform(self): @@ -100,17 +103,19 @@ def get_dataset(self, split, transform, **kwargs): @hydra.main(version_base="1.3", config_path="config", config_name="mono_modal_3_channels_model") def main(cfg: DictConfig) -> None: - + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(Path(log_dir)/Path(cfg.loggers.log_dir_name), name=cfg.loggers.exp_name, version="") logger_tb.log_hyperparams(cfg) - datamodule = GeoLifeCLEF2022DataModule(**cfg.data) - - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + # Datamodule & Model + datamodule = GeoLifeCLEF2022DataModule(**cfg.data, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( @@ -123,9 +128,23 @@ def main(cfg: DictConfig) -> None: ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - trainer.fit(model, datamodule=datamodule) - trainer.validate(model, datamodule=datamodule) + # Run + if cfg.run.predict: + model_loaded = ClassificationSystem.load_from_checkpoint(classif_system.checkpoint_path, + model=classif_system.model, + hparams_preprocess=False) + + # Option 1: Predict on the entire test dataset (Pytorch Lightning) + predictions = model_loaded.predict(datamodule, trainer) + preds, probas = datamodule.predict_logits_to_class(predictions, + np.arange(datamodule.get_test_dataset().n_classes)) + datamodule.export_predict_csv(preds, probas, + out_dir=log_dir, out_name='predictions_test_dataset', top_k=3, return_csv=True) + print('Test dataset prediction (extract) : ', predictions[:1]) + else: + trainer.fit(classif_system, datamodule=datamodule) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/benchmarks/geolifeclef/geolifeclef2022/config/homogeneous_multi_modal_model.yaml b/examples/benchmarks/geolifeclef/geolifeclef2022/config/homogeneous_multi_modal_model.yaml index dbbf247d..be9199db 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2022/config/homogeneous_multi_modal_model.yaml +++ b/examples/benchmarks/geolifeclef/geolifeclef2022/config/homogeneous_multi_modal_model.yaml @@ -2,6 +2,10 @@ hydra: run: dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S} +run: + predict: false + checkpoint_path: + trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes accelerator: 'gpu' diff --git a/examples/benchmarks/geolifeclef/geolifeclef2022/config/mono_modal_3_channels_model.yaml b/examples/benchmarks/geolifeclef/geolifeclef2022/config/mono_modal_3_channels_model.yaml index 6d667f1b..5cee64ef 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2022/config/mono_modal_3_channels_model.yaml +++ b/examples/benchmarks/geolifeclef/geolifeclef2022/config/mono_modal_3_channels_model.yaml @@ -2,6 +2,10 @@ hydra: run: dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S} +run: + predict: false + checkpoint_path: + trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes accelerator: 'gpu' diff --git a/examples/benchmarks/geolifeclef/geolifeclef2023/cnn_on_rgbnir_glc23_patches.py b/examples/benchmarks/geolifeclef/geolifeclef2023/cnn_on_rgbnir_glc23_patches.py index ce4bcd4a..6ca3034c 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2023/cnn_on_rgbnir_glc23_patches.py +++ b/examples/benchmarks/geolifeclef/geolifeclef2023/cnn_on_rgbnir_glc23_patches.py @@ -166,31 +166,36 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = Sentinel2PatchesDataModule(**cfg.data, **cfg.task) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", save_on_train_epoch_end=True, save_last=True, ), ] - trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, num_sanity_val_steps=0, **cfg.trainer) + + # Run if cfg.run.predict: model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, + model=classif_system.model, hparams_preprocess=False) # Option 1: Predict on the entire test dataset (Pytorch Lightning) @@ -225,8 +230,8 @@ def main(cfg: DictConfig) -> None: print('Point prediction : ', prediction.shape, prediction) else: CrashHandler(trainer) - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) - trainer.validate(model, datamodule=datamodule) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/benchmarks/geolifeclef/geolifeclef2023/config/cnn_on_rgbnir_glc23_patches_train_multiclass.yaml b/examples/benchmarks/geolifeclef/geolifeclef2023/config/cnn_on_rgbnir_glc23_patches_train_multiclass.yaml index 550f125d..beaa22d1 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2023/config/cnn_on_rgbnir_glc23_patches_train_multiclass.yaml +++ b/examples/benchmarks/geolifeclef/geolifeclef2023/config/cnn_on_rgbnir_glc23_patches_train_multiclass.yaml @@ -10,7 +10,7 @@ data: inference_batch_size: 2 num_workers: 8 units: "pixel" - crs: 4326 + crs: 4326 trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes @@ -22,8 +22,8 @@ trainer: run: predict: false - checkpoint_path: - + checkpoint_path: # 'outputs/cnn_on_rgbnir_glc23_patches/2024-08-05_10-13-52/last.ckpt' + model: provider_name: "timm" # choose from ["timm", "torchvision"] model_name: "resnet18" @@ -63,4 +63,4 @@ optimizer: # num_labels: ${data.num_classes} task: - task: 'classification_multiclass' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] + task: 'classification_multiclass' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] diff --git a/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/config/glc24_cnn_multimodal_ensemble.yaml b/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/config/glc24_cnn_multimodal_ensemble.yaml index b11075ef..e1a84382 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/config/glc24_cnn_multimodal_ensemble.yaml +++ b/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/config/glc24_cnn_multimodal_ensemble.yaml @@ -2,86 +2,85 @@ hydra: run: dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S} +run: + predict: false + checkpoint_path: # "outputs/glc24_cnn_multimodal_ensemble/runOK_2024-08-12_11-50-01/last.ckpt" + +data: + root: "dataset/geolifeclef-2024/" + data_paths: + train: + landsat_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-landsat_time_series/" + bioclim_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-bioclimatic_monthly/" + sentinel_data_dir: "${data.root}PA_Train_SatellitePatches_RGB/pa_train_patches_rgb/" + test: + landsat_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-test-landsat_time_series/" + bioclim_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-test-bioclimatic_monthly/" + sentinel_data_dir: "${data.root}PA_Test_SatellitePatches_RGB/pa_test_patches_rgb/" + metadata_paths: + train: "${data.root}GLC24_PA_metadata_train_train-10.0min.csv" + val: "${data.root}GLC24_PA_metadata_train_val-10.0min.csv" + test: "${data.root}GLC24_PA_metadata_test.csv" + num_classes: &num_classes 11255 + download_data: True + train_batch_size: 64 + inference_batch_size: 16 + num_workers: 16 + +task: + task: "classification_multilabel" # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] + trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes accelerator: "gpu" devices: 'auto' max_epochs: 20 - # val_check_interval: 2 + val_check_interval: 100 check_val_every_n_epoch: 1 # log_every_n_steps: 100 -run: - predict: false - checkpoint_path: # "outputs/glc24_cnn_multimodal_ensemble/runOK_2024-06-24_19-14-48/last.ckpt" - model: - positive_weigh_factor: 10.0 - provider_name: "timm" # choose from ["timm", "torchvision"] - model_name: "resnet18" + provider_name: "malpolon" # choose from ["malpolon", "timm", "torchvision"] + model_name: "glc24_multimodal_ensemble" model_kwargs: pretrained: true # Deprecated in torchvision since 0.13 (replaced by "weights") but used by timm modifiers: - change_first_convolutional_layer: - num_input_channels: 6 change_last_layer: - num_outputs: 11255 + num_outputs: *num_classes optimizer: lr: 0.00025 weight_decay: 0 momentum: 0.9 nesterov: true + loss_kwargs: + pos_weight: 10.0 metrics: multilabel_accuracy: # callable: 'Fmetrics.classification.multilabel_accuracy' kwargs: - num_labels: 11255 + num_labels: *num_classes # threshold: 0.1 average: micro multilabel_recall: callable: 'Fmetrics.classification.multilabel_recall' kwargs: - num_labels: 11255 + num_labels: *num_classes # threshold: 0.1 average: micro multilabel_precision: callable: 'Fmetrics.classification.multilabel_precision' kwargs: - num_labels: 11255 + num_labels: *num_classes # threshold: 0.1 average: micro multilabel_f1-score: callable: 'Fmetrics.classification.multilabel_f1_score' kwargs: - num_labels: 11255 + num_labels: *num_classes # threshold: 0.1 average: micro -data: - root: "dataset/geolifeclef-2024/" - data_paths: - train: - landsat_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-landsat_time_series/" - bioclim_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-bioclimatic_monthly/" - sentinel_data_dir: "${data.root}PA_Train_SatellitePatches_RGB/pa_train_patches_rgb/" - test: - landsat_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-test-landsat_time_series/" - bioclim_data_dir: "${data.root}TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-test-bioclimatic_monthly/" - sentinel_data_dir: "${data.root}PA_Test_SatellitePatches_RGB/pa_test_patches_rgb/" - metadata_paths: - train: "${data.root}GLC24_PA_metadata_train_train-10.0min.csv" - val: "${data.root}GLC24_PA_metadata_train_val-10.0min.csv" - test: "${data.root}GLC24_PA_metadata_test-10.0min.csv" - num_classes: 11255 - download_data: True - train_batch_size: 64 - inference_batch_size: 16 - num_workers: 16 - -task: - task: "classification_multilabel" # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] - loggers: exp_name: "multimodal_resnet18-swint_ensemble" # Name of your experiment log_dir_name: "tensorboard_logs/" # Name of the logs directory diff --git a/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/glc24_cnn_multimodal_ensemble.py b/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/glc24_cnn_multimodal_ensemble.py index e320b562..f081fbc4 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/glc24_cnn_multimodal_ensemble.py +++ b/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/glc24_cnn_multimodal_ensemble.py @@ -1,5 +1,5 @@ - import logging +from pathlib import Path import hydra import numpy as np @@ -11,8 +11,10 @@ from malpolon.data.datasets.geolifeclef2024_pre_extracted import \ GLC24Datamodule from malpolon.logging import Summary -from malpolon.models.geolifeclef2024_multimodal_ensemble import ( - ClassificationSystemGLC24, MultimodalEnsemble) +from malpolon.models.custom_models.glc2024_multimodal_ensemble_model import \ + MultimodalEnsemble +from malpolon.models.custom_models.glc2024_pre_extracted_prediction_system import \ + ClassificationSystemGLC24 def set_seed(seed): @@ -43,20 +45,22 @@ def main(cfg: DictConfig) -> None: associated with this script. """ set_seed(69) + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version=cfg.loggers.exp_name) logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name=cfg.loggers.log_dir_name, version=cfg.loggers.exp_name) logger_tb.log_hyperparams(cfg) - logger = logging.getLogger("lightning.pytorch.core") logger.addHandler(logging.FileHandler(f"{log_dir}/core.log")) + # Datamodule & Model datamodule = GLC24Datamodule(**cfg.data, **cfg.task) - model = MultimodalEnsemble(num_classes=cfg.model.modifiers.change_last_layer.num_outputs, - positive_weigh_factor=cfg.model.positive_weigh_factor) - classif_system = ClassificationSystemGLC24(model, **cfg.optimizer) # multilabel + classif_system = ClassificationSystemGLC24(cfg.model, **cfg.optimizer, + checkpoint_path=cfg.run.checkpoint_path, + weights_dir=log_dir) # multilabel + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( @@ -71,21 +75,23 @@ def main(cfg: DictConfig) -> None: ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer, deterministic=True) + # Run if cfg.run.predict: - model_loaded = ClassificationSystemGLC24.load_from_checkpoint(cfg.run.checkpoint_path, + model_loaded = ClassificationSystemGLC24.load_from_checkpoint(classif_system.checkpoint_path, model=classif_system.model, hparams_preprocess=False, strict=False) predictions = model_loaded.predict(datamodule, trainer) preds, probas = datamodule.predict_logits_to_class(predictions, - np.arange(cfg.data.num_classes)) + np.arange(cfg.data.num_classes), + activation_fn=torch.nn.Sigmoid()) datamodule.export_predict_csv(preds, probas, out_dir=log_dir, out_name='predictions_test_dataset', top_k=25, return_csv=True) print('Test dataset prediction (extract) : ', predictions[:1]) else: - trainer.fit(classif_system, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=classif_system.checkpoint_path) trainer.validate(classif_system, datamodule=datamodule) diff --git a/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/inference_bioclim_time_series_resnet_softma_binary_loss.py b/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/inference_bioclim_time_series_resnet_softma_binary_loss.py index 24836c28..6f57c229 100644 --- a/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/inference_bioclim_time_series_resnet_softma_binary_loss.py +++ b/examples/benchmarks/geolifeclef/geolifeclef2024_pre_extracted/inference_bioclim_time_series_resnet_softma_binary_loss.py @@ -1,62 +1,91 @@ -import csv -import itertools -import os -import ssl +"""This script computes metrics off of model inference predictions. + +It computes the Precision, Recall, F1-score (micro, samples and macro) +for the top-25 predictions of a model inference predictions (in a CSV); +as well as the AUC (micro, samples and macro) for all the probabilities +(not just the top-25). + +Author: Theo Larcher + Alexis Joly +""" +from copy import deepcopy import numpy as np import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision.models as models -import torchvision.transforms as transforms -from sklearn.metrics import (f1_score, precision_recall_fscore_support, - roc_auc_score) -from sklearn.model_selection import train_test_split -from torch.optim.lr_scheduler import ReduceLROnPlateau -from torch.utils.data import Subset # Import Subset class -from torch.utils.data import DataLoader, Dataset -from torchvision.models import ResNet50_Weights +from sklearn.metrics import precision_recall_fscore_support, roc_auc_score +from tqdm import tqdm +# 0. Load data df_gt = pd.read_csv('GLC24_SOLUTION_FILE.csv') -df_preds = pd.read_csv('predictions_top25_GLC24_SOLUTION_FILE.csv', sep=';') +df_preds = pd.read_csv('predictions_GLC24_SOLUTION_FILE_logits.csv', sep=';') +for rowi, row in deepcopy(df_gt).iterrows(): + tsi = np.array(row['target_species_ids'].split()).astype(int) # Split the predictions string by space and convert to int + inds = np.where(tsi > 11254)[0] + vals = tsi[inds] + if inds.size > 0: + df_gt = df_gt.drop(rowi) + df_preds = df_preds.drop(rowi) + print(f"obs {rowi} of surveyId {row['surveyId']} removed because target_species_ids value {vals} out of range") + + +# 1. Convert data to usable types and compute one-hot encodings +res = pd.DataFrame(columns=['Precision_micro', 'Recall_micro', 'F1_micro', + 'Precision_samples', 'Recall_samples', 'F1_samples', + 'Precision_macro', 'Recall_macro', 'F1_macro', + 'AUC_micro', 'AUC_samples', 'AUC_macro']) obs_id = df_gt['surveyId'] targets = df_gt['target_species_ids'] targets = [list(map(int, x.split())) for x in targets] preds = df_preds['predictions'] -preds = [list(map(int, x.split())) for x in preds] +preds = np.array([list(map(int, x.split())) for x in preds]) probas = df_preds['probas'] -probas = [list(map(float, x.split())) for x in probas] +probas = np.array([list(map(float, x.split())) for x in probas]) +all_targets_oh = np.zeros((len(df_gt), 11255)) +all_probas = np.zeros_like(probas) +all_predictions_top25_oh = np.zeros((len(df_preds), 11255)) -# Calculate F1 score -all_targets = list(itertools.chain.from_iterable(targets)) -all_predictions = list(itertools.chain.from_iterable(preds)) -all_probas = list(itertools.chain.from_iterable(probas)) -print("SAMPLE: Precision, Recall, F1") -print(precision_recall_fscore_support(all_targets, all_predictions, average='samples', zero_division='warn')) -print("MACRO: Precision, Recall, F1 ") -print(precision_recall_fscore_support(all_targets, all_predictions, average='macro', zero_division='warn')) -print("MICRO: Precision, Recall, F1 ") -print(precision_recall_fscore_support(all_targets, all_predictions, average='micro', zero_division='warn')) +for k, (p, t) in tqdm(enumerate(zip(preds, targets)), total=len(targets)): + all_probas[k] = probas[k][np.argsort(p)] + for t2 in t: + all_targets_oh[k, t2] = 1 + for p2 in p[:25]: + all_predictions_top25_oh[k, p2] = 1 -# Find rows and columns with all zeros in both arrays -zero_cols_targets = np.all(all_targets == 0, axis=0) -ones_cols_targets = np.all(all_targets == 1, axis=0) +# 2. Compute Precision / Recall / F1-score +print('\nComputing Precision, Recall, F1-scores...') +prfs = {} +for avg in ['micro', 'samples', 'macro']: + prf = precision_recall_fscore_support(all_targets_oh, all_predictions_top25_oh, average=avg, zero_division=np.nan)[:3] + prfs[f'Precision_{avg}'] = prf[0] + prfs[f'Recall_{avg}'] = prf[1] + prfs[f'F1_{avg}'] = prf[2] + print(f"{avg.upper()}: Precision, Recall, F1", prf) -# Combine zero rows and columns from both arrays -zero_cols = zero_cols_targets #& ones_cols_targets +# 3. Compute AUCs +print('\nComputing AUCs...') +# Find rows and columns with all zeros in both arrays, that is to say +# species that are never observed in any plot according to the ground truth +zero_cols_targets = np.all(all_targets_oh == 0, axis=0) +ones_cols_targets = np.all(all_targets_oh == 1, axis=0) +zero_cols = zero_cols_targets | ones_cols_targets # Filter out rows and columns containing only zeros -filtered_targets = all_targets[:][:,~zero_cols] -filtered_predictions = all_predictions[:][:,~zero_cols] +filtered_targets = all_targets_oh[:][:, ~zero_cols] +filtered_probas = all_probas[:][:, ~zero_cols] +filtered_predictions_top25 = all_predictions_top25_oh[:][:, ~zero_cols] +aucs = {} +for avg in ['micro', 'samples', 'macro']: + auc = roc_auc_score(filtered_targets, filtered_probas, average=avg) + aucs[f'AUC_{avg}'] = auc + print(f"{avg.upper()}: AUC", auc) -np.savetxt('filtered_targets.txt', np.sum(filtered_targets, axis=0)) -print("micro:",roc_auc_score(filtered_targets, filtered_predictions, average='micro')) -print("samples:",roc_auc_score(filtered_targets, filtered_predictions, average='samples')) -print("macro:",roc_auc_score(filtered_targets, filtered_predictions, average='macro')) +# 4. Save results +res.loc[0] = prfs | aucs +res.to_csv('Inference_PRC-AUC.csv', index=False) +print('\nResults saved to Inference_PRC-AUC.csv') diff --git a/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py b/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py index 345bccfc..04fd7419 100644 --- a/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py +++ b/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py @@ -134,34 +134,38 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(Path(log_dir)/Path(cfg.loggers.log_dir_name), name=cfg.loggers.exp_name, version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = MicroGeoLifeCLEF2022DataModule(**cfg.data) - cfg_model = hydra.utils.instantiate(cfg.model) - model = ClassificationSystem(cfg_model, **cfg.optimizer, **cfg.task) + classif_system = ClassificationSystem(cfg_model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", save_on_train_epoch_end=True, save_last=True, - every_n_train_steps=100, + every_n_train_steps=20, ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) + # Run if cfg.run.predict: model_loaded = ClassificationSystem.load_from_checkpoint( - cfg.run.checkpoint_path, model=model.model, hparams_preprocess=False + cfg.run.checkpoint_path, model=classif_system.model, hparams_preprocess=False ) # Option 1: Predict on the entire test dataset (Pytorch Lightning) @@ -190,8 +194,8 @@ def main(cfg: DictConfig) -> None: out_dir=log_dir, out_name='prediction_point', single_point_query=query_point, return_csv=True) print('Point prediction : ', prediction.shape, prediction) else: - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) - trainer.validate(model, datamodule=datamodule) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_patches.py b/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_patches.py index 241a0d81..abbce64f 100644 --- a/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_patches.py +++ b/examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_patches.py @@ -108,22 +108,25 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(Path(log_dir)/Path(cfg.loggers.log_dir_name), name=cfg.loggers.exp_name, version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = MicroGeoLifeCLEF2022DataModule(**cfg.data) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) - + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", save_on_train_epoch_end=True, save_last=True, @@ -132,9 +135,10 @@ def main(cfg: DictConfig) -> None: ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) + # Run if cfg.run.predict: - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, + model_loaded = ClassificationSystem.load_from_checkpoint(classif_system.checkpoint_path, + model=classif_system.model, hparams_preprocess=False) # Option 1: Predict on the entire test dataset (Pytorch Lightning) @@ -154,7 +158,7 @@ def main(cfg: DictConfig) -> None: test_data_point = test_data[0][0] test_data_point = test_data_point.resize_(1, *test_data_point.shape) - prediction = model_loaded.predict_point(cfg.run.checkpoint_path, + prediction = model_loaded.predict_point(classif_system.checkpoint_path, test_data_point, ['model.', '']) preds, probas = datamodule.predict_logits_to_class(prediction, @@ -163,8 +167,8 @@ def main(cfg: DictConfig) -> None: out_dir=log_dir, out_name='prediction_point', single_point_query=query_point, return_csv=True) print('Point prediction : ', prediction.shape, prediction) else: - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) - trainer.validate(model, datamodule=datamodule) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=classif_system.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/custom_train/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py b/examples/custom_train/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py index 694ba235..60a62804 100644 --- a/examples/custom_train/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py +++ b/examples/custom_train/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py @@ -31,21 +31,25 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = Sentinel2TorchGeoDataModule(**cfg.data, **cfg.task) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", save_on_train_epoch_end=True, save_last=True, @@ -54,9 +58,10 @@ def main(cfg: DictConfig) -> None: ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) + # Run if cfg.run.predict: - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, + model_loaded = ClassificationSystem.load_from_checkpoint(classif_system.checkpoint_path, + model=classif_system.model, hparams_preprocess=False) # Option 1: Predict on the entire test dataset (Pytorch Lightning) @@ -86,8 +91,8 @@ def main(cfg: DictConfig) -> None: out_dir=log_dir, out_name='prediction_point', single_point_query=query_point, return_csv=True) print('Point prediction : ', prediction.shape, prediction) else: - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) - trainer.validate(model, datamodule=datamodule) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=classif_system.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/custom_train/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml b/examples/custom_train/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml index 0ca8be41..d055361f 100644 --- a/examples/custom_train/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml +++ b/examples/custom_train/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml @@ -4,7 +4,7 @@ hydra: run: predict: false - checkpoint_path: + checkpoint_path: # 'outputs/cnn_on_rgbnir_torchgeo/train_multilabel/last.ckpt' data: num_classes: 5 diff --git a/examples/custom_train/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py b/examples/custom_train/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py index 1bcc50d7..4b933663 100644 --- a/examples/custom_train/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py +++ b/examples/custom_train/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py @@ -29,21 +29,25 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = ConcatTorchGeoDataModule(**cfg.data, **cfg.task) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task, + checkpoint_path=cfg.run.checkpoint_path) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", save_on_train_epoch_end=True, save_last=True, @@ -53,9 +57,10 @@ def main(cfg: DictConfig) -> None: ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) + # Run if cfg.run.predict: - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, + model_loaded = ClassificationSystem.load_from_checkpoint(classif_system.checkpoint_path, + model=classif_system.model, hparams_preprocess=False) # Option 1: Predict on the entire test dataset (Pytorch Lightning) @@ -84,8 +89,8 @@ def main(cfg: DictConfig) -> None: out_dir=log_dir, out_name='prediction_point', single_point_query=query_point, return_csv=True) print('Point prediction : ', prediction.shape, prediction) else: - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.run.checkpoint_path) - trainer.validate(model, datamodule=datamodule) + trainer.fit(classif_system, datamodule=datamodule, ckpt_path=classif_system.checkpoint_path) + trainer.validate(classif_system, datamodule=datamodule) if __name__ == "__main__": diff --git a/examples/custom_train/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml b/examples/custom_train/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml index 5cd202a7..d7b6bcde 100644 --- a/examples/custom_train/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml +++ b/examples/custom_train/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml @@ -4,7 +4,7 @@ hydra: run: predict: false - checkpoint_path: + checkpoint_path: 'outputs/cnn_on_rgbnir_concat/train_multilabel/last.ckpt' data: num_classes: &num_classes 5 diff --git a/examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py b/examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py index fb7de0c9..3251adf7 100644 --- a/examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py +++ b/examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py @@ -129,38 +129,40 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = MicroGeoLifeCLEF2022DataModule(**cfg.data) - cfg_model = hydra.utils.instantiate(cfg.model) - model = ClassificationSystem(cfg_model, **cfg.optimizer, **cfg.task) - change_first_convolutional_layer_modifier(model, + classif_system = ClassificationSystem(cfg_model, **cfg.optimizer, **cfg.task) + change_first_convolutional_layer_modifier(classif_system, num_input_channels=4, new_conv_layer_init_func=NewConvolutionalLayerInitFuncStrategy( strategy='red_pretraining', rescaling=True )) + model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, + model=classif_system.model, + hparams_preprocess=False) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, - hparams_preprocess=False) - + # Run if cfg.run.predict_type == 'test_dataset': predictions = model_loaded.predict(datamodule, trainer) preds, probas = datamodule.predict_logits_to_class(predictions, diff --git a/examples/inference/micro_geolifeclef2022/cnn_on_rgb_patches.py b/examples/inference/micro_geolifeclef2022/cnn_on_rgb_patches.py index 0aac151c..9228ef2c 100644 --- a/examples/inference/micro_geolifeclef2022/cnn_on_rgb_patches.py +++ b/examples/inference/micro_geolifeclef2022/cnn_on_rgb_patches.py @@ -106,31 +106,33 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = MicroGeoLifeCLEF2022DataModule(**cfg.data) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, + model=classif_system.model, + hparams_preprocess=False) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) - + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, - hparams_preprocess=False) - + # Run if cfg.run.predict_type == 'test_dataset': predictions = model_loaded.predict(datamodule, trainer) preds, probas = datamodule.predict_logits_to_class(predictions, diff --git a/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_nir_patches_config.yaml b/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_nir_patches_config.yaml index 3945fc99..dae98b1f 100644 --- a/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_nir_patches_config.yaml +++ b/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_nir_patches_config.yaml @@ -20,4 +20,3 @@ model: rescaling: true change_last_layer: num_outputs: *num_classes - diff --git a/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_patches_config.yaml b/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_patches_config.yaml index 03d26991..012a6214 100644 --- a/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_patches_config.yaml +++ b/examples/inference/micro_geolifeclef2022/config/cnn_on_rgb_patches_config.yaml @@ -12,9 +12,9 @@ data: train_batch_size: 32 inference_batch_size: 256 num_workers: 8 - + task: - task: 'classification_multiclass' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] + task: 'classification_multiclass' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes @@ -45,4 +45,3 @@ optimizer: multiclass_accuracy: kwargs: num_classes: *num_classes - diff --git a/examples/inference/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py b/examples/inference/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py index 2d4bb636..1936f63c 100644 --- a/examples/inference/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py +++ b/examples/inference/sentinel-2a-rgbnir/cnn_on_rgbnir_torchgeo.py @@ -29,30 +29,33 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = Sentinel2TorchGeoDataModule(**cfg.data, **cfg.task) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + classif_system = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, + model=classif_system.model, + hparams_preprocess=False) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_system.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_system.metrics.keys()))}/val", mode="max", ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, - hparams_preprocess=False) - + # Run if cfg.run.predict_type == 'test_dataset': # Option 1: Predict on the entire test dataset (Pytorch Lightning) predictions = model_loaded.predict(datamodule, trainer) diff --git a/examples/inference/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml b/examples/inference/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml index 72dd4eb1..93b138dc 100644 --- a/examples/inference/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml +++ b/examples/inference/sentinel-2a-rgbnir/config/cnn_on_rgbnir_torchgeo_config.yaml @@ -7,7 +7,7 @@ run: checkpoint_path: ??? data: - num_classes: &num_classes 10 + num_classes: &num_classes 5 dataset_path: "dataset/" labels_name: "observations.csv" download_data_sample: True @@ -25,7 +25,7 @@ data: "split": "subset"} task: - task: 'classification_multilabel' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] + task: 'classification_multilabel' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes diff --git a/examples/inference/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py b/examples/inference/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py index adf040ba..2ee7ef27 100644 --- a/examples/inference/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py +++ b/examples/inference/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py @@ -29,30 +29,33 @@ def main(cfg: DictConfig) -> None: hydra config dictionary created from the .yaml config file associated with this script. """ + # Loggers log_dir = hydra.core.hydra_config.HydraConfig.get().runtime.output_dir logger_csv = pl.loggers.CSVLogger(log_dir, name="", version="") logger_csv.log_hyperparams(cfg) logger_tb = pl.loggers.TensorBoardLogger(log_dir, name="tensorboard_logs", version="") logger_tb.log_hyperparams(cfg) + # Datamodule & Model datamodule = ConcatTorchGeoDataModule(**cfg.data, **cfg.task) - model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + classif_model = ClassificationSystem(cfg.model, **cfg.optimizer, **cfg.task) + model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, + model=classif_model.model, + hparams_preprocess=False) + # Lightning Trainer callbacks = [ Summary(), ModelCheckpoint( dirpath=log_dir, - filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(model.metrics.keys()))}/val" + ":.4f}", - monitor=f"{next(iter(model.metrics.keys()))}/val", + filename="checkpoint-{epoch:02d}-{step}-{" + f"{next(iter(classif_model.metrics.keys()))}/val" + ":.4f}", + monitor=f"{next(iter(classif_model.metrics.keys()))}/val", mode="max", ), ] trainer = pl.Trainer(logger=[logger_csv, logger_tb], callbacks=callbacks, **cfg.trainer) - model_loaded = ClassificationSystem.load_from_checkpoint(cfg.run.checkpoint_path, - model=model.model, - hparams_preprocess=False) - + # Run if cfg.run.predict_type == 'test_dataset': # Option 1: Predict on the entire test dataset (Pytorch Lightning) predictions = model_loaded.predict(datamodule, trainer) diff --git a/examples/inference/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml b/examples/inference/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml index 0e7098c0..7f503aed 100644 --- a/examples/inference/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml +++ b/examples/inference/sentinel-2a-rgbnir_bioclim/config/cnn_on_rgbnir_concat_config.yaml @@ -41,7 +41,7 @@ data: # binary_positive_classes: [1] # For binary classification, define which classes are considered as 1. Others will be considered 0 task: - task: 'classification_multilabel' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] + task: 'classification_multilabel' # ['classification_binary', 'classification_multiclass', 'classification_multilabel'] trainer: # gpus: 1 # Deprecated since pytorchlightning 1.7, removed in 2.0. Replaced by the 2 next attributes @@ -50,7 +50,7 @@ trainer: max_epochs: 2 # val_check_interval: 10 check_val_every_n_epoch: 1 - + model: provider_name: "timm" # choose from ["timm", "torchvision"] model_name: "resnet18" diff --git a/malpolon/data/data_module.py b/malpolon/data/data_module.py index df632ea2..fad09ad2 100644 --- a/malpolon/data/data_module.py +++ b/malpolon/data/data_module.py @@ -260,7 +260,7 @@ class labels and corresponding probabilities """ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") classes = torch.tensor(classes).to(device) - probas = activation_fn(predictions) + probas = activation_fn(predictions) if activation_fn is not None else predictions if 'binary' in self.task: class_preds = probas.round() else: @@ -392,7 +392,8 @@ def export_predict_csv(self, 'probas': tuple(probas[:, :top_k].astype(str))}) else: test_ds = self.get_test_dataset() - targets = test_ds.targets + targets = test_ds.targets if test_ds.targets is not None else [-1] * len(predictions) + print('Constructing predictions CSV file...') df = pd.DataFrame({'observation_id': test_ds.observation_ids, 'lon': [None] * len(test_ds) if not hasattr(test_ds, 'coordinates') else test_ds.coordinates[:, 0], 'lat': [None] * len(test_ds) if not hasattr(test_ds, 'coordinates') else test_ds.coordinates[:, 1], @@ -409,6 +410,7 @@ def export_predict_csv(self, for key in ['probas', 'predictions', 'target_species_id']: if not isinstance(df.loc[0, key], str) and len(df.loc[0, key]) >= 1: df[key] = df[key].apply(' '.join) + print('Writing predictions CSV file...') df.to_csv(fp, index=False, sep=';', **kwargs) if return_csv: return df diff --git a/malpolon/data/datasets/geolifeclef2022.py b/malpolon/data/datasets/geolifeclef2022.py index 296dac0d..74f275a8 100644 --- a/malpolon/data/datasets/geolifeclef2022.py +++ b/malpolon/data/datasets/geolifeclef2022.py @@ -445,7 +445,7 @@ def __getitem__( target = self.target_transform(target) return patches, target - return patches + return patches, -1 class MiniGeoLifeCLEF2022Dataset(GeoLifeCLEF2022Dataset): @@ -522,23 +522,27 @@ def _load_observation_data( index_col="observation_id", )[:600] - file_name = "minigeolifeclef2022_species_details.csv" - with resources.path(DATA_MODULE, file_name) as species_file_path: - df_species = pd.read_csv( - species_file_path, - sep=";", - index_col="species_id", - )[:600] - - df = df[np.isin(df["species_id"], df_species.index)] - value_counts = df.species_id.value_counts() - species_id = value_counts.iloc[:100].index - df_species = df_species.loc[species_id] - df = df[np.isin(df["species_id"], df_species.index)] - - label_encoder = LabelEncoder().fit(df_species.index) - df["species_id"] = label_encoder.transform(df["species_id"]) - df_species.index = label_encoder.transform(df_species.index) + if subset == 'test': + df = df.iloc[np.random.randint(0, len(df), 100)] + df['species_id'] = [None] * len(df) + else: + file_name = "minigeolifeclef2022_species_details.csv" + with resources.path(DATA_MODULE, file_name) as species_file_path: + df_species = pd.read_csv( + species_file_path, + sep=";", + index_col="species_id", + )[:600] + + df = df[np.isin(df["species_id"], df_species.index)] + value_counts = df.species_id.value_counts() + species_id = value_counts.iloc[:100].index + df_species = df_species.loc[species_id] + df = df[np.isin(df["species_id"], df_species.index)] + + label_encoder = LabelEncoder().fit(df_species.index) + df["species_id"] = label_encoder.transform(df["species_id"]) + df_species.index = label_encoder.transform(df_species.index) if subset not in ["train+val", "test"]: ind = df.index[df["subset"] == subset] diff --git a/malpolon/models/__init__.py b/malpolon/models/__init__.py index 704476dd..49e322ff 100644 --- a/malpolon/models/__init__.py +++ b/malpolon/models/__init__.py @@ -1,4 +1,5 @@ -from .standard_prediction_systems import * # noqa: F403 +from .standard_prediction_systems import (ClassificationSystem, + GenericPredictionSystem) __all__ = [ # noqa: F405 "GenericPredictionSystem", diff --git a/malpolon/models/custom_models/__init__.py b/malpolon/models/custom_models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/malpolon/models/custom_models/glc2024_multimodal_ensemble_model.py b/malpolon/models/custom_models/glc2024_multimodal_ensemble_model.py new file mode 100644 index 00000000..6f72cebf --- /dev/null +++ b/malpolon/models/custom_models/glc2024_multimodal_ensemble_model.py @@ -0,0 +1,79 @@ +"""This module provides a Multimodal Ensemble model for GeoLifeCLEF2024 data. + +Author: Lukas Picek + Theo Larcher + +License: GPLv3 +Python version: 3.10.6 +""" +from typing import Any, Callable, Mapping, Optional, Union + +import omegaconf +import torch +from omegaconf import OmegaConf +from torch import Tensor, nn +from torch.optim.lr_scheduler import CosineAnnealingLR +from torchvision import models + + +class MultimodalEnsemble(nn.Module): + """Multimodal ensemble model processing Sentinel-2A, Landsat & Bioclimatic data. + + Inherits torch nn.Module. + """ + def __init__(self, + num_classes: int = 11255, + pretrained: bool = False, + **kwargs): + """Class constructor. + + Parameters + ---------- + num_classes : int, optional + numbre of classes, by default 11255 + pretrained : bool, optional + if True, downloads the model's weights from our remote + storage platform, by default False + """ + super().__init__(**kwargs) + self.pretrained = pretrained + self.landsat_model = models.resnet18(weights=None) + self.landsat_norm = nn.LayerNorm([6, 4, 21]) + # Modify the first convolutional layer to accept 6 channels instead of 3 + self.landsat_model.conv1 = nn.Conv2d(6, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.landsat_model.maxpool = nn.Identity() + + self.bioclim_model = models.resnet18(weights=None) + self.bioclim_norm = nn.LayerNorm([4, 19, 12]) + # Modify the first convolutional layer to accept 4 channels instead of 3 + self.bioclim_model.conv1 = nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bioclim_model.maxpool = nn.Identity() + + self.sentinel_model = models.swin_t(weights="IMAGENET1K_V1") + # Modify the first layer to accept 4 channels instead of 3 + self.sentinel_model.features[0][0] = nn.Conv2d(4, 96, kernel_size=(4, 4), stride=(4, 4)) + self.sentinel_model.head = nn.Identity() + + self.ln1 = nn.LayerNorm(1000) + self.ln2 = nn.LayerNorm(1000) + self.fc1 = nn.Linear(2768, 4096) + self.fc2 = nn.Linear(4096, num_classes) + + self.dropout = nn.Dropout(p=0.1) + + def forward(self, x, y, z): # noqa: D102 pylint: disable=C0116 + x = self.landsat_norm(x) + x = self.landsat_model(x) + x = self.ln1(x) + + y = self.bioclim_norm(y) + y = self.bioclim_model(y) + y = self.ln2(y) + + z = self.sentinel_model(z) + + xyz = torch.cat((x, y, z), dim=1) + xyz = self.fc1(xyz) + xyz = self.dropout(xyz) + out = self.fc2(xyz) + return out \ No newline at end of file diff --git a/malpolon/models/geolifeclef2024_multimodal_ensemble.py b/malpolon/models/custom_models/glc2024_pre_extracted_prediction_system.py similarity index 51% rename from malpolon/models/geolifeclef2024_multimodal_ensemble.py rename to malpolon/models/custom_models/glc2024_pre_extracted_prediction_system.py index 78551489..5f81dabb 100644 --- a/malpolon/models/geolifeclef2024_multimodal_ensemble.py +++ b/malpolon/models/custom_models/glc2024_pre_extracted_prediction_system.py @@ -6,6 +6,7 @@ License: GPLv3 Python version: 3.10.6 """ +from pathlib import Path from typing import Any, Callable, Mapping, Optional, Union import omegaconf @@ -15,9 +16,8 @@ from torch.optim.lr_scheduler import CosineAnnealingLR from torchvision import models -from malpolon.models import ClassificationSystem - -from .utils import check_optimizer +from malpolon.models.standard_prediction_systems import ClassificationSystem +from malpolon.models.utils import check_optimizer class ClassificationSystemGLC24(ClassificationSystem): @@ -33,18 +33,66 @@ def __init__( momentum: float = 0.9, nesterov: bool = True, metrics: Optional[dict[str, Callable]] = None, - task: str = 'classification_binary', + task: str = 'classification_multilabel', loss_kwargs: Optional[dict] = {}, - hparams_preprocess: bool = True + hparams_preprocess: bool = True, + weights_dir: str = 'outputs/glc24_cnn_multimodal_ensemble/', + checkpoint_path: Optional[str] = None ): + """Class constructor + + Parameters + ---------- + model : Union[torch.nn.Module, Mapping] + model to use, either a torch model object, or a mapping + (dictionary from config file) used to load and build + the model + lr : float, optional + learning rate, by default 1e-2 + weight_decay : float, optional + weight decay, by default 0 + momentum : float + value of momentum + nesterov : bool + if True, uses Nesterov's momentum + metrics : dict + dictionnary containing the metrics to compute. + Keys must match metrics' names and have a subkey with each + metric's functional methods as value. This subkey is either + created from the `malpolon.models.utils.FMETRICS_CALLABLES` + constant or supplied, by the user directly. + task : str, optional + Machine learning task (used to format labels accordingly), + by default 'classification_multiclass'. The value determines + the loss to be selected. if 'multilabel' or 'binary' is + in the task, the BCEWithLogitsLoss is selected, otherwise + the CrossEntropyLoss is used. + loss_kwargs : Optional[dict], optional + loss parameters, by default {} + hparams_preprocess : bool, optional + if True performs preprocessing operations on the hyperparameters, + by default True + weights_dir : str, optional + directory where to download the model weights, + by default 'outputs/glc24_cnn_multimodal_ensemble/' + checkpoint_path : Optional[str], optional + path to the model checkpoint to load either to resume + a previous training, perform transfer learning or run in + prediction mode (inference), by default None + """ if isinstance(loss_kwargs, omegaconf.dictconfig.DictConfig): loss_kwargs = OmegaConf.to_container(loss_kwargs, resolve=True) if 'pos_weight' in loss_kwargs.keys(): length = metrics['multilabel_f1-score'].kwargs.num_labels loss_kwargs['pos_weight'] = Tensor([loss_kwargs['pos_weight']] * length) - super().__init__(model, lr, weight_decay, momentum, nesterov, metrics, task, loss_kwargs, hparams_preprocess) - optimizer = torch.optim.AdamW(model.parameters(), lr=lr) + super().__init__(model, lr, weight_decay, momentum, nesterov, metrics, task, loss_kwargs, hparams_preprocess, checkpoint_path) + optimizer = torch.optim.AdamW(self.model.parameters(), lr=lr) self.optimizer = check_optimizer(optimizer) + if self.model.pretrained and not self.checkpoint_path: + self.download_weights("https://lab.plantnet.org/seafile/f/d780d4ab7f6b419194f9/?dl=1", + weights_dir, + filename="pretrained.ckpt", + md5="69111dd8013fcd8e8f4504def774f3a5") def configure_optimizers(self): """Override default optimizer and scheduler. @@ -77,11 +125,11 @@ def _step( x_landsat, x_bioclim, x_sentinel, y, survey_id = batch y_hat = self(x_landsat, x_bioclim, x_sentinel) - pos_weight = y * self.model.positive_weigh_factor - self.loss.pos_weight = pos_weight # Proper way would be to forward pos_weight to loss instantiation via loss_kwargs, but pos_weight must be a tensor, i.e. have access to y -> Not possible in Malpolon as datamodule and optimizer instantiations are separate - + loss_pos_weight = self.loss.pos_weight # save initial loss parameter value + self.loss.pos_weight = y * torch.Tensor(self.loss.pos_weight).to(y) # Proper way would be to forward pos_weight to loss instantiation via loss_kwargs, but pos_weight must be a tensor, i.e. have access to y -> Not possible in Malpolon as datamodule and optimizer instantiations are separate loss = self.loss(y_hat, self._cast_type_to_loss(y)) # Shape mismatch for binary: need to 'y = y.unsqueeze(1)' (or use .reshape(2)) to cast from [2] to [2,1] and cast y to float with .float() self.log(f"loss/{split}", loss, **log_kwargs) + self.loss.pos_weight = loss_pos_weight # restore initial loss parameter value to not alter lightning module state_dict for metric_name, metric_func in self.metrics.items(): if isinstance(metric_func, dict): @@ -95,54 +143,3 @@ def _step( def predict_step(self, batch, batch_idx, dataloader_idx=0): # noqa: D102 pylint: disable=C0116 x_landsat, x_bioclim, x_sentinel, y, survey_id = batch return self(x_landsat, x_bioclim, x_sentinel) - - -class MultimodalEnsemble(nn.Module): - """Multimodal ensemble model processing Sentinel-2A, Landsat & Bioclimatic data. - - Inherits torch nn.Module. - """ - def __init__(self, num_classes=11255, positive_weigh_factor=1.0, **kwargs): - super().__init__(**kwargs) - self.positive_weigh_factor = positive_weigh_factor - - self.landsat_model = models.resnet18(weights=None) - self.landsat_norm = nn.LayerNorm([6, 4, 21]) - # Modify the first convolutional layer to accept 6 channels instead of 3 - self.landsat_model.conv1 = nn.Conv2d(6, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.landsat_model.maxpool = nn.Identity() - - self.bioclim_model = models.resnet18(weights=None) - self.bioclim_norm = nn.LayerNorm([4, 19, 12]) - # Modify the first convolutional layer to accept 4 channels instead of 3 - self.bioclim_model.conv1 = nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.bioclim_model.maxpool = nn.Identity() - - self.sentinel_model = models.swin_t(weights="IMAGENET1K_V1") - # Modify the first layer to accept 4 channels instead of 3 - self.sentinel_model.features[0][0] = nn.Conv2d(4, 96, kernel_size=(4, 4), stride=(4, 4)) - self.sentinel_model.head = nn.Identity() - - self.ln1 = nn.LayerNorm(1000) - self.ln2 = nn.LayerNorm(1000) - self.fc1 = nn.Linear(2768, 4096) - self.fc2 = nn.Linear(4096, num_classes) - - self.dropout = nn.Dropout(p=0.1) - - def forward(self, x, y, z): # noqa: D102 pylint: disable=C0116 - x = self.landsat_norm(x) - x = self.landsat_model(x) - x = self.ln1(x) - - y = self.bioclim_norm(y) - y = self.bioclim_model(y) - y = self.ln2(y) - - z = self.sentinel_model(z) - - xyz = torch.cat((x, y, z), dim=1) - xyz = self.fc1(xyz) - xyz = self.dropout(xyz) - out = self.fc2(xyz) - return out diff --git a/malpolon/models/multi_modal.py b/malpolon/models/custom_models/multi_modal.py similarity index 99% rename from malpolon/models/multi_modal.py rename to malpolon/models/custom_models/multi_modal.py index 8a0fdad9..2bef0060 100644 --- a/malpolon/models/multi_modal.py +++ b/malpolon/models/custom_models/multi_modal.py @@ -13,7 +13,7 @@ from pytorch_lightning.utilities import move_data_to_device from torch import nn -from .utils import check_model +from malpolon.models.utils import check_model if TYPE_CHECKING: from typing import Any, Mapping, Optional, Union diff --git a/malpolon/models/model_builder.py b/malpolon/models/model_builder.py index a2aa0fe0..87850410 100644 --- a/malpolon/models/model_builder.py +++ b/malpolon/models/model_builder.py @@ -17,12 +17,17 @@ from torch import nn from torchvision import models +from malpolon.models.custom_models.glc2024_multimodal_ensemble_model import \ + MultimodalEnsemble + if TYPE_CHECKING: from typing import Any, Callable, Optional Provider = Callable[..., nn.Module] Modifier = Callable[..., nn.Module] +MALPOLON_MODELS = {'glc24_multimodal_ensemble': MultimodalEnsemble,} + class _ModelBuilder: """General class to build models.""" @@ -152,6 +157,27 @@ def timm_model_provider( ) return model +def malpolon_model_provider( + model_name: str, *model_args: Any, **model_kwargs: Any +) -> nn.Module: + """Return a model from Malpolon's models list. + + This method uses Malpolon's internal model listing to retrieve a + model. + + Parameters + ---------- + model_name : str + name of the model to retrieve from torchvision's library + + Returns + ------- + nn.Module + model object + """ + model = MALPOLON_MODELS[model_name] + model = model(*model_args, **model_kwargs) + return model def _find_module_of_type( module: nn.Module, module_type: type, order: str @@ -306,6 +332,7 @@ def change_last_layer_to_identity_modifier(model: nn.Module) -> nn.Module: ModelBuilder.register_provider("torchvision", torchvision_model_provider) ModelBuilder.register_provider("timm", timm_model_provider) +ModelBuilder.register_provider("malpolon", malpolon_model_provider) ModelBuilder.register_modifier( "change_first_convolutional_layer", diff --git a/malpolon/models/standard_prediction_systems.py b/malpolon/models/standard_prediction_systems.py index 32968801..0615146a 100644 --- a/malpolon/models/standard_prediction_systems.py +++ b/malpolon/models/standard_prediction_systems.py @@ -6,11 +6,14 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytorch_lightning as pl import torch import torchmetrics.functional as Fmetrics +from torchvision.datasets.utils import (download_and_extract_archive, + download_url, extract_archive) from malpolon.models.utils import check_metric @@ -36,6 +39,8 @@ class GenericPredictionSystem(pl.LightningModule): metrics: dict Dictionary containing the metrics to monitor during the training and to compute at test time. + save_hyperparameters: bool + Save arguments to hparams attribute. """ def __init__( @@ -44,7 +49,7 @@ def __init__( loss: torch.nn.modules.loss._Loss, optimizer: torch.optim.Optimizer, metrics: Optional[dict[str, Callable]] = None, - save_hyperparameters: Optional[bool] = True + save_hyperparameters: Optional[bool] = True, ): if save_hyperparameters: self.save_hyperparameters(ignore=['model', 'loss']) @@ -53,13 +58,71 @@ def __init__( # indefinitely after returning self.optimizer. It is unclear why. super().__init__() + self.checkpoint_path = None if not hasattr(self, 'checkpoint_path') else self.checkpoint_path # Avoids overwriting the attribute. This class will need to be re-written properly alongside ClassificationSystem self.model = check_model(model) self.optimizer = check_optimizer(optimizer) self.loss = check_loss(loss) self.metrics = metrics or {} - def forward(self, x: Any) -> Any: - return self.model(x) + def _check_integrity(self, fp: str) -> bool: + return (fp).exists() + + def download_weights( + self, + url: str, + out_path: str, + filename: str, + md5: Optional[str] = None, + ): + """Download pretrained weihgts from a remote repository. + + Downloads weigths and ajusts self.checkpoint_path accordingly. + This method is intended to be used to perform transfer learning + or resume a model training later on and/or on a different + machine. + Downloaded content can either be a single file or a pre-zipped + directory containing all training filee, in which case the + value of checkpoint_path is updated to point inside that + unzipped folder. + + Parameters + ---------- + url : str + url to the path or directory to download + out_path : str + local root path where to to extract the downloaded content + filename : str + name of the file (in case of a single file download) or the + directory (in case of a zip download) on local disk + md5 : Optional[str], optional + checksum value to verify the integrity of the downloaded + content, by default None + """ + path = self.checkpoint_path + if Path(filename).suffix == '.zip': + path = Path(out_path) / Path(filename).stem / 'pretrained.ckpt' + if self._check_integrity(path): + print("Files already downloaded and verified") + return + download_and_extract_archive( + url, + out_path, + filename=filename, + md5=md5, + remove_finished=True, + ) + else: + path = Path(out_path) / 'pretrained.ckpt' + if self._check_integrity(path): + print("Files already downloaded and verified") + return + download_url( + url, + out_path, + filename=filename, + md5=md5, + ) + self.checkpoint_path = path def _cast_type_to_loss(self, y): if isinstance(self.loss, torch.nn.CrossEntropyLoss) and len(y.shape) == 1 or\ @@ -69,6 +132,9 @@ def _cast_type_to_loss(self, y): y = y.to(torch.float32) return y + def forward(self, x: Any) -> Any: + return self.model(x) + def _step( self, split: str, batch: tuple[Any, Any], batch_idx: int ) -> Union[Tensor, dict[str, Any]]: @@ -253,6 +319,7 @@ def __init__( task: str = 'classification_binary', loss_kwargs: Optional[dict] = {}, hparams_preprocess: bool = True, + checkpoint_path: Optional[str] = None ): """Class constructor. @@ -293,6 +360,7 @@ def __init__( self.momentum = momentum self.nesterov = nesterov + self.checkpoint_path = checkpoint_path model = check_model(model) optimizer = torch.optim.SGD( diff --git a/malpolon/models/utils.py b/malpolon/models/utils.py index bebe5505..a840c1fc 100644 --- a/malpolon/models/utils.py +++ b/malpolon/models/utils.py @@ -21,7 +21,6 @@ 'multiclass_accuracy': Fmetrics.classification.multiclass_accuracy, 'multilabel_accuracy': Fmetrics.classification.multilabel_accuracy, } - class CrashHandler(): """Saves the model in case of unexpected crash or user interruption.""" def __init__(self, trainer): diff --git a/malpolon/tests/test_examples.py b/malpolon/tests/test_examples.py index 36e1395d..83e242db 100644 --- a/malpolon/tests/test_examples.py +++ b/malpolon/tests/test_examples.py @@ -142,25 +142,25 @@ {"ref": "Inference, classification_binary, inference_point", "path": Path("examples/inference/sentinel-2a-rgbnir_bioclim/cnn_on_rgbnir_concat.py"), "hydra_args": f"{GPU_ARGS} {INFER_ARGS} run.predict_type=test_point " + BINARY_ARGS},], - # "micro_geolifeclef2022": [ - # # Multiclass classif - # ## Training (raw, transfer learning, inference) - # {"ref": "Train (custom dataset), classification_multiclass, training_raw", - # "path": Path("examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), - # "hydra_args": f"{GPU_ARGS} {TRAIN_ARGS} run.checkpoint_path=null"}, - # {"ref": "Train (custom dataset), classification_multiclass, training_transfer_learning", - # "path": Path("examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), - # "hydra_args": f"{GPU_ARGS} {TRAIN_ARGS} run.checkpoint_path={OUT_DIR}_training_raw/last.ckpt"}, - # {"ref": "Train (custom dataset), classification_multiclass, training_inference", - # "path": Path("examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), - # "hydra_args": f"{GPU_ARGS} run.predict=True run.checkpoint_path={OUT_DIR}_training_raw/last.ckpt"}, - # ## Inference (test_dataset & test_point) - # {"ref": "Inference, classification_multiclass, inference_dataset", - # "path": Path("examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), - # "hydra_args": f"{GPU_ARGS} {INFER_ARGS}"}, - # {"ref": "Inference, classification_multiclass, inference_point", - # "path": Path("examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), - # "hydra_args": f"{GPU_ARGS} {INFER_ARGS} run.predict_type=test_point"},], + "micro_geolifeclef2022": [ + # Multiclass classif + ## Training (raw, transfer learning, inference) + {"ref": "Train (custom dataset), classification_multiclass, training_raw", + "path": Path("examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), + "hydra_args": f"{GPU_ARGS} {TRAIN_ARGS} run.checkpoint_path=null"}, + {"ref": "Train (custom dataset), classification_multiclass, training_transfer_learning", + "path": Path("examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), + "hydra_args": f"{GPU_ARGS} {TRAIN_ARGS} run.checkpoint_path={OUT_DIR}_training_raw/last.ckpt"}, + {"ref": "Train (custom dataset), classification_multiclass, training_inference", + "path": Path("examples/custom_train/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), + "hydra_args": f"{GPU_ARGS} run.predict=True run.checkpoint_path={OUT_DIR}_training_raw/last.ckpt"}, + ## Inference (test_dataset & test_point) + {"ref": "Inference, classification_multiclass, inference_dataset", + "path": Path("examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), + "hydra_args": f"{GPU_ARGS} {INFER_ARGS}"}, + {"ref": "Inference, classification_multiclass, inference_point", + "path": Path("examples/inference/micro_geolifeclef2022/cnn_on_rgb_nir_patches.py"), + "hydra_args": f"{GPU_ARGS} {INFER_ARGS} run.predict_type=test_point"},], } GLC22_EXAMPLE_PATHS = { @@ -213,7 +213,7 @@ "hydra_args": ""}, ], } - +@pytest.mark.skip(reason="Slow and no guarantee of having the data available.") def test_train_inference_examples(): ckpt_path = '' for expe_name, v in EXAMPLE_PATHS.items(): @@ -258,7 +258,7 @@ def test_train_inference_examples(): print(f'{INFO} > {LINK}{path}{RESET}') print(f'\n{INFO}[INFO] Done. {RESET}') -@pytest.mark.skip(reason="Slow and no guarantee of having the data available.") + def test_GLC22_examples(): ckpt_path = '' for expe_name, v in GLC22_EXAMPLE_PATHS.items(): @@ -306,7 +306,7 @@ def test_GLC22_examples(): print(f'{INFO} > {LINK}{path}{RESET}') print(f'\n{INFO}[INFO] Done. {RESET}') - +@pytest.mark.skip(reason="Slow and no guarantee of having the data available.") def test_GLC23_examples(): ckpt_path = '' for expe_name, v in GLC23_EXAMPLE_PATHS.items(): diff --git a/malpolon/tests/test_geolifeclef2022_dataset.py b/malpolon/tests/test_geolifeclef2022_dataset.py index 3eaf70ab..62ae3882 100644 --- a/malpolon/tests/test_geolifeclef2022_dataset.py +++ b/malpolon/tests/test_geolifeclef2022_dataset.py @@ -51,13 +51,13 @@ def test_dataset_load_only_patches(subset): assert len(dataset) == SUBSET_SIZE[subset] result = dataset[0] + assert len(result) == 2 + + data, target = result if subset == "test": - assert len(result) > 2 - data = result + assert target == -1 else: - assert len(result) == 2 - data, target = result assert type(target) == np.int64 assert len(data) == 4 @@ -71,12 +71,13 @@ def test_dataset_load_localisation(subset): result = dataset[0] + assert len(result) == 2 + + data, target = result + if subset == "test": - assert len(result) > 2 - data = result + assert target == -1 else: - assert len(result) == 2 - data, target = result assert type(target) == np.int64 assert len(data) == 5 @@ -95,12 +96,13 @@ def test_dataset_load_one_raster(subset): result = dataset[0] + assert len(result) == 2 + + data, target = result + if subset == "test": - assert len(result) > 2 - data = result + assert target == -1 else: - assert len(result) == 2 - data, target = result assert type(target) == np.int64 assert len(data) == 5