mlc_bgc_base.json

{
    "type": "train_test_log_to_wandb",
    "dataset_reader": {
        "type": "bgc",
        "tokenizer": {
            "type": "pretrained_transformer",
            "model_name": "bert-base-uncased",
            "max_length": 512
        },
        "token_indexers": {
            "x": {
                "type": "pretrained_transformer",
                "model_name": "bert-base-uncased"
            }
        }
    },
    "data_loader": {
        "batch_sampler": {
            "type": "bucket",
            "batch_size": 2,
            "sorting_keys": [
                "x"
            ]
        }
    },
    "validation_dataset_reader": {
        "type": "bgc",
        "tokenizer": {
            "type": "pretrained_transformer",
            "model_name": "bert-base-uncased",
            "max_length": 512
        },
        "token_indexers": {
            "x": {
                "type": "pretrained_transformer",
                "model_name": "bert-base-uncased"
            }
        }
    },
    "validation_data_loader": {
        "batch_sampler": {
            "type": "bucket",
            "batch_size": 16,
            "sorting_keys": [
                "x"
            ]
        }
    },
    "train_data_path": "./data/bgc/train-fold-0.jsonl",
    "validation_data_path": "./bgc/train-fold-0.jsonl",
    "test_data_path": "./bgc/train-fold-0.jsonl",
    "model": {
        "type": "multi-label-classification-with-infnet",
        "inference_module": {
            "type": "multi-label-basic",
            "log_key": "inference_module",
            "loss_fn": {
                "type": "combination-loss",
                "log_key": "loss",
                "reduction": "mean",
                "loss_weights": [
                    3,
                    1
                ],
                "constituent_losses": [
                    {
                        "type": "multi-label-score-loss",
                        "log_key": "neg_nce_score",
                        "reduction": "none",
                        "normalize_y": true
                    },
                    {
                        "type": "multi-label-bce",
                        "log_key": "bce",
                        "reduction": "none"
                    }
                ]
            }
        },
        "initializer": {
            "regexes": [
                [
                    ".*feedforward._linear_layers.*weight",
                    {
                        "nonlinearity": "relu",
                        "type": "kaiming_uniform"
                    }
                ],
                [
                    ".*feedforward._linear_layers.*bias",
                    {
                        "type": "zero"
                    }
                ]
            ]
        },
        "label_smoothing": {
            "alpha": 0.1,
            "use_ls": false
        },
        "loss_fn": {
            "type": "multi-label-nce-ranking-with-discrete-sampling",
            "log_key": "nce",
            "num_samples": 200,
            "sign": "-",
            "loss_scaling": {
                "use_scaling": false
            }
        },
        "oracle_value_function": {
            "type": "per-instance-f1",
            "differentiable": false
        },
        "sampler": {
            "type": "appending-container",
            "constituent_samplers": [],
            "log_key": "sampler"
        },
        "score_nn": {
            "type": "multi-label-classification",
            "global_score": {
                "type": "multi-label-feedforward",
                "feedforward": {
                    "activations": "softplus",
                    "hidden_dims": 300,
                    "input_dim": 146,
                    "num_layers": 1
                }
            },
            "task_nn": {
                "type": "multi-label-text-classification",
                "feature_network": {
                    "feedforward": {
                        "activations": [
                            "softplus",
                            "softplus"
                        ],
                        "dropout": [
                            0.1,
                            0
                        ],
                        "hidden_dims": [
                            1536,
                            768
                        ],
                        "input_dim": 768,
                        "num_layers": 2
                    },
                    "final_dropout": 0,
                    "seq2vec_encoder": {
                        "type": "bert_pooler",
                        "pretrained_model": "bert-base-uncased"
                    },
                    "text_field_embedder": {
                        "token_embedders": {
                            "x": {
                                "type": "pretrained_transformer_with_adapter",
                                "model_name": "bert-base-uncased"
                            }
                        }
                    }
                },
                "label_embeddings": {
                    "embedding_dim": 768,
                    "vocab_namespace": "labels"
                }
            }
        },
        "task_nn": {
            "type": "multi-label-text-classification",
            "feature_network": {
                "feedforward": {
                    "activations": [
                        "softplus",
                        "softplus"
                    ],
                    "dropout": [
                        0.1,
                        0
                    ],
                    "hidden_dims": [
                        1536,
                        768
                    ],
                    "input_dim": 768,
                    "num_layers": 2
                },
                "final_dropout": 0,
                "seq2vec_encoder": {
                    "type": "bert_pooler",
                    "pretrained_model": "bert-base-uncased"
                },
                "text_field_embedder": {
                    "token_embedders": {
                        "x": {
                            "type": "pretrained_transformer_with_adapter",
                            "model_name": "bert-base-uncased"
                        }
                    }
                }
            },
            "label_embeddings": {
                "embedding_dim": 768,
                "vocab_namespace": "labels"
            }
        },
        "thresholding": {
            "use_th": true,
            "method": "score",
            "score_conf": {
                "cut_type": "discrim",
                "quantile": 0.75,
                "score_name": "score",
                "threshold": -100
            }
        }
    },
    "trainer": {
        "type": "gradient_descent_minimax",
        "callbacks": [
            "track_epoch_callback",
            "slurm",
            {
                "save_model_archive": false,
                "sub_callbacks": [
                    {
                        "priority": 100,
                        "type": "log_best_validation_metric"
                    }
                ],
                "type": "wandb_allennlp",
                "watch_model": false
            },
            "thresholding"
        ],
        "checkpointer": {
            "keep_most_recent_by_count": 1,
            "save_every_num_batches": 20000
        },
        "cuda_device": 0,
        "grad_norm": {
            "score_nn": 1,
            "task_nn": 1
        },
        "inner_mode": "score_nn",
        "learning_rate_schedulers": {
            "task_nn": {
                "type": "reduce_on_plateau",
                "factor": 0.5,
                "mode": "max",
                "patience": 1,
                "verbose": true
            }
        },
        "num_epochs": 100,
        "num_gradient_accumulation_steps":2,  
        "num_steps": {
            "score_nn": 1,
            "task_nn": 1
        },
        "optimizer": {
            "optimizers": {
                "score_nn": {
                    "type": "huggingface_adamw",
                    "lr": 0.0001,
                    "weight_decay": 0.1
                },
                "task_nn": {
                    "type": "huggingface_adamw",
                    "lr": 0.0001,
                    "weight_decay": 0.1
                }
            }
        },
        "patience": 4,
        "pre_training": 0,
        "validation_metric": "+fixed_f1",
        "filter_unlab": {
            "use_fu": false,
            "model_to_run": "./srl_onto_multi_from_scratch"
        }    
    },
    "vocabulary": {
        "type": "from_files",
        "directory": "./data/bgc/vocabulary"
    },
    "evaluate_on_test": true,
    "numpy_seed": 234,
    "pytorch_seed":234,
    "random_seed": 234
}