diff --git a/README.md b/README.md index f99777d5c7..c33b032fd9 100644 --- a/README.md +++ b/README.md @@ -58,5 +58,5 @@ This work has been funded by grants from [Wellcome](https://wellcome.org/), [arc-link]: https://www.ucl.ac.uk/arc [york-link]: https://www.york.ac.uk/ [che-link]: https://www.york.ac.uk/che/ -[ecsea-hc-link]: https://ecsahc.org/ +[ecsa-hc-link]: https://ecsahc.org/ [contributors-link]: https://www.tlomodel.org/contributors.html diff --git a/pyproject.toml b/pyproject.toml index a8622e235d..ce24c3a3cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ dev = [ # Profiling "ansi2html", "psutil", - "pyinstrument>=4.7", + "pyinstrument>=4.3", # Building requirements files "pip-tools", ] diff --git a/requirements/dev.txt b/requirements/dev.txt index e8bbc2e0c9..a6e0468a19 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -164,7 +164,7 @@ psutil==5.9.5 # via tlo (pyproject.toml) pycparser==2.21 # via cffi -pyinstrument==4.7.3 +pyinstrument==4.5.3 # via tlo (pyproject.toml) pyjwt[crypto]==2.8.0 # via diff --git a/resources/ResourceFile_AntenatalCare.xlsx b/resources/ResourceFile_AntenatalCare.xlsx deleted file mode 100644 index 907ee62369..0000000000 --- a/resources/ResourceFile_AntenatalCare.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4da00560fd62d440d90dd3f054d999370a9cb3a83dd5516347ba8edef8275662 -size 19087 diff --git a/resources/ResourceFile_AntenatalCare/parameter_values.csv b/resources/ResourceFile_AntenatalCare/parameter_values.csv new file mode 100644 index 0000000000..546fd29a26 --- /dev/null +++ b/resources/ResourceFile_AntenatalCare/parameter_values.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5cf6bb4312c4e315d004ef769bac35560be3fe423fe648e9ea5549b5e3342d1 +size 1258 diff --git a/resources/ResourceFile_EPI_WHO_estimates.xlsx b/resources/ResourceFile_EPI_WHO_estimates.xlsx deleted file mode 100644 index 21bab15fef..0000000000 --- a/resources/ResourceFile_EPI_WHO_estimates.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cfea099178ece7999341cb1666da3a7ada0f459b3aa29fb0c9ffac8538c749e5 -size 15627 diff --git a/resources/ResourceFile_EPI_WHO_estimates/WHO_estimates.csv b/resources/ResourceFile_EPI_WHO_estimates/WHO_estimates.csv new file mode 100644 index 0000000000..6f0396f7ea --- /dev/null +++ b/resources/ResourceFile_EPI_WHO_estimates/WHO_estimates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9985cf3918d267a9147c844ce60bb2809fe6f2d5f56accfa4ac960ac920e8a91 +size 2762 diff --git a/resources/ResourceFile_EPI_WHO_estimates/parameters.csv b/resources/ResourceFile_EPI_WHO_estimates/parameters.csv new file mode 100644 index 0000000000..724c3564ba --- /dev/null +++ b/resources/ResourceFile_EPI_WHO_estimates/parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca507e9a48d12cea8a2c143f142b9d3d997b852355d6034c607ef03cbeb41f20 +size 76 diff --git a/resources/ResourceFile_EPI_WHO_estimates/vaccine_schedule.csv b/resources/ResourceFile_EPI_WHO_estimates/vaccine_schedule.csv new file mode 100644 index 0000000000..2ced217d5b --- /dev/null +++ b/resources/ResourceFile_EPI_WHO_estimates/vaccine_schedule.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27341d0df15927ea78658668fd24de74d391244213e8875f53fbd07d02ebe7c8 +size 307 diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx deleted file mode 100644 index 00f7b684db..0000000000 --- a/resources/ResourceFile_HIV.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b34a88635b02ee8a465462c8eb67a485d721c9159a5bba1df8e63609b803ebe9 -size 161679 diff --git a/resources/ResourceFile_HIV/DHS_prevalence.csv b/resources/ResourceFile_HIV/DHS_prevalence.csv new file mode 100644 index 0000000000..5e09941567 --- /dev/null +++ b/resources/ResourceFile_HIV/DHS_prevalence.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a15b3cb70827b7ff122fc09b37c6d922c983b7e23244d57758c60c504a349c4 +size 638 diff --git a/resources/ResourceFile_HIV/LHC_samples.csv b/resources/ResourceFile_HIV/LHC_samples.csv new file mode 100644 index 0000000000..f43a36f081 --- /dev/null +++ b/resources/ResourceFile_HIV/LHC_samples.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e625ec870aa0ac80c971f26a36bd99054e95d44b57c30dddbecf39401e72dc12 +size 1230 diff --git a/resources/ResourceFile_HIV/MPHIA_incidence2020.csv b/resources/ResourceFile_HIV/MPHIA_incidence2020.csv new file mode 100644 index 0000000000..fc31be8fe6 --- /dev/null +++ b/resources/ResourceFile_HIV/MPHIA_incidence2020.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:319b562d8c2c81ebdde4ddfe2624087b3d03ac994c95b050f22c7e82b2a06f80 +size 720 diff --git a/resources/ResourceFile_HIV/MPHIA_prevalence_art2020.csv b/resources/ResourceFile_HIV/MPHIA_prevalence_art2020.csv new file mode 100644 index 0000000000..c79ed4ceb3 --- /dev/null +++ b/resources/ResourceFile_HIV/MPHIA_prevalence_art2020.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9cc554f255d8e79c082ef88cb1ba21c0eb57132e1ffc24b910ccfd9d6451878 +size 1423 diff --git a/resources/ResourceFile_HIV/MoH_CPT_IPT2020.csv b/resources/ResourceFile_HIV/MoH_CPT_IPT2020.csv new file mode 100644 index 0000000000..aba73e9023 --- /dev/null +++ b/resources/ResourceFile_HIV/MoH_CPT_IPT2020.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f984c6cb04d89b7489a6848b04b0e921322ac816d7c87178438b1521e97432fa +size 20001 diff --git a/resources/ResourceFile_HIV/MoH_number_art.csv b/resources/ResourceFile_HIV/MoH_number_art.csv new file mode 100644 index 0000000000..952d2524b8 --- /dev/null +++ b/resources/ResourceFile_HIV/MoH_number_art.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ad0d95f6269e25787e5f39028adccb84d9b443059aed82d0569ce3965e47ca +size 5311 diff --git a/resources/ResourceFile_HIV/MoH_numbers_tests.csv b/resources/ResourceFile_HIV/MoH_numbers_tests.csv new file mode 100644 index 0000000000..fe2142f4b4 --- /dev/null +++ b/resources/ResourceFile_HIV/MoH_numbers_tests.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e626e9de6d36137b022388d312027c3bc13d03ae4940592a1c16cf91db032fd4 +size 1910 diff --git a/resources/ResourceFile_HIV/art_coverage.csv b/resources/ResourceFile_HIV/art_coverage.csv new file mode 100644 index 0000000000..9c77e9cb0c --- /dev/null +++ b/resources/ResourceFile_HIV/art_coverage.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c858c19795eddc9d73a2186314f56d52ab6a65fbe0a548c749cb3ab1f4cb02f7 +size 48299 diff --git a/resources/ResourceFile_HIV/calibration_from_aids_info.csv b/resources/ResourceFile_HIV/calibration_from_aids_info.csv new file mode 100644 index 0000000000..763edea949 --- /dev/null +++ b/resources/ResourceFile_HIV/calibration_from_aids_info.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fac0b9ec3659e1316e444ad78742ae6ead4f069ad416fb319474ae539144bc20 +size 3770 diff --git a/resources/ResourceFile_HIV/children0_14_prev_AIDSinfo.csv b/resources/ResourceFile_HIV/children0_14_prev_AIDSinfo.csv new file mode 100644 index 0000000000..f2dbfdad79 --- /dev/null +++ b/resources/ResourceFile_HIV/children0_14_prev_AIDSinfo.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abfb9c5a639bce18bbc7c4f6030716862374df18e78c831fc621cbad878c3d72 +size 2231 diff --git a/resources/ResourceFile_HIV/hiv_prevalence.csv b/resources/ResourceFile_HIV/hiv_prevalence.csv new file mode 100644 index 0000000000..ed2ca50de7 --- /dev/null +++ b/resources/ResourceFile_HIV/hiv_prevalence.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efa5457968f06835f64af43710ae4c34f919f0a8ac1d491201bfb728a55a25f7 +size 12643 diff --git a/resources/ResourceFile_HIV/parameters.csv b/resources/ResourceFile_HIV/parameters.csv new file mode 100644 index 0000000000..835dfa942a --- /dev/null +++ b/resources/ResourceFile_HIV/parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c662f8cd5d5242cc9d73abce71d781adc514271bd73bec7587465fccc8ba2a10 +size 3360 diff --git a/resources/ResourceFile_HIV/scaleup_parameters.csv b/resources/ResourceFile_HIV/scaleup_parameters.csv new file mode 100644 index 0000000000..1e3bcd9fa2 --- /dev/null +++ b/resources/ResourceFile_HIV/scaleup_parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31392b914573011c37159f09947d088e8f316cf6dc55141547c8a0a538da8f18 +size 464 diff --git a/resources/ResourceFile_HIV/spectrum_treatment_cascade.csv b/resources/ResourceFile_HIV/spectrum_treatment_cascade.csv new file mode 100644 index 0000000000..94c6834d03 --- /dev/null +++ b/resources/ResourceFile_HIV/spectrum_treatment_cascade.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917e970a76f4281c3317ee347570a06c26a6ac6394763d5b5bc5e1833a7a0f40 +size 2168 diff --git a/resources/ResourceFile_HIV/time_since_infection_at_baselin.csv b/resources/ResourceFile_HIV/time_since_infection_at_baselin.csv new file mode 100644 index 0000000000..8303b5097e --- /dev/null +++ b/resources/ResourceFile_HIV/time_since_infection_at_baselin.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:379e28ba185789a361c993b1771f4889864f8d2e8ca5c424b60215ef7d8a0bf1 +size 1951 diff --git a/resources/ResourceFile_HIV/unaids_infections_art2021.csv b/resources/ResourceFile_HIV/unaids_infections_art2021.csv new file mode 100644 index 0000000000..2e5f90a46f --- /dev/null +++ b/resources/ResourceFile_HIV/unaids_infections_art2021.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05e0c19f7720120287e167898658d8464e0f71d868545ec547c6faaf775d906c +size 3695 diff --git a/resources/ResourceFile_HIV/unaids_mortality_dalys2021.csv b/resources/ResourceFile_HIV/unaids_mortality_dalys2021.csv new file mode 100644 index 0000000000..f3f3b17d1f --- /dev/null +++ b/resources/ResourceFile_HIV/unaids_mortality_dalys2021.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14603337879d9a927af25d3f7af751e90c97256ecd816c3713d67d06f35fe535 +size 1053 diff --git a/resources/ResourceFile_HIV/unaids_pmtct2021.csv b/resources/ResourceFile_HIV/unaids_pmtct2021.csv new file mode 100644 index 0000000000..f768747cda --- /dev/null +++ b/resources/ResourceFile_HIV/unaids_pmtct2021.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e56e37657147beab6ac90ee40274bc1b03f2eb45bc29c9ebb3bfd11122b01c9 +size 2488 diff --git a/resources/ResourceFile_HIV/unaids_program_perf.csv b/resources/ResourceFile_HIV/unaids_program_perf.csv new file mode 100644 index 0000000000..f3a1f0a566 --- /dev/null +++ b/resources/ResourceFile_HIV/unaids_program_perf.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6701b4eb8081951f3402853c40f2c4a0aa03d1a276f014758a4f6981a999e29d +size 528 diff --git a/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx b/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx deleted file mode 100644 index 9d61093db8..0000000000 --- a/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ebd3aff1d77254f7cd7bb1dc019e2e6a86700dcd6009cecd11a70da4b85a3c93 -size 31777 diff --git a/resources/ResourceFile_LabourSkilledBirthAttendance/parameter_values.csv b/resources/ResourceFile_LabourSkilledBirthAttendance/parameter_values.csv new file mode 100644 index 0000000000..e7d18b8251 --- /dev/null +++ b/resources/ResourceFile_LabourSkilledBirthAttendance/parameter_values.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be621298177fff272051d8b519dd890f812c427603f24e70a66386e528157047 +size 11243 diff --git a/resources/ResourceFile_Measles.xlsx b/resources/ResourceFile_Measles.xlsx deleted file mode 100644 index d0c96cc876..0000000000 --- a/resources/ResourceFile_Measles.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:947273d8b5a1cdc5c187f01f5e23a6f9ac89d188c550550cd1e7bb6850f744dd -size 19416 diff --git a/resources/ResourceFile_Measles/beta.csv b/resources/ResourceFile_Measles/beta.csv new file mode 100644 index 0000000000..a7ba7e5efa --- /dev/null +++ b/resources/ResourceFile_Measles/beta.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:427be2f0f44c3da740657ee64a0a10f4bccca798ca655712cc14f7eba08575b0 +size 125 diff --git a/resources/ResourceFile_Measles/cfr.csv b/resources/ResourceFile_Measles/cfr.csv new file mode 100644 index 0000000000..0d84e74583 --- /dev/null +++ b/resources/ResourceFile_Measles/cfr.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af172fa61482e208e10a8c247ce5ec79faf60e1f4fb83af77bfd8347fc1dc27 +size 643 diff --git a/resources/ResourceFile_Measles/parameters.csv b/resources/ResourceFile_Measles/parameters.csv new file mode 100644 index 0000000000..f1962afba7 --- /dev/null +++ b/resources/ResourceFile_Measles/parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203376e1d0b74fe76c66ac2ff66a4db4efb24d46635db05afcb04a2fdd141205 +size 249 diff --git a/resources/ResourceFile_Measles/symptoms.csv b/resources/ResourceFile_Measles/symptoms.csv new file mode 100644 index 0000000000..c5209ee073 --- /dev/null +++ b/resources/ResourceFile_Measles/symptoms.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d967b5051c192d15eb202efad5cd6897f980b440d174f4e6b4a21d4a78e28361 +size 4466 diff --git a/resources/ResourceFile_NewbornOutcomes.xlsx b/resources/ResourceFile_NewbornOutcomes.xlsx deleted file mode 100644 index e915c6814a..0000000000 --- a/resources/ResourceFile_NewbornOutcomes.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f81564a2128891c5e97da78a6d8dd63819016857d5ef9817993874002cb2a658 -size 15349 diff --git a/resources/ResourceFile_NewbornOutcomes/parameter_values.csv b/resources/ResourceFile_NewbornOutcomes/parameter_values.csv new file mode 100644 index 0000000000..85ae021aa4 --- /dev/null +++ b/resources/ResourceFile_NewbornOutcomes/parameter_values.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f805646e16fe9912a68da5a10b901086e8a531f4802b04a7ec2b1216eedfac4e +size 3737 diff --git a/resources/ResourceFile_PostnatalSupervisor.xlsx b/resources/ResourceFile_PostnatalSupervisor.xlsx deleted file mode 100644 index 4c5e85cfc7..0000000000 --- a/resources/ResourceFile_PostnatalSupervisor.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:28757e4b9e6ffc0f0d12e8a76d4b4ac6704ce5bcbe781ec0c0d99bdce852cf04 -size 22888 diff --git a/resources/ResourceFile_PostnatalSupervisor/parameter_values.csv b/resources/ResourceFile_PostnatalSupervisor/parameter_values.csv new file mode 100644 index 0000000000..ac0f082139 --- /dev/null +++ b/resources/ResourceFile_PostnatalSupervisor/parameter_values.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dac89c4cc638a6f65cc21effb521380e3d6ca06600af737244e41ef0f71b4a9 +size 3573 diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx deleted file mode 100644 index 3ee3406d7a..0000000000 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fbd0974016b374a3286e379ef3cb1f297307ef410880c3f902e9439194a6d37d -size 22225 diff --git a/resources/ResourceFile_PregnancySupervisor/parameter_values.csv b/resources/ResourceFile_PregnancySupervisor/parameter_values.csv new file mode 100644 index 0000000000..05cfe2f3e2 --- /dev/null +++ b/resources/ResourceFile_PregnancySupervisor/parameter_values.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:702d26f19582a9fc603265da9f0b61f89fede8aa3582ab8cee551cf4109a49b1 +size 12966 diff --git a/resources/ResourceFile_Schisto.xlsx b/resources/ResourceFile_Schisto.xlsx deleted file mode 100644 index af2eaed4a7..0000000000 --- a/resources/ResourceFile_Schisto.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:245f6bb53087f86ae8df92b4bffdc0611d892333ff33e7c4d94bd58e6e39f592 -size 34479 diff --git a/resources/ResourceFile_Schisto/DALYs.csv b/resources/ResourceFile_Schisto/DALYs.csv new file mode 100644 index 0000000000..8dbfe6482e --- /dev/null +++ b/resources/ResourceFile_Schisto/DALYs.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a8a184620e63cbf7587761edd70a18a56aa8d67ab1021cff18b8f9105afb21 +size 2027 diff --git a/resources/ResourceFile_Schisto/District_Params_haematobium.csv b/resources/ResourceFile_Schisto/District_Params_haematobium.csv new file mode 100644 index 0000000000..a46ac624bf --- /dev/null +++ b/resources/ResourceFile_Schisto/District_Params_haematobium.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78b3a5c327b89ca43e2f44524b0c1ce89bb40848fc83c39bf24f8e7de3b4f7ff +size 2712 diff --git a/resources/ResourceFile_Schisto/District_Params_mansoni.csv b/resources/ResourceFile_Schisto/District_Params_mansoni.csv new file mode 100644 index 0000000000..ec471e35b0 --- /dev/null +++ b/resources/ResourceFile_Schisto/District_Params_mansoni.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd3549c918ee378a5b74361d1c7acd406eddb70f2143b30fe001729b47570a8 +size 2372 diff --git a/resources/ResourceFile_Schisto/MDA_historical_Coverage.csv b/resources/ResourceFile_Schisto/MDA_historical_Coverage.csv new file mode 100644 index 0000000000..35a50334fd --- /dev/null +++ b/resources/ResourceFile_Schisto/MDA_historical_Coverage.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54cf3ab4d0c1c0f5912cc82aad8f7c4cd1adc168c715864c89115f7a3a3c6b6a +size 7335 diff --git a/resources/ResourceFile_Schisto/MDA_prognosed_Coverage.csv b/resources/ResourceFile_Schisto/MDA_prognosed_Coverage.csv new file mode 100644 index 0000000000..6af146f6b0 --- /dev/null +++ b/resources/ResourceFile_Schisto/MDA_prognosed_Coverage.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec4a908a3f6a2fa3098e097dde706edd4d2a66a864a22636fd701dc688eb582 +size 1104 diff --git a/resources/ResourceFile_Schisto/Parameters.csv b/resources/ResourceFile_Schisto/Parameters.csv new file mode 100644 index 0000000000..68e7a111a6 --- /dev/null +++ b/resources/ResourceFile_Schisto/Parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c1702528ef59b66dfe7f884560c55b5a59ecac1ff8ecbd18cff4f9de7cc769b +size 1300 diff --git a/resources/ResourceFile_Schisto/Symptoms.csv b/resources/ResourceFile_Schisto/Symptoms.csv new file mode 100644 index 0000000000..f994878002 --- /dev/null +++ b/resources/ResourceFile_Schisto/Symptoms.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57bf11a659f0e11946cec71d803224ba6aee302a162936945c625376432a5327 +size 676 diff --git a/resources/ResourceFile_TB.xlsx b/resources/ResourceFile_TB.xlsx deleted file mode 100644 index 2b612ad6ec..0000000000 --- a/resources/ResourceFile_TB.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:93d7bf76c8bece548e08e3f0cb6e9e28a09ca2b5760a408399bf9641f7ed2001 -size 56523 diff --git a/resources/ResourceFile_TB/IPTdistricts.csv b/resources/ResourceFile_TB/IPTdistricts.csv new file mode 100644 index 0000000000..1c3d68794b --- /dev/null +++ b/resources/ResourceFile_TB/IPTdistricts.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e560877adfdd0a74f5cd25afc0d879f00b036c9d731d22891ed31e2cd6912a +size 211 diff --git a/resources/ResourceFile_TB/NTP2019.csv b/resources/ResourceFile_TB/NTP2019.csv new file mode 100644 index 0000000000..2c7f1331bf --- /dev/null +++ b/resources/ResourceFile_TB/NTP2019.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fa3feadb3c88be3e779811bff53eb2ad8c7feffc0daadf0440f2bc424a89b51 +size 725 diff --git a/resources/ResourceFile_TB/WHO_activeTB2023.csv b/resources/ResourceFile_TB/WHO_activeTB2023.csv new file mode 100644 index 0000000000..79b5d4693c --- /dev/null +++ b/resources/ResourceFile_TB/WHO_activeTB2023.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4cf542d0d7ee6c3a926c4fbfd18cf6ecf10685a257a2389263e9c5b905fe792 +size 6180 diff --git a/resources/ResourceFile_TB/WHO_latentTB2017.csv b/resources/ResourceFile_TB/WHO_latentTB2017.csv new file mode 100644 index 0000000000..08c355063c --- /dev/null +++ b/resources/ResourceFile_TB/WHO_latentTB2017.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fada3235151a7d9d5ac3c89ad1a46e792d283a64677601cd941c6ccee8caf527 +size 1871 diff --git a/resources/ResourceFile_TB/WHO_mdrTB2017.csv b/resources/ResourceFile_TB/WHO_mdrTB2017.csv new file mode 100644 index 0000000000..3dd3ce23d0 --- /dev/null +++ b/resources/ResourceFile_TB/WHO_mdrTB2017.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79a899213defc36589ab63f2063a8a7582a2ccc4726ebddbc0c8c2a39fccda5d +size 2436 diff --git a/resources/ResourceFile_TB/WHO_tx_success_rates2021.csv b/resources/ResourceFile_TB/WHO_tx_success_rates2021.csv new file mode 100644 index 0000000000..4e3e01a9cf --- /dev/null +++ b/resources/ResourceFile_TB/WHO_tx_success_rates2021.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5345e87d02441d2ef1e26ad86ef6599f4a3c0ed7ba0b6ee54901a7c0bab6c90d +size 3508 diff --git a/resources/ResourceFile_TB/additional_params.csv b/resources/ResourceFile_TB/additional_params.csv new file mode 100644 index 0000000000..6ae4a289e7 --- /dev/null +++ b/resources/ResourceFile_TB/additional_params.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97e8dc97188f7ffd640b7a95219d88c303c7a2715d6c83080d4810b300f62577 +size 179 diff --git a/resources/ResourceFile_TB/all_districts.csv b/resources/ResourceFile_TB/all_districts.csv new file mode 100644 index 0000000000..41b350e2f7 --- /dev/null +++ b/resources/ResourceFile_TB/all_districts.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae7b7414240c5290bb38582751f7a9d22e85b1f3cffc40e03a6959624e9abf12 +size 317 diff --git a/resources/ResourceFile_TB/calibrated_transmission_rates.csv b/resources/ResourceFile_TB/calibrated_transmission_rates.csv new file mode 100644 index 0000000000..9526ba9522 --- /dev/null +++ b/resources/ResourceFile_TB/calibrated_transmission_rates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036d2aed1d9abbbfb5221c3f6162dff4662ef51d7faed85162d1ba98e6050330 +size 169 diff --git a/resources/ResourceFile_TB/cases2010district.csv b/resources/ResourceFile_TB/cases2010district.csv new file mode 100644 index 0000000000..7295a73d8c --- /dev/null +++ b/resources/ResourceFile_TB/cases2010district.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10a0000be36a5932aeaeb72554f613932447b9cef5c0c4e918881b9aeae4a86a +size 2916 diff --git a/resources/ResourceFile_TB/details_rates.csv b/resources/ResourceFile_TB/details_rates.csv new file mode 100644 index 0000000000..908eb084b0 --- /dev/null +++ b/resources/ResourceFile_TB/details_rates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2918bf8b9c4d62601a9e9dde52cf60de3cc8caac9b2a68965f5f470914ce57fd +size 1422 diff --git a/resources/ResourceFile_TB/followup.csv b/resources/ResourceFile_TB/followup.csv new file mode 100644 index 0000000000..788d30782f --- /dev/null +++ b/resources/ResourceFile_TB/followup.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49fc006e40d3a0f52e093f5fc304f9c23f3a886b95a267890a2f8f04d3ff1e35 +size 522 diff --git a/resources/ResourceFile_TB/ipt_coverage.csv b/resources/ResourceFile_TB/ipt_coverage.csv new file mode 100644 index 0000000000..38d14ca71f --- /dev/null +++ b/resources/ResourceFile_TB/ipt_coverage.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12e36bbb0575034962240002f46aa14f84bc430889c46f9092eaaa51870c1f5a +size 1100 diff --git a/resources/ResourceFile_TB/latent_TB2014_summary.csv b/resources/ResourceFile_TB/latent_TB2014_summary.csv new file mode 100644 index 0000000000..32e8f05bb4 --- /dev/null +++ b/resources/ResourceFile_TB/latent_TB2014_summary.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:997d879b2d67658acbaa64d27c765276770d5034d350da51ae323d8166f05f68 +size 397 diff --git a/resources/ResourceFile_TB/parameters.csv b/resources/ResourceFile_TB/parameters.csv new file mode 100644 index 0000000000..82311e8dc7 --- /dev/null +++ b/resources/ResourceFile_TB/parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1009c49ab55ec72cbbf11dc8cf6df5703ad0cf07bd225f53b5903da6bdfdc07 +size 2037 diff --git a/resources/ResourceFile_TB/pulm_tb.csv b/resources/ResourceFile_TB/pulm_tb.csv new file mode 100644 index 0000000000..908a146b4c --- /dev/null +++ b/resources/ResourceFile_TB/pulm_tb.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b62fb56ca5ad495a530853707ab3af0922e807edda5cbca8917354ca7988f198 +size 2742 diff --git a/resources/ResourceFile_TB/scaleup_parameters.csv b/resources/ResourceFile_TB/scaleup_parameters.csv new file mode 100644 index 0000000000..9cb3d2ea0c --- /dev/null +++ b/resources/ResourceFile_TB/scaleup_parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f5f29cc8816eaafbb60add9d69b4d0e86f93082e76c42fc6b29b85e454bbd3 +size 340 diff --git a/resources/ResourceFile_TB/testing_rates.csv b/resources/ResourceFile_TB/testing_rates.csv new file mode 100644 index 0000000000..ae3f43586c --- /dev/null +++ b/resources/ResourceFile_TB/testing_rates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee52c1e343b0e634c05c2eb281f925174fe951b356856acaede2435e8f543d94 +size 1016 diff --git a/resources/contraception/ResourceFile_Contraception.xlsx b/resources/contraception/ResourceFile_Contraception.xlsx deleted file mode 100644 index 660f29830b..0000000000 --- a/resources/contraception/ResourceFile_Contraception.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:86fc4eae9bd0525d08994b51bc25de83bffb7f142b43fa8816cd44ed842878bf -size 975198 diff --git a/resources/contraception/ResourceFile_Contraception/Discontinuation_ByAge.csv b/resources/contraception/ResourceFile_Contraception/Discontinuation_ByAge.csv new file mode 100644 index 0000000000..c4b70b9182 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Discontinuation_ByAge.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3b8d2085d10f1680cf7d521ba415777b194ef26a964c14bca473cbdd76c7982 +size 770 diff --git a/resources/contraception/ResourceFile_Contraception/Discontinuation_ByMethod.csv b/resources/contraception/ResourceFile_Contraception/Discontinuation_ByMethod.csv new file mode 100644 index 0000000000..0736824d4b --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Discontinuation_ByMethod.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ab5f87adb1c35d4ff2fa0e6226a32840ef4d3d4a8d130404cf74e9f2a13528a +size 296 diff --git a/resources/contraception/ResourceFile_Contraception/Failure_ByMethod.csv b/resources/contraception/ResourceFile_Contraception/Failure_ByMethod.csv new file mode 100644 index 0000000000..399ed60f1d --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Failure_ByMethod.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d54deeb3a78189d18f1e18e20132aa4e965d5e780ab6299e4ef5434627449dfe +size 259 diff --git a/resources/contraception/ResourceFile_Contraception/Initiation_AfterBirth.csv b/resources/contraception/ResourceFile_Contraception/Initiation_AfterBirth.csv new file mode 100644 index 0000000000..1aa855046b --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Initiation_AfterBirth.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec00bfe6c52595dbac43cc05bfa00771408dd3ceddc2a0c843ffe232453fd04 +size 270 diff --git a/resources/contraception/ResourceFile_Contraception/Initiation_ByAge.csv b/resources/contraception/ResourceFile_Contraception/Initiation_ByAge.csv new file mode 100644 index 0000000000..e1bf896912 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Initiation_ByAge.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb59b96ca917f218d48b349790092ea04d612c3545669c219ae25a65930a38d1 +size 810 diff --git a/resources/contraception/ResourceFile_Contraception/Initiation_ByMethod.csv b/resources/contraception/ResourceFile_Contraception/Initiation_ByMethod.csv new file mode 100644 index 0000000000..658921c98f --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Initiation_ByMethod.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62709e4159aff94254a843ad726cfb0b5d655bce4c7d209cff03fed2adde3534 +size 268 diff --git a/resources/contraception/ResourceFile_Contraception/Interventions_PPFP.csv b/resources/contraception/ResourceFile_Contraception/Interventions_PPFP.csv new file mode 100644 index 0000000000..64cf469ff6 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Interventions_PPFP.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66624dbc15b3f8299e218c97e1f9ebd2e79fc382e25b55f214cdfbe50cd85ee3 +size 150 diff --git a/resources/contraception/ResourceFile_Contraception/Interventions_Pop.csv b/resources/contraception/ResourceFile_Contraception/Interventions_Pop.csv new file mode 100644 index 0000000000..64cf469ff6 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Interventions_Pop.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66624dbc15b3f8299e218c97e1f9ebd2e79fc382e25b55f214cdfbe50cd85ee3 +size 150 diff --git a/resources/contraception/ResourceFile_Contraception/Method_Use_In_2010.csv b/resources/contraception/ResourceFile_Contraception/Method_Use_In_2010.csv new file mode 100644 index 0000000000..84880f7550 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Method_Use_In_2010.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fafc2deddb6c9f1a341f0dc45097aaf0770cb09b49ec26ffa7133f35e748044 +size 5983 diff --git a/resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_HIVeffect.csv b/resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_HIVeffect.csv new file mode 100644 index 0000000000..bd377c879b --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_HIVeffect.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6848831b52ab636cf78cd6c3ddc999ad01a46d9d5471987d0e406c479f75ef0 +size 263 diff --git a/resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_In_2010.csv b/resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_In_2010.csv new file mode 100644 index 0000000000..c05c8a2649 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_In_2010.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57d060bdd1a4ad87630853291de818144eeb15c5d2a5ba54a4d2507b31f879ff +size 509 diff --git a/resources/contraception/ResourceFile_Contraception/Prob_Switch_From.csv b/resources/contraception/ResourceFile_Contraception/Prob_Switch_From.csv new file mode 100644 index 0000000000..072f28484a --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Prob_Switch_From.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9340fdca9143fd824c0074986c0fc4c7620120517ecbf67bce75b32d4fa240d +size 295 diff --git a/resources/contraception/ResourceFile_Contraception/Prob_Switch_From_And_To.csv b/resources/contraception/ResourceFile_Contraception/Prob_Switch_From_And_To.csv new file mode 100644 index 0000000000..6a3770c232 --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/Prob_Switch_From_And_To.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3784438c4ccc28e3886cc8244880364a3b73581598e2b73e515f2309fd14a281 +size 1219 diff --git a/resources/contraception/ResourceFile_Contraception/simplified_labour_parameters.csv b/resources/contraception/ResourceFile_Contraception/simplified_labour_parameters.csv new file mode 100644 index 0000000000..38c6f58d9e --- /dev/null +++ b/resources/contraception/ResourceFile_Contraception/simplified_labour_parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c1d00707d77dcba8ec87532a9ac3f122cb4bbb2c6cd473d57c0157259b16e3 +size 95 diff --git a/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv b/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv index 25249531b2..19ab070507 100644 --- a/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv +++ b/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c358a643e4def0e574b75f89f83d77f9c3366f668422e005150f4d69ebe8d7a7 -size 6169152 +oid sha256:8bf105eb266c173feaef4068d100af4ea51f2542c3cac9505a704abade360820 +size 6202574 diff --git a/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv b/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv index 7ab675ecba..703faf4549 100644 --- a/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv +++ b/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b5b0f417681cbdd2489e2f9c6634b2825c32beb9637dc045b56e308c910a102c -size 90569 +oid sha256:793e3b2a94949fdf025bb5297de40a0092c41ad61f6fa0a4de4898de5cfdf2f3 +size 90677 diff --git a/resources/healthsystem/human_resources/actual/ResourceFile_Daily_Capabilities.csv b/resources/healthsystem/human_resources/actual/ResourceFile_Daily_Capabilities.csv index 79095353a1..7ccce7a281 100644 --- a/resources/healthsystem/human_resources/actual/ResourceFile_Daily_Capabilities.csv +++ b/resources/healthsystem/human_resources/actual/ResourceFile_Daily_Capabilities.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:29ad972989450daf2cf6d339bae4cc4396b37afe6a9fb6675b9f34bf90519307 -size 103717 +oid sha256:ac9106f76300f262d9d0889b7df05ad450b57cfc65db85159aa49239ec4765fd +size 103724 diff --git a/resources/healthsystem/human_resources/definitions/ResourceFile_Appt_Time_Table.csv b/resources/healthsystem/human_resources/definitions/ResourceFile_Appt_Time_Table.csv index 9527b32238..408fc73c9c 100644 --- a/resources/healthsystem/human_resources/definitions/ResourceFile_Appt_Time_Table.csv +++ b/resources/healthsystem/human_resources/definitions/ResourceFile_Appt_Time_Table.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a49b3fb503c92ddf96f9df375bea5b7b091f324d3488b72aeda4219b04eb1a07 -size 10985 +oid sha256:aa45a22a3a93e152bbe04e95f971087fe709ca1326d41dca43e50afab11aabe8 +size 10957 diff --git a/resources/healthsystem/human_resources/funded/ResourceFile_Daily_Capabilities.csv b/resources/healthsystem/human_resources/funded/ResourceFile_Daily_Capabilities.csv index f48cd26edb..9713c93363 100644 --- a/resources/healthsystem/human_resources/funded/ResourceFile_Daily_Capabilities.csv +++ b/resources/healthsystem/human_resources/funded/ResourceFile_Daily_Capabilities.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:26949115de16e7002d81891aba92df1b2c5f406e57c6bae91ecdb248f15db57c -size 102159 +oid sha256:4a9aa79441c1adef3b57f230b6901dc54830293fb69db252dce31e1e561a4fae +size 102157 diff --git a/resources/healthsystem/human_resources/funded_plus/ResourceFile_Daily_Capabilities.csv b/resources/healthsystem/human_resources/funded_plus/ResourceFile_Daily_Capabilities.csv index 5d88a6da69..237fad58e8 100644 --- a/resources/healthsystem/human_resources/funded_plus/ResourceFile_Daily_Capabilities.csv +++ b/resources/healthsystem/human_resources/funded_plus/ResourceFile_Daily_Capabilities.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01d1eb90260e0f4daa4e0359ab8af35bae2c5b00e2976d959f5986ccdd13fd9c -size 71433 +oid sha256:98551ae882f43e795d3d0c68bedb908c2dd847ec57365aab66e3f59c8f3e15e4 +size 103089 diff --git a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx index a633e6fc92..d63e93403c 100644 --- a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx +++ b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d2d74390498e497ee0bf68773327868f6b199c1c9569337b173fa330c0f2f926 -size 24593 +oid sha256:a8889a273541bb315617df8260b59fc7c6a8913b531c38468c21df0da150a547 +size 18431 diff --git a/resources/healthsystem/priority_policies/ResourceFile_PriorityRanking_ALLPOLICIES.xlsx b/resources/healthsystem/priority_policies/ResourceFile_PriorityRanking_ALLPOLICIES.xlsx index d9dbac2e99..1748d3f5e9 100644 --- a/resources/healthsystem/priority_policies/ResourceFile_PriorityRanking_ALLPOLICIES.xlsx +++ b/resources/healthsystem/priority_policies/ResourceFile_PriorityRanking_ALLPOLICIES.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:734d46d83dccf15bf38ee171a487664f01035da6cf68660d4af62097a6160fb6 -size 42716 +oid sha256:59a7b6737589f04bc5fc80c3ca3c60f6dae2e1cf95e41ebefa995294298fbc84 +size 42958 diff --git a/resources/malaria/ResourceFile_malaria.xlsx b/resources/malaria/ResourceFile_malaria.xlsx deleted file mode 100644 index 7537f3ace9..0000000000 --- a/resources/malaria/ResourceFile_malaria.xlsx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7f256d5007b36e2428ae844747bd766bb6086540c5135408d606dd821e185d9f -size 69578 diff --git a/resources/malaria/ResourceFile_malaria/MAP_CommoditiesData2023.csv b/resources/malaria/ResourceFile_malaria/MAP_CommoditiesData2023.csv new file mode 100644 index 0000000000..5a2903781b --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/MAP_CommoditiesData2023.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cb717d1639a4b212f1b974c9aecfa26f7982c4e40eb8c8d68a9a2808ae6944b +size 9783 diff --git a/resources/malaria/ResourceFile_malaria/MAP_IRSrates.csv b/resources/malaria/ResourceFile_malaria/MAP_IRSrates.csv new file mode 100644 index 0000000000..ba363aa865 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/MAP_IRSrates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79c9362873e293e421f3c939f29a375140b7fd1e9ac0cc77db4448230421cfce +size 10120 diff --git a/resources/malaria/ResourceFile_malaria/MAP_ITNrates.csv b/resources/malaria/ResourceFile_malaria/MAP_ITNrates.csv new file mode 100644 index 0000000000..fca1c2b759 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/MAP_ITNrates.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7295ed12e41236c5e7ce30931cc48e46dea8ab235352a5025e1f96f6cbdcb6c +size 7608 diff --git a/resources/malaria/ResourceFile_malaria/MAP_InfectionData2023.csv b/resources/malaria/ResourceFile_malaria/MAP_InfectionData2023.csv new file mode 100644 index 0000000000..f62ad813f4 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/MAP_InfectionData2023.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fc91a5288e169397932495e126e231ec0ade0bea0767c48bb2d11eb93ece1c8 +size 1108 diff --git a/resources/malaria/ResourceFile_malaria/NMCP.csv b/resources/malaria/ResourceFile_malaria/NMCP.csv new file mode 100644 index 0000000000..1e50d26083 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/NMCP.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93611c646e97f540f521fc58808d1180ff7347fed50228cd5b18ce5bedbeba12 +size 1651 diff --git a/resources/malaria/ResourceFile_malaria/PfPR_MAPdata.csv b/resources/malaria/ResourceFile_malaria/PfPR_MAPdata.csv new file mode 100644 index 0000000000..fc420097a6 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/PfPR_MAPdata.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3404a4d076b57c59c92e0c9377bd84cee33c8b0ab3fd3c9533a5b7715218dd1 +size 1641 diff --git a/resources/malaria/ResourceFile_malaria/WHO_CaseData2023.csv b/resources/malaria/ResourceFile_malaria/WHO_CaseData2023.csv new file mode 100644 index 0000000000..edb4212014 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/WHO_CaseData2023.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d8f62efe3cabd8046f1f3bb31acaabdc5f697ba67cc458bb73ef9bd2a3a3d3 +size 1358 diff --git a/resources/malaria/ResourceFile_malaria/WHO_MalReport.csv b/resources/malaria/ResourceFile_malaria/WHO_MalReport.csv new file mode 100644 index 0000000000..8b70346d9a --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/WHO_MalReport.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d210f3135ea22d4f5eea0b1d6c361df1d75af4dd2ba915f8b8c0d2f3c82a722 +size 1667 diff --git a/resources/malaria/ResourceFile_malaria/WHO_TestData2023.csv b/resources/malaria/ResourceFile_malaria/WHO_TestData2023.csv new file mode 100644 index 0000000000..067d8bc6ab --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/WHO_TestData2023.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:166944bd1976b1cf6c15dabe2d89b7d3291977761d7991de44a3fe7e869c8f1c +size 1322 diff --git a/resources/malaria/ResourceFile_malaria/WHOcommodities.csv b/resources/malaria/ResourceFile_malaria/WHOcommodities.csv new file mode 100644 index 0000000000..d3307b52b5 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/WHOcommodities.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a9e39a2b8f081e24eb180386275d90262b15ceaa1114e77eb1616d970b105f5 +size 398 diff --git a/resources/malaria/ResourceFile_malaria/highrisk_districts.csv b/resources/malaria/ResourceFile_malaria/highrisk_districts.csv new file mode 100644 index 0000000000..8501c5f516 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/highrisk_districts.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:525c87516dd3cf56fdf72f29ef0ac2d2dcafd14edf6169ea841187f217cb2282 +size 75 diff --git a/resources/malaria/ResourceFile_malaria/inc1000py_MAPdata.csv b/resources/malaria/ResourceFile_malaria/inc1000py_MAPdata.csv new file mode 100644 index 0000000000..563941dce4 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/inc1000py_MAPdata.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a68cf7c4270ab37e53ff78a1457db6a9a0dde3794072437bddc6a715611f438 +size 486 diff --git a/resources/malaria/ResourceFile_malaria/interventions.csv b/resources/malaria/ResourceFile_malaria/interventions.csv new file mode 100644 index 0000000000..413d513965 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/interventions.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51822b5693c75e056362a7a52d22a685084e2b46d7184df144728e254a5339b4 +size 1144 diff --git a/resources/malaria/ResourceFile_malaria/mortalityRate_MAPdata.csv b/resources/malaria/ResourceFile_malaria/mortalityRate_MAPdata.csv new file mode 100644 index 0000000000..56cbca1243 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/mortalityRate_MAPdata.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181ebe0b5371f2a607353e6e9a42371880eb7a1ec60f24bb5927be420fc07413 +size 1881 diff --git a/resources/malaria/ResourceFile_malaria/parameters.csv b/resources/malaria/ResourceFile_malaria/parameters.csv new file mode 100644 index 0000000000..8c29a8d524 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3616d41e17fe9f394268835709f502da39cfed8c1df253c88a7811641c05afb1 +size 784 diff --git a/resources/malaria/ResourceFile_malaria/scaleup_parameters.csv b/resources/malaria/ResourceFile_malaria/scaleup_parameters.csv new file mode 100644 index 0000000000..92788c7637 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/scaleup_parameters.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7710c479094fd4ba5cc3d49487ea85730e8d632dd8e82d9648d7948833aa439f +size 208 diff --git a/resources/malaria/ResourceFile_malaria/severe_symptoms.csv b/resources/malaria/ResourceFile_malaria/severe_symptoms.csv new file mode 100644 index 0000000000..9ac7fbd6b9 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/severe_symptoms.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7003d947ee06b6a28e3a4eafd783ea96488383a1bcf0d9e1ad4e9d4bbec3152 +size 352 diff --git a/resources/malaria/ResourceFile_malaria/txCov_MAPdata.csv b/resources/malaria/ResourceFile_malaria/txCov_MAPdata.csv new file mode 100644 index 0000000000..68d662a5c4 --- /dev/null +++ b/resources/malaria/ResourceFile_malaria/txCov_MAPdata.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60c450a83c86f2e08df790cb645fc530667d8b6e960269143778b1112b86deed +size 710 diff --git a/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py b/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py index 99349ab326..da337130af 100644 --- a/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py +++ b/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py @@ -657,6 +657,52 @@ def get_treatment_id_affecting_by_missing_consumables(_df): plt.close(fig) +def table_2_relative_frequency_of_cons_use(results_folder: Path, output_folder: Path, resourcefilepath: Path): + """Table 2: The relative frequency Consumables that are used.""" + + # Load the mapping between item_code and item_name + cons_names = pd.read_csv( + resourcefilepath / 'healthsystem' / 'consumables' / 'ResourceFile_Consumables_Items_and_Packages.csv' + )[['Item_Code', 'Items']].set_index('Item_Code').drop_duplicates() + + def get_number_of_call_to_an_item_code(_df): + """This summarizes the number of calls to a particular item_code, irrespective of the quantity requested.""" + _df = drop_outside_period(_df) + + counts_of_available = defaultdict(int) + counts_of_not_available = defaultdict(int) + + for _, row in _df.iterrows(): + for item, _ in eval(row['Item_Available']).items(): + counts_of_available[item] += 1 + for item, _ in eval(row['Item_NotAvailable']).items(): + counts_of_not_available[item] += 1 + + return pd.concat( + {'Available': pd.Series(counts_of_available), 'Not_Available': pd.Series(counts_of_not_available)}, + axis=1 + ).fillna(0).astype(int).stack() + + items_called = summarize( + extract_results( + results_folder, + module='tlo.methods.healthsystem', + key='Consumables', + custom_generate_series=get_number_of_call_to_an_item_code, + do_scaling=True + ), + only_mean=True, + collapse_columns=True + ) + + total_calls = items_called.unstack().sum(axis=1) # total calls by item_code (summing Available and Not_Availbale) + rfreq_calls = (total_calls / total_calls.sum()).sort_values(ascending=False).reset_index(name='rel_freq').rename(columns={'index': 'Item_Code'}) + rfreq_calls = rfreq_calls.merge(cons_names.reset_index(), left_on='Item_Code', right_on='Item_Code', how='left') + rfreq_calls.to_csv( + output_folder / f"{PREFIX_ON_FILENAME}_Table_Of_Frequency_Consumables_Requested.csv", + index=False + ) + def figure7_squeeze_factors(results_folder: Path, output_folder: Path, resourcefilepath: Path): """ 'Figure 7': Squeeze Factors for the HSIs""" make_graph_file_name = lambda stub: output_folder / f"{PREFIX_ON_FILENAME}_Fig7_{stub}.png" # noqa: E731 @@ -754,6 +800,10 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No results_folder=results_folder, output_folder=output_folder, resourcefilepath=resourcefilepath ) + table_2_relative_frequency_of_cons_use( + results_folder=results_folder, output_folder=output_folder, resourcefilepath=resourcefilepath + ) + figure7_squeeze_factors( results_folder=results_folder, output_folder=output_folder, resourcefilepath=resourcefilepath ) diff --git a/src/scripts/contraception/f_steril_use_2010vs2020.py b/src/scripts/contraception/f_steril_use_2010vs2020.py index a266533136..013e704312 100644 --- a/src/scripts/contraception/f_steril_use_2010vs2020.py +++ b/src/scripts/contraception/f_steril_use_2010vs2020.py @@ -1,7 +1,7 @@ """ A helper script to see the numbers of women of reproductive age having female sterilisation per 5-years age categories + total, and the number of all women in the population in 2010 and 2020, to help to calibrate the intervention multipliers -(saved in ResourceFile_Contraception.xlsx in the sheets Interventions_Pop & Interventions_PPFP). +(saved in ResourceFile_Contraception folder in the Interventions_Pop & Interventions_PPFP CSV files). """ from pathlib import Path diff --git a/src/scripts/data_file_processing/healthsystem/human_resources/formatting_human_resources_and_appt_data.py b/src/scripts/data_file_processing/healthsystem/human_resources/formatting_human_resources_and_appt_data.py index 3bcf0c2a94..fa94e4bf12 100644 --- a/src/scripts/data_file_processing/healthsystem/human_resources/formatting_human_resources_and_appt_data.py +++ b/src/scripts/data_file_processing/healthsystem/human_resources/formatting_human_resources_and_appt_data.py @@ -44,18 +44,18 @@ resourcefilepath = Path('./resources') -path_to_dropbox = Path( - '/Users/jdbb1/Dropbox/Thanzi La Onse') # <-- point to the TLO dropbox locally +path_to_onedrive = Path( + '/Users/jdbb1/Imperial College London/TLOModel - WP - Documents') # <-- point to the TLO onedrive locally -workingfile = (path_to_dropbox / +workingfile = (path_to_onedrive / '05 - Resources' / 'Module-healthsystem' / 'chai ehp resource use data' / 'ORIGINAL' / 'Malawi optimization model import_2022-02-11.xlsx') -working_file_old = (path_to_dropbox / +working_file_old = (path_to_onedrive / '05 - Resources' / 'Module-healthsystem' / 'chai ehp resource use data' / 'ORIGINAL' / 'Optimization model import_Malawi_20180315 v10.xlsx') -path_to_auxiliaryfiles = (path_to_dropbox / +path_to_auxiliaryfiles = (path_to_onedrive / '05 - Resources' / 'Module-healthsystem' / 'chai ehp resource use data' / @@ -283,8 +283,9 @@ # --- Generate assumptions of established/funded staff distribution at facility levels 0&1a&1b&2 # Read 2018-03-09 Facility-level establishment MOH & CHAM from CHAI auxiliary datasets -fund_staff_2018_raw = pd.read_excel(path_to_auxiliaryfiles / '2018-03-09 Facility-level establishment MOH & CHAM.xlsx', - sheet_name='Establishment listing') +fund_staff_2018_raw = pd.read_csv(path_to_auxiliaryfiles / '2018-03-09 Facility-level establishment MOH & CHAM.csv') +fund_staff_2018_raw['Number of positions'] = fund_staff_2018_raw['Number of positions'].fillna(0) +fund_staff_2018_raw['Number of positions'] = fund_staff_2018_raw['Number of positions'].astype(int) # Get relevant columns fund_staff_2018 = fund_staff_2018_raw[['Number of positions', 'Facility', 'Facility Type', 'WFOM Cadre']].copy() @@ -556,7 +557,9 @@ 'CenHos'].index, 'Facility_Level'] = 'Facility_Level_3' # Group staff by levels -immed_need_distribution = pd.DataFrame(immed_need_distribution.groupby(by=['Facility_Level'], sort=False).sum()) +immed_need_distribution = pd.DataFrame( + immed_need_distribution.groupby(by=['Facility_Level'], sort=False).sum() +).drop(columns=['FacilityType', 'FacilityName']) # Drop level 3 immed_need_distribution.drop(index='Facility_Level_3', inplace=True) # Reset index @@ -773,7 +776,8 @@ # Group the referral hospitals QECH and ZCH as Referral Hospital_Southern Is_DistrictLevel = fund_staffing_table['Is_DistrictLevel'].values # Save the column 'Is_DistrictLevel' first fund_staffing_table = pd.DataFrame( - fund_staffing_table.groupby(by=['District_Or_Hospital'], sort=False).sum()).reset_index() + fund_staffing_table.groupby(by=['District_Or_Hospital'], sort=False).sum() +).reset_index().drop(columns=['Is_DistrictLevel']) fund_staffing_table.insert(1, 'Is_DistrictLevel', Is_DistrictLevel[:-1]) # Add the column 'Is_DistrictLevel' # Check that in fund_staffing_table every staff count entry >= 0 @@ -809,7 +813,7 @@ record['Is_DistrictLevel'] = True # get total staff level from the super districts - cols = set(fund_staffing_table.columns).intersection(set(officer_types_table.Officer_Type_Code)) + cols = list(set(fund_staffing_table.columns).intersection(set(officer_types_table.Officer_Type_Code))) total_staff = fund_staffing_table.loc[ fund_staffing_table['District_Or_Hospital'] == super_district, cols].values.squeeze() @@ -823,7 +827,8 @@ # assign w * 100% staff to the new district record.loc[cols] = w * total_staff - fund_staffing_table = fund_staffing_table.append(record).reset_index(drop=True) + assert (record.to_frame().T.columns == fund_staffing_table.columns).all() + fund_staffing_table = pd.concat([fund_staffing_table, record.to_frame().T], axis=0).reset_index(drop=True) # take staff away from the super district fund_staffing_table.loc[fund_staffing_table['District_Or_Hospital'] == super_district, cols] = \ @@ -907,7 +912,7 @@ 'Facility_Level_4'] # Check that in fund_staffing_table every staff count entry >= 0 -assert (fund_staffing_table.loc[:, 'M01':'R04'].values >= 0).all() +assert (fund_staffing_table.loc[:, 'M01':'R04'] >= 0).all().all() # fund_staffing_table ready! # Save the table without column 'Is_DistrictLevel'; staff counts in floats @@ -960,7 +965,8 @@ # Group the referral hospitals QECH and ZCH as Referral Hospital_Southern Is_DistrictLevel = curr_staffing_table['Is_DistrictLevel'].values # Save the column 'Is_DistrictLevel' first curr_staffing_table = pd.DataFrame( - curr_staffing_table.groupby(by=['District_Or_Hospital'], sort=False).sum()).reset_index() + curr_staffing_table.groupby(by=['District_Or_Hospital'], sort=False).sum() +).reset_index().drop(columns='Is_DistrictLevel') curr_staffing_table.insert(1, 'Is_DistrictLevel', Is_DistrictLevel[:-1]) # Add the column 'Is_DistrictLevel' # No need to add a row for Zomba Mental Hospital, as the updated CHAI data has this row for ZMH. @@ -993,7 +999,7 @@ record['Is_DistrictLevel'] = True # get total staff level from the super districts - cols = set(curr_staffing_table.columns).intersection(set(officer_types_table.Officer_Type_Code)) + cols = list(set(curr_staffing_table.columns).intersection(set(officer_types_table.Officer_Type_Code))) total_staff = curr_staffing_table.loc[ curr_staffing_table['District_Or_Hospital'] == super_district, cols].values.squeeze() @@ -1008,7 +1014,8 @@ # assign w * 100% staff to the new district record.loc[cols] = w * total_staff - curr_staffing_table = curr_staffing_table.append(record).reset_index(drop=True) + assert (record.to_frame().T.columns == curr_staffing_table.columns).all() + curr_staffing_table = pd.concat([curr_staffing_table, record.to_frame().T], axis=0).reset_index(drop=True) # take staff away from the super district curr_staffing_table.loc[curr_staffing_table['District_Or_Hospital'] == super_district, cols] = \ @@ -1105,23 +1112,23 @@ for d in pop_districts: df = pd.DataFrame({'Facility_Level': Facility_Levels[0:4], 'District': d, 'Region': pop.loc[pop['District'] == d, 'Region'].values[0]}) - mfl = mfl.append(df, ignore_index=True, sort=True) + mfl = pd.concat([mfl, df], ignore_index=True, sort=True) # Add in the Referral Hospitals, one for each region for r in pop_regions: - mfl = mfl.append(pd.DataFrame({ - 'Facility_Level': Facility_Levels[4], 'District': None, 'Region': r - }, index=[0]), ignore_index=True, sort=True) + df = pd.DataFrame({ + 'Facility_Level': Facility_Levels[4], 'District': None, 'Region': r}, index=[0]) + mfl = pd.concat([mfl, df], ignore_index=True, sort=True) # Add the ZMH -mfl = mfl.append(pd.DataFrame({ - 'Facility_Level': Facility_Levels[5], 'District': None, 'Region': None -}, index=[0]), ignore_index=True, sort=True) +df = pd.DataFrame({ + 'Facility_Level': Facility_Levels[5], 'District': None, 'Region': None}, index=[0]) +mfl = pd.concat([mfl, df], ignore_index=True, sort=True) # Add the HQ -mfl = mfl.append(pd.DataFrame({ - 'Facility_Level': Facility_Levels[6], 'District': None, 'Region': None -}, index=[0]), ignore_index=True, sort=True) +df = pd.DataFrame({ + 'Facility_Level': Facility_Levels[6], 'District': None, 'Region': None}, index=[0]) +mfl = pd.concat([mfl, df], ignore_index=True, sort=True) # Create the Facility_ID mfl.loc[:, 'Facility_ID'] = mfl.index @@ -1409,7 +1416,7 @@ # Generate appt_time_table_coarse with officer_category, instead of officer_type appt_time_table_coarse = pd.DataFrame( ApptTimeTable.groupby(['Appt_Type_Code', 'Facility_Level', 'Officer_Category']).sum() -).reset_index() +).reset_index().drop(columns=['Officer_Type', 'Officer_Type_Code']) # Save # ApptTimeTable.to_csv( @@ -1475,19 +1482,14 @@ if len(block) == 0: # no requirement expressed => The appt is not possible at this location - Officers_Need_For_Appt = Officers_Need_For_Appt.append( - {'Facility_Level': f, - 'Appt_Type_Code': a, - 'Officer_Type_Codes': False - }, ignore_index=True) + df = pd.DataFrame({'Facility_Level': f, 'Appt_Type_Code': a, 'Officer_Type_Codes': False}, index=[0]) + Officers_Need_For_Appt = pd.concat([Officers_Need_For_Appt, df], ignore_index=True) else: need_officer_types = list(block['Officer_Type_Code']) - Officers_Need_For_Appt = Officers_Need_For_Appt.append( - {'Facility_Level': f, - 'Appt_Type_Code': a, - 'Officer_Type_Codes': need_officer_types - }, ignore_index=True) + df = pd.DataFrame({'Facility_Level': f, 'Appt_Type_Code': a, 'Officer_Type_Codes': need_officer_types}, + index=range(len(block))) + Officers_Need_For_Appt = pd.concat([Officers_Need_For_Appt, df], ignore_index=True) # Turn this into the the set of staff that are required for each type of appointment FacLevel_By_Officer = pd.DataFrame(columns=Facility_Levels, @@ -1675,7 +1677,8 @@ HosHC_patient_facing_time_old.iloc[:, 1:].values) / HosHC_patient_facing_time_old.iloc[:, 1:].values ) -HosHC_pft_diff = HosHC_pft_diff.append(HosHC_pft_diff.iloc[:, 1:].mean(axis=0), ignore_index=True) +df = HosHC_pft_diff.iloc[:, 1:].mean(axis=0).to_frame().T +HosHC_pft_diff = pd.concat([HosHC_pft_diff, df], ignore_index=True) # save # HosHC_pft_diff.to_csv( @@ -1746,13 +1749,8 @@ funded_daily_capability.drop(columns='District_Or_Hospital', inplace=True) # Add info from mfl: Region and Facility ID -for i in funded_daily_capability.index: - the_facility_name = funded_daily_capability.loc[i, 'Facility_Name'] - the_ID = mfl.loc[mfl['Facility_Name'] == the_facility_name, 'Facility_ID'] - the_region = mfl.loc[mfl['Facility_Name'] == the_facility_name, 'Region'] - - funded_daily_capability.loc[i, 'Facility_ID'] = the_ID.values - funded_daily_capability.loc[i, 'Region'] = the_region.values +funded_daily_capability = funded_daily_capability.merge( + mfl[['Facility_Name', 'Facility_ID', 'Region']], on='Facility_Name', how='left') # Add 'officer_category' info funded_daily_capability = funded_daily_capability.merge(officer_types_table, on='Officer_Type_Code', how='left') @@ -1763,6 +1761,9 @@ ['Facility_ID', 'Facility_Name', 'Facility_Level', 'District', 'Region', 'Officer_Category'], dropna=False)[['Total_Mins_Per_Day', 'Staff_Count']].sum() ).reset_index() +# None-necessary changes of data format; just to keep in consistency with TLO master resource files +funded_daily_capability_coarse['Staff_Count'] = funded_daily_capability_coarse['Staff_Count'].astype(float) +funded_daily_capability_coarse['Facility_ID'] = funded_daily_capability_coarse['Facility_ID'].astype(float) # Since not dropped zero-minute rows in lines 1717-1718, # check that there are entries for all coarse cadres and all facility id @@ -1825,13 +1826,8 @@ curr_daily_capability.drop(columns='District_Or_Hospital', inplace=True) # Add info from mfl: Region and Facility ID -for i in curr_daily_capability.index: - the_facility_name = curr_daily_capability.loc[i, 'Facility_Name'] - the_ID = mfl.loc[mfl['Facility_Name'] == the_facility_name, 'Facility_ID'] - the_region = mfl.loc[mfl['Facility_Name'] == the_facility_name, 'Region'] - - curr_daily_capability.loc[i, 'Facility_ID'] = the_ID.values - curr_daily_capability.loc[i, 'Region'] = the_region.values +curr_daily_capability = curr_daily_capability.merge( + mfl[['Facility_Name', 'Facility_ID', 'Region']], on='Facility_Name', how='left') # Add 'officer_category' info curr_daily_capability = curr_daily_capability.merge(officer_types_table, on='Officer_Type_Code', how='left') @@ -1842,6 +1838,9 @@ ['Facility_ID', 'Facility_Name', 'Facility_Level', 'District', 'Region', 'Officer_Category'], dropna=False)[['Total_Mins_Per_Day', 'Staff_Count']].sum() ).reset_index() +# None-necessary changes of data format; just to keep in consistency with TLO master resource files +curr_daily_capability_coarse['Staff_Count'] = curr_daily_capability_coarse['Staff_Count'].astype(float) +curr_daily_capability_coarse['Facility_ID'] = curr_daily_capability_coarse['Facility_ID'].astype(float) # Since not dropped zero-minute rows in lines 1797-1798, # check that there are entries for all coarse cadres and all facility id diff --git a/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py b/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py index 3902e5d49b..59023a7544 100644 --- a/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py +++ b/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py @@ -70,7 +70,7 @@ resourcefilepath=resourcefilepath, service_availability=["*"], # all treatment allowed mode_appt_constraints=1, # mode of constraints to do with officer numbers and time - cons_availability="all", # mode for consumable constraints (if ignored, all consumables available) + cons_availability="default", # mode for consumable constraints (if ignored, all consumables available) ignore_priority=False, # do not use the priority information in HSI event to schedule capabilities_coefficient=1.0, # multiplier for the capabilities of health officers use_funded_or_actual_staffing="actual", # actual: use numbers/distribution of staff available currently diff --git a/src/scripts/hiv/projections_jan2023/calibration_script.py b/src/scripts/hiv/projections_jan2023/calibration_script.py index fb36a5ed8d..87c1ce0548 100644 --- a/src/scripts/hiv/projections_jan2023/calibration_script.py +++ b/src/scripts/hiv/projections_jan2023/calibration_script.py @@ -69,7 +69,7 @@ def __init__(self): self.runs_per_draw = runs_per_draw self.sampled_parameters = pd.read_excel( - os.path.join(self.resources, "../../../../resources/ResourceFile_HIV.xlsx"), + os.path.join(self.resources, "../../../../resources/ResourceFile_HIV/parameters.csv"), sheet_name="LHC_samples", ) diff --git a/src/scripts/hiv/projections_jan2023/output_plots.py b/src/scripts/hiv/projections_jan2023/output_plots.py index 43c2cfcf77..d580ee53c9 100644 --- a/src/scripts/hiv/projections_jan2023/output_plots.py +++ b/src/scripts/hiv/projections_jan2023/output_plots.py @@ -90,7 +90,7 @@ def make_plot(model=None, data_mid=None, data_low=None, data_high=None, title_st data_tb_ntp = data_tb_ntp.drop(columns=["year"]) # HIV resourcefile -xls = pd.ExcelFile(resourcefilepath / "ResourceFile_HIV.xlsx") +xls = pd.ExcelFile(resourcefilepath / "ResourceFile_HIV/parameters.csv") # HIV UNAIDS data data_hiv_unaids = pd.read_excel(xls, sheet_name="unaids_infections_art2021") diff --git a/src/scripts/hiv/projections_jan2023/plot_batch_outputs.py b/src/scripts/hiv/projections_jan2023/plot_batch_outputs.py deleted file mode 100644 index 808cadce49..0000000000 --- a/src/scripts/hiv/projections_jan2023/plot_batch_outputs.py +++ /dev/null @@ -1,1065 +0,0 @@ -"""This file uses the results of the batch file to make some summary statistics. -The results of the batchrun were put into the 'outputspath' results_folder -function weighted_mean_for_data_comparison can be used to select which parameter sets to use -make plots for top 5 parameter sets just to make sure they are looking ok -""" - -import datetime -from pathlib import Path - -import matplotlib.lines as mlines -import matplotlib.patches as mpatches -import matplotlib.pyplot as plt -import pandas as pd - -from tlo.analysis.utils import ( - compare_number_of_deaths, - extract_params, - extract_results, - get_scenario_info, - get_scenario_outputs, - load_pickled_dataframes, - summarize, -) - -resourcefilepath = Path("./resources") -datestamp = datetime.date.today().strftime("__%Y_%m_%d") - -outputspath = Path("./outputs/t.mangal@imperial.ac.uk") - -# %% read in data files for plots -# load all the data for calibration - -# TB WHO data -xls_tb = pd.ExcelFile(resourcefilepath / "ResourceFile_TB.xlsx") - -data_tb_who = pd.read_excel(xls_tb, sheet_name="WHO_activeTB2023") -data_tb_who = data_tb_who.loc[ - (data_tb_who.year >= 2010) -] # include only years post-2010 -data_tb_who.index = data_tb_who["year"] -data_tb_who = data_tb_who.drop(columns=["year"]) - -# TB latent data (Houben & Dodd 2016) -data_tb_latent = pd.read_excel(xls_tb, sheet_name="latent_TB2014_summary") -data_tb_latent_all_ages = data_tb_latent.loc[data_tb_latent.Age_group == "0_80"] -data_tb_latent_estimate = data_tb_latent_all_ages.proportion_latent_TB.values[0] -data_tb_latent_lower = abs( - data_tb_latent_all_ages.proportion_latent_TB_lower.values[0] - - data_tb_latent_estimate -) -data_tb_latent_upper = abs( - data_tb_latent_all_ages.proportion_latent_TB_upper.values[0] - - data_tb_latent_estimate -) -data_tb_latent_yerr = [data_tb_latent_lower, data_tb_latent_upper] - -# TB deaths WHO -deaths_2010_2014 = data_tb_who.loc["2010-01-01":"2014-01-01"] -deaths_2015_2019 = data_tb_who.loc["2015-01-01":"2019-01-01"] - -deaths_2010_2014_average = deaths_2010_2014.loc[:, "num_deaths_tb_nonHiv"].values.mean() -deaths_2010_2014_average_low = deaths_2010_2014.loc[:, "num_deaths_tb_nonHiv_low"].values.mean() -deaths_2010_2014_average_high = deaths_2010_2014.loc[:, "num_deaths_tb_nonHiv_high"].values.mean() - -deaths_2015_2019_average = deaths_2015_2019.loc[:, "num_deaths_tb_nonHiv"].values.mean() -deaths_2015_2019_average_low = deaths_2015_2019.loc[:, "num_deaths_tb_nonHiv_low"].values.mean() -deaths_2015_2019_average_high = deaths_2015_2019.loc[:, "num_deaths_tb_nonHiv_high"].values.mean() - -# TB treatment coverage -data_tb_ntp = pd.read_excel(xls_tb, sheet_name="NTP2019") -data_tb_ntp.index = data_tb_ntp["year"] -data_tb_ntp = data_tb_ntp.drop(columns=["year"]) - -# HIV resourcefile -xls = pd.ExcelFile(resourcefilepath / "ResourceFile_HIV.xlsx") - -# HIV UNAIDS data -data_hiv_unaids = pd.read_excel(xls, sheet_name="unaids_infections_art2021") -data_hiv_unaids.index = data_hiv_unaids["year"] -data_hiv_unaids = data_hiv_unaids.drop(columns=["year"]) - -# HIV UNAIDS data -data_hiv_unaids_deaths = pd.read_excel(xls, sheet_name="unaids_mortality_dalys2021") -data_hiv_unaids_deaths.index = data_hiv_unaids_deaths["year"] -data_hiv_unaids_deaths = data_hiv_unaids_deaths.drop(columns=["year"]) - -# AIDSinfo (UNAIDS) -data_hiv_aidsinfo = pd.read_excel(xls, sheet_name="children0_14_prev_AIDSinfo") -data_hiv_aidsinfo.index = data_hiv_aidsinfo["year"] -data_hiv_aidsinfo = data_hiv_aidsinfo.drop(columns=["year"]) - -# unaids program performance -data_hiv_program = pd.read_excel(xls, sheet_name="unaids_program_perf") -data_hiv_program.index = data_hiv_program["year"] -data_hiv_program = data_hiv_program.drop(columns=["year"]) - -# MPHIA HIV data - age-structured -data_hiv_mphia_inc = pd.read_excel(xls, sheet_name="MPHIA_incidence2015") -data_hiv_mphia_inc_estimate = data_hiv_mphia_inc.loc[ - (data_hiv_mphia_inc.age == "15-49"), "total_percent_annual_incidence" -].values[0] -data_hiv_mphia_inc_lower = data_hiv_mphia_inc.loc[ - (data_hiv_mphia_inc.age == "15-49"), "total_percent_annual_incidence_lower" -].values[0] -data_hiv_mphia_inc_upper = data_hiv_mphia_inc.loc[ - (data_hiv_mphia_inc.age == "15-49"), "total_percent_annual_incidence_upper" -].values[0] -data_hiv_mphia_inc_yerr = [ - abs(data_hiv_mphia_inc_lower - data_hiv_mphia_inc_estimate), - abs(data_hiv_mphia_inc_upper - data_hiv_mphia_inc_estimate), -] - -data_hiv_mphia_prev = pd.read_excel(xls, sheet_name="MPHIA_prevalence_art2015") - -# DHS HIV data -data_hiv_dhs_prev = pd.read_excel(xls, sheet_name="DHS_prevalence") - -# MoH HIV testing data -data_hiv_moh_tests = pd.read_excel(xls, sheet_name="MoH_numbers_tests") -data_hiv_moh_tests.index = data_hiv_moh_tests["year"] -data_hiv_moh_tests = data_hiv_moh_tests.drop(columns=["year"]) - -# MoH HIV ART data -# todo this is quarterly -data_hiv_moh_art = pd.read_excel(xls, sheet_name="MoH_number_art") - -# %% Analyse results of runs - -# 0) Find results_folder associated with a given batch_file (and get most recent [-1]) -results_folder = get_scenario_outputs("batch_test_runs.py", outputspath)[-1] - -# Declare path for output graphs from this script -make_graph_file_name = lambda stub: results_folder / f"{stub}.png" # noqa: E731 - -# look at one log (so can decide what to extract) -log = load_pickled_dataframes(results_folder) - -# get basic information about the results -info = get_scenario_info(results_folder) - -# 1) Extract the parameters that have varied over the set of simulations -params = extract_params(results_folder) - -# choose which draw to summarise / visualise -draw = 0 - -# %% extract results -# Load and format model results (with year as integer): - -# ---------------------------------- HIV ---------------------------------- # -model_hiv_adult_prev = summarize( - extract_results( - results_folder, - module="tlo.methods.hiv", - key="summary_inc_and_prev_for_adults_and_children_and_fsw", - column="hiv_prev_adult_15plus", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) -model_hiv_adult_prev.index = model_hiv_adult_prev.index.year - -model_hiv_adult_inc = summarize( - extract_results( - results_folder, - module="tlo.methods.hiv", - key="summary_inc_and_prev_for_adults_and_children_and_fsw", - column="hiv_adult_inc_1549", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) -model_hiv_adult_inc.index = model_hiv_adult_inc.index.year - -model_hiv_child_prev = summarize( - extract_results( - results_folder, - module="tlo.methods.hiv", - key="summary_inc_and_prev_for_adults_and_children_and_fsw", - column="hiv_prev_child", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) -model_hiv_child_prev.index = model_hiv_child_prev.index.year - -model_hiv_child_inc = summarize( - extract_results( - results_folder, - module="tlo.methods.hiv", - key="summary_inc_and_prev_for_adults_and_children_and_fsw", - column="hiv_child_inc", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) -model_hiv_child_inc.index = model_hiv_child_inc.index.year - -model_hiv_fsw_prev = summarize( - extract_results( - results_folder, - module="tlo.methods.hiv", - key="summary_inc_and_prev_for_adults_and_children_and_fsw", - column="hiv_prev_fsw", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) -model_hiv_fsw_prev.index = model_hiv_fsw_prev.index.year - - -# ---------------------------------- PERSON-YEARS ---------------------------------- # - -# function to extract person-years by year -# call this for each run and then take the mean to use as denominator for mortality / incidence etc. -def get_person_years(draw, run): - log = load_pickled_dataframes(results_folder, draw, run) - - py_ = log["tlo.methods.demography"]["person_years"] - years = pd.to_datetime(py_["date"]).dt.year - py = pd.Series(dtype="int64", index=years) - for year in years: - tot_py = ( - (py_.loc[pd.to_datetime(py_["date"]).dt.year == year]["M"]).apply(pd.Series) + - (py_.loc[pd.to_datetime(py_["date"]).dt.year == year]["F"]).apply(pd.Series) - ).transpose() - py[year] = tot_py.sum().values[0] - - py.index = pd.to_datetime(years, format="%Y") - - return py - - -# for draw 0, get py for all runs -number_runs = info["runs_per_draw"] -py_summary = pd.DataFrame(data=None, columns=range(0, number_runs)) - -# draw number (default = 0) is specified above -for run in range(0, number_runs): - py_summary.iloc[:, run] = get_person_years(draw, run) - -py_summary["mean"] = py_summary.mean(axis=1) - -# ---------------------------------- TB ---------------------------------- # - -model_tb_inc = summarize( - extract_results( - results_folder, - module="tlo.methods.tb", - key="tb_incidence", - column="num_new_active_tb", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) -model_tb_inc.index = model_tb_inc.index.year -activeTB_inc_rate = (model_tb_inc.divide(py_summary["mean"].values[1:10], axis=0)) * 100000 - -model_tb_mdr = summarize( - extract_results( - results_folder, - module="tlo.methods.tb", - key="tb_mdr", - column="tbPropActiveCasesMdr", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) - -model_tb_mdr.index = model_tb_mdr.index.year - -model_tb_hiv_prop = summarize( - extract_results( - results_folder, - module="tlo.methods.tb", - key="tb_incidence", - column="prop_active_tb_in_plhiv", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) - -model_tb_hiv_prop.index = model_tb_hiv_prop.index.year - -# ---------------------------------- DEATHS ---------------------------------- # - -results_deaths = extract_results( - results_folder, - module="tlo.methods.demography", - key="death", - custom_generate_series=( - lambda df: df.assign(year=df["date"].dt.year).groupby( - ["year", "cause"])["person_id"].count() - ), - do_scaling=False, -) - -results_deaths = results_deaths.reset_index() - -# summarise across runs -aids_non_tb_deaths_table = results_deaths.loc[results_deaths.cause == "AIDS_non_TB"] -aids_tb_deaths_table = results_deaths.loc[results_deaths.cause == "AIDS_TB"] -tb_deaths_table = results_deaths.loc[results_deaths.cause == "TB"] - -# ------------ summarise deaths producing df for each draw - -# AIDS deaths -aids_deaths = {} # dict of df - -for draw in info["number_of_draws"]: - draw = draw - - # rename dataframe - name = "model_deaths_AIDS_draw" + str(draw) - # select cause of death - tmp = results_deaths.loc[ - (results_deaths.cause == "AIDS_TB") | (results_deaths.cause == "AIDS_non_TB") - ] - - # select draw - drop columns where draw != 0, but keep year and cause - tmp2 = tmp.loc[ - :, ("draw" == draw) - ].copy() # selects only columns for draw=0 (removes year/cause) - # join year and cause back to df - needed for groupby - frames = [tmp["year"], tmp["cause"], tmp2] - tmp3 = pd.concat(frames, axis=1) - - # create new column names, dependent on number of runs in draw - base_columns = ["year", "cause"] - run_columns = ["run" + str(x) for x in range(0, info["runs_per_draw"])] - base_columns.extend(run_columns) - tmp3.columns = base_columns - tmp3 = tmp3.set_index("year") - - # sum rows for each year (2 entries = 2 causes of death) - # for each run need to combine deaths in each year, may have different numbers of runs - aids_deaths[name] = pd.DataFrame(tmp3.groupby(["year"]).sum()) - - # double check all columns are float64 or quantile argument will fail - cols = [ - col - for col in aids_deaths[name].columns - if aids_deaths[name][col].dtype == "float64" - ] - aids_deaths[name]["median"] = ( - aids_deaths[name][cols].astype(float).quantile(0.5, axis=1) - ) - aids_deaths[name]["lower"] = ( - aids_deaths[name][cols].astype(float).quantile(0.025, axis=1) - ) - aids_deaths[name]["upper"] = ( - aids_deaths[name][cols].astype(float).quantile(0.975, axis=1) - ) - - # AIDS mortality rates per 100k person-years - aids_deaths[name]["aids_deaths_rate_100kpy"] = ( - aids_deaths[name]["median"].values / py_summary["mean"].values) * 100000 - - aids_deaths[name]["aids_deaths_rate_100kpy_lower"] = ( - aids_deaths[name]["lower"].values / py_summary["mean"].values) * 100000 - - aids_deaths[name]["aids_deaths_rate_100kpy_upper"] = ( - aids_deaths[name]["upper"].values / py_summary["mean"].values) * 100000 - - -# HIV/TB deaths -aids_tb_deaths = {} # dict of df - -for draw in info["number_of_draws"]: - draw = draw - - # rename dataframe - name = "model_deaths_AIDS_TB_draw" + str(draw) - # select cause of death - tmp = results_deaths.loc[ - (results_deaths.cause == "AIDS_TB") - ] - # select draw - drop columns where draw != 0, but keep year and cause - tmp2 = tmp.loc[ - :, ("draw" == draw) - ].copy() # selects only columns for draw=0 (removes year/cause) - # join year and cause back to df - needed for groupby - frames = [tmp["year"], tmp["cause"], tmp2] - tmp3 = pd.concat(frames, axis=1) - - # create new column names, dependent on number of runs in draw - base_columns = ["year", "cause"] - run_columns = ["run" + str(x) for x in range(0, info["runs_per_draw"])] - base_columns.extend(run_columns) - tmp3.columns = base_columns - tmp3 = tmp3.set_index("year") - - # sum rows for each year (2 entries) - # for each run need to combine deaths in each year, may have different numbers of runs - aids_tb_deaths[name] = pd.DataFrame(tmp3.groupby(["year"]).sum()) - - # double check all columns are float64 or quantile argument will fail - cols = [ - col - for col in aids_tb_deaths[name].columns - if aids_tb_deaths[name][col].dtype == "float64" - ] - aids_tb_deaths[name]["median"] = ( - aids_tb_deaths[name][cols].astype(float).quantile(0.5, axis=1) - ) - aids_tb_deaths[name]["lower"] = ( - aids_tb_deaths[name][cols].astype(float).quantile(0.025, axis=1) - ) - aids_tb_deaths[name]["upper"] = ( - aids_tb_deaths[name][cols].astype(float).quantile(0.975, axis=1) - ) - - # AIDS_TB mortality rates per 100k person-years - aids_tb_deaths[name]["aids_TB_deaths_rate_100kpy"] = ( - aids_tb_deaths[name]["median"].values / py_summary["mean"].values) * 100000 - - aids_tb_deaths[name]["aids_TB_deaths_rate_100kpy_lower"] = ( - aids_tb_deaths[name]["lower"].values / py_summary["mean"].values) * 100000 - - aids_tb_deaths[name]["aids_TB_deaths_rate_100kpy_upper"] = ( - aids_tb_deaths[name]["upper"].values / py_summary["mean"].values) * 100000 - - -# TB deaths excluding HIV -tb_deaths = {} # dict of df - -for draw in info["number_of_draws"]: - draw = draw - - # rename dataframe - name = "model_deaths_TB_draw" + str(draw) - # select cause of death - tmp = results_deaths.loc[ - (results_deaths.cause == "TB") - ] - # select draw - drop columns where draw != 0, but keep year and cause - tmp2 = tmp.loc[ - :, ("draw" == draw) - ].copy() # selects only columns for draw=0 (removes year/cause) - # join year and cause back to df - needed for groupby - frames = [tmp["year"], tmp["cause"], tmp2] - tmp3 = pd.concat(frames, axis=1) - - # create new column names, dependent on number of runs in draw - base_columns = ["year", "cause"] - run_columns = ["run" + str(x) for x in range(0, info["runs_per_draw"])] - base_columns.extend(run_columns) - tmp3.columns = base_columns - tmp3 = tmp3.set_index("year") - - # sum rows for each year (2 entries) - # for each run need to combine deaths in each year, may have different numbers of runs - tb_deaths[name] = pd.DataFrame(tmp3.groupby(["year"]).sum()) - - # double check all columns are float64 or quantile argument will fail - cols = [ - col - for col in tb_deaths[name].columns - if tb_deaths[name][col].dtype == "float64" - ] - tb_deaths[name]["median"] = ( - tb_deaths[name][cols].astype(float).quantile(0.5, axis=1) - ) - tb_deaths[name]["lower"] = ( - tb_deaths[name][cols].astype(float).quantile(0.025, axis=1) - ) - tb_deaths[name]["upper"] = ( - tb_deaths[name][cols].astype(float).quantile(0.975, axis=1) - ) - - # AIDS_TB mortality rates per 100k person-years - tb_deaths[name]["TB_death_rate_100kpy"] = ( - tb_deaths[name]["median"].values / py_summary["mean"].values) * 100000 - - tb_deaths[name]["TB_death_rate_100kpy_lower"] = ( - tb_deaths[name]["lower"].values / py_summary["mean"].values) * 100000 - - tb_deaths[name]["TB_death_rate_100kpy_upper"] = ( - tb_deaths[name]["upper"].values / py_summary["mean"].values) * 100000 - - -# ---------------------------------- PROGRAM COVERAGE ---------------------------------- # - -# TB treatment coverage -model_tb_tx = summarize( - extract_results( - results_folder, - module="tlo.methods.tb", - key="tb_treatment", - column="tbTreatmentCoverage", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) - -model_tb_tx.index = model_tb_tx.index.year - -# HIV treatment coverage -model_hiv_tx = summarize( - extract_results( - results_folder, - module="tlo.methods.hiv", - key="hiv_program_coverage", - column="art_coverage_adult", - index="date", - do_scaling=False, - ), - collapse_columns=True, -) - -model_hiv_tx.index = model_hiv_tx.index.year - - -# %% Function to make standard plot to compare model and data -def make_plot( - model=None, - model_low=None, - model_high=None, - data_name=None, - data_mid=None, - data_low=None, - data_high=None, - xlab=None, - ylab=None, - title_str=None, -): - assert model is not None - assert title_str is not None - - # Make plot - fig, ax = plt.subplots() - ax.plot(model.index, model.values, "-", color="C3") - if (model_low is not None) and (model_high is not None): - ax.fill_between(model_low.index, model_low, model_high, color="C3", alpha=0.2) - - if data_mid is not None: - ax.plot(data_mid.index, data_mid.values, "-", color="C0") - if (data_low is not None) and (data_high is not None): - ax.fill_between(data_low.index, data_low, data_high, color="C0", alpha=0.2) - - if xlab is not None: - ax.set_xlabel(xlab) - - if ylab is not None: - ax.set_xlabel(ylab) - - plt.title(title_str) - plt.legend(["TLO", data_name]) - # plt.gca().set_ylim(bottom=0) - # plt.savefig(outputspath / (title_str.replace(" ", "_") + datestamp + ".pdf"), format='pdf') - - -# %% make plots - -# HIV - prevalence among in adults aged 15-49 - -make_plot( - title_str="HIV Prevalence in Adults Aged 15-49 (%)", - model=model_hiv_adult_prev["mean"] * 100, - model_low=model_hiv_adult_prev["lower"] * 100, - model_high=model_hiv_adult_prev["upper"] * 100, - data_name="UNAIDS", - data_mid=data_hiv_unaids["prevalence_age15_49"] * 100, - data_low=data_hiv_unaids["prevalence_age15_49_lower"] * 100, - data_high=data_hiv_unaids["prevalence_age15_49_upper"] * 100, -) - -# data: MPHIA -plt.plot( - model_hiv_adult_prev.index[6], - data_hiv_mphia_prev.loc[ - data_hiv_mphia_prev.age == "Total 15-49", "total percent hiv positive" - ].values[0], - "gx", -) - -# data: DHS -x_values = [model_hiv_adult_prev.index[0], model_hiv_adult_prev.index[5]] -y_values = data_hiv_dhs_prev.loc[ - (data_hiv_dhs_prev.Year >= 2010), "HIV prevalence among general population 15-49" -] -y_lower = abs( - y_values - - ( - data_hiv_dhs_prev.loc[ - (data_hiv_dhs_prev.Year >= 2010), - "HIV prevalence among general population 15-49 lower", - ] - ) -) -y_upper = abs( - y_values - - ( - data_hiv_dhs_prev.loc[ - (data_hiv_dhs_prev.Year >= 2010), - "HIV prevalence among general population 15-49 upper", - ] - ) -) -plt.errorbar(x_values, y_values, yerr=[y_lower, y_upper], fmt="ko") - -plt.ylim((0, 15)) -plt.xlabel = ("Year",) -plt.ylabel = "HIV prevalence (%)" - -# handles for legend -red_line = mlines.Line2D([], [], color="C3", markersize=15, label="TLO") -blue_line = mlines.Line2D([], [], color="C0", markersize=15, label="UNAIDS") -green_cross = mlines.Line2D( - [], [], linewidth=0, color="g", marker="x", markersize=7, label="MPHIA" -) -orange_ci = mlines.Line2D([], [], color="black", marker=".", markersize=15, label="DHS") -plt.legend(handles=[red_line, blue_line, green_cross, orange_ci]) -# plt.savefig(make_graph_file_name("HIV_Prevalence_in_Adults")) - -plt.show() - -# ---------------------------------------------------------------------- # - -# HIV Incidence in adults aged 15-49 per 100 population -make_plot( - title_str="HIV Incidence in Adults Aged 15-49 per 100 population", - model=model_hiv_adult_inc["mean"] * 100, - model_low=model_hiv_adult_inc["lower"] * 100, - model_high=model_hiv_adult_inc["upper"] * 100, - data_name="UNAIDS", - data_mid=data_hiv_unaids["incidence_per1000_age15_49"] / 10, - data_low=data_hiv_unaids["incidence_per1000_age15_49_lower"] / 10, - data_high=data_hiv_unaids["incidence_per1000_age15_49_upper"] / 10, -) - -plt.xlabel = ("Year",) -plt.ylabel = "HIV incidence per 1000 population" - -# MPHIA -plt.errorbar( - model_hiv_adult_inc.index[6], - data_hiv_mphia_inc_estimate, - yerr=[[data_hiv_mphia_inc_yerr[0]], [data_hiv_mphia_inc_yerr[1]]], - fmt="gx", -) - -plt.ylim(0, 1.0) -plt.xlim(2010, 2020) -# -# handles for legend -red_line = mlines.Line2D([], [], color="C3", markersize=15, label="TLO") -blue_line = mlines.Line2D([], [], color="C0", markersize=15, label="UNAIDS") -orange_ci = mlines.Line2D( - [], [], color="green", marker="x", markersize=8, label="MPHIA" -) -plt.legend(handles=[red_line, blue_line, orange_ci]) - -# plt.savefig(make_graph_file_name("HIV_Incidence_in_Adults")) - -plt.show() - -# ---------------------------------------------------------------------- # - -# HIV Prevalence Children -make_plot( - title_str="HIV Prevalence in Children 0-14 (%)", - model=model_hiv_child_prev["mean"] * 100, - model_low=model_hiv_child_prev["lower"] * 100, - model_high=model_hiv_child_prev["upper"] * 100, - data_name="UNAIDS", - data_mid=data_hiv_aidsinfo["prevalence_0_14"] * 100, - data_low=data_hiv_aidsinfo["prevalence_0_14_lower"] * 100, - data_high=data_hiv_aidsinfo["prevalence_0_14_upper"] * 100, - xlab="Year", - ylab="HIV prevalence (%)", -) - -# MPHIA -plt.plot( - model_hiv_child_prev.index[6], - data_hiv_mphia_prev.loc[ - data_hiv_mphia_prev.age == "Total 0-14", "total percent hiv positive" - ].values[0], - "gx", -) - -plt.xlim = (2010, 2020) -plt.ylim = (0, 5) - -# handles for legend -red_line = mlines.Line2D([], [], color="C3", markersize=15, label="TLO") -blue_line = mlines.Line2D([], [], color="C0", markersize=15, label="UNAIDS") -green_cross = mlines.Line2D( - [], [], linewidth=0, color="g", marker="x", markersize=7, label="MPHIA" -) -plt.legend(handles=[red_line, blue_line, green_cross]) -# plt.savefig(make_graph_file_name("HIV_Prevalence_in_Children")) - -plt.show() - -# ---------------------------------------------------------------------- # - -# HIV Incidence Children -make_plot( - title_str="HIV Incidence in Children (0-14) (per 100 pyar)", - model=model_hiv_child_inc["mean"] * 100, - model_low=model_hiv_child_inc["lower"] * 100, - model_high=model_hiv_child_inc["upper"] * 100, - data_mid=data_hiv_aidsinfo["incidence0_14_per100py"], - data_low=data_hiv_aidsinfo["incidence0_14_per100py_lower"], - data_high=data_hiv_aidsinfo["incidence0_14_per100py_upper"], -) -# plt.savefig(make_graph_file_name("HIV_Incidence_in_Children")) - -plt.show() - -# ---------------------------------------------------------------------- # - -# HIV prevalence among female sex workers: -make_plot( - title_str="HIV Prevalence among Female Sex Workers (%)", - model=model_hiv_fsw_prev["mean"] * 100, - model_low=model_hiv_fsw_prev["lower"] * 100, - model_high=model_hiv_fsw_prev["upper"] * 100, -) -# plt.savefig(make_graph_file_name("HIV_Prevalence_FSW")) - -plt.show() - -# ----------------------------- TB -------------------------------------- # - -# Active TB incidence per 100,000 person-years - annual outputs - -make_plot( - title_str="Active TB Incidence (per 100k person-years)", - model=activeTB_inc_rate["mean"], - model_low=activeTB_inc_rate["lower"], - model_high=activeTB_inc_rate["upper"], - data_name="WHO_TB", - data_mid=data_tb_who["incidence_per_100k"], - data_low=data_tb_who["incidence_per_100k_low"], - data_high=data_tb_who["incidence_per_100k_high"], -) - -# plt.savefig(make_graph_file_name("TB_Incidence")) - -plt.show() - -# ---------------------------------------------------------------------- # - -# proportion TB cases that are MDR - -make_plot( - title_str="Proportion of active TB cases that are MDR", - model=model_tb_mdr["mean"], - model_low=model_tb_mdr["lower"], - model_high=model_tb_mdr["upper"], -) -# data from ResourceFile_TB sheet WHO_mdrTB2017 -plt.errorbar(model_tb_mdr.index[7], 0.0075, yerr=[[0.0059], [0.0105]], fmt="o") -plt.legend(["TLO", "", "WHO"]) -# plt.ylim((0, 15)) -# plt.savefig(make_graph_file_name("Proportion_TB_Cases_MDR")) -plt.show() - -# ---------------------------------------------------------------------- # - -# proportion TB cases that are HIV+ -# expect around 60% falling to 50% by 2017 - -make_plot( - title_str="Proportion of active cases that are HIV+", - model=model_tb_hiv_prop["mean"], - model_low=model_tb_hiv_prop["lower"], - model_high=model_tb_hiv_prop["upper"], -) -# plt.savefig(make_graph_file_name("Proportion_TB_Cases_MDR")) - -plt.show() - -# ---------------------------------------------------------------------- # -# -# # AIDS deaths (including HIV/TB deaths) -# make_plot( -# title_str="Mortality to HIV-AIDS per 100,000 capita", -# model=total_aids_deaths_rate_100kpy, -# model_low=total_aids_deaths_rate_100kpy_lower, -# model_high=total_aids_deaths_rate_100kpy_upper, -# data_name="UNAIDS", -# data_mid=data_hiv_unaids_deaths["AIDS_mortality_per_100k"], -# data_low=data_hiv_unaids_deaths["AIDS_mortality_per_100k_lower"], -# data_high=data_hiv_unaids_deaths["AIDS_mortality_per_100k_upper"], -# ) -# plt.savefig(make_graph_file_name("AIDS_mortality")) -# -# plt.show() - -# # ---------------------------------------------------------------------- # -# -# # AIDS/TB deaths -# make_plot( -# title_str="Mortality to HIV-AIDS-TB per 100,000 capita", -# model=total_aids_TB_deaths_rate_100kpy, -# model_low=total_aids_TB_deaths_rate_100kpy_lower, -# model_high=total_aids_TB_deaths_rate_100kpy_upper, -# data_name="WHO", -# data_mid=data_tb_who["mortality_tb_hiv_per_100k"], -# data_low=data_tb_who["mortality_tb_hiv_per_100k_low"], -# data_high=data_tb_who["mortality_tb_hiv_per_100k_high"], -# ) -# plt.savefig(make_graph_file_name("AIDS_TB_mortality")) -# -# plt.show() -# -# # ---------------------------------------------------------------------- # -# -# # TB deaths (excluding HIV/TB deaths) -# make_plot( -# title_str="TB mortality rate per 100,000 population", -# model=tot_tb_non_hiv_deaths_rate_100kpy, -# model_low=tot_tb_non_hiv_deaths_rate_100kpy_lower, -# model_high=tot_tb_non_hiv_deaths_rate_100kpy_upper, -# data_name="WHO", -# data_mid=data_tb_who["mortality_tb_excl_hiv_per_100k"], -# data_low=data_tb_who["mortality_tb_excl_hiv_per_100k_low"], -# data_high=data_tb_who["mortality_tb_excl_hiv_per_100k_high"], -# ) -# plt.savefig(make_graph_file_name("TB_mortality")) -# -# plt.show() - -# ---------------------------------------------------------------------- # - -# TB treatment coverage -make_plot( - title_str="TB treatment coverage", - model=model_tb_tx["mean"] * 100, - model_low=model_tb_tx["lower"] * 100, - model_high=model_tb_tx["upper"] * 100, - data_name="NTP", - data_mid=data_tb_ntp["treatment_coverage"], -) -# plt.savefig(make_graph_file_name("TB_treatment_coverage")) - -plt.show() - -# ---------------------------------------------------------------------- # - -# HIV treatment coverage -make_plot( - title_str="HIV treatment coverage", - model=model_hiv_tx["mean"] * 100, - model_low=model_hiv_tx["lower"] * 100, - model_high=model_hiv_tx["upper"] * 100, - data_name="UNAIDS", - data_mid=data_hiv_unaids["ART_coverage_all_HIV_adults"], - data_low=data_hiv_unaids["ART_coverage_all_HIV_adults_lower"], - data_high=data_hiv_unaids["ART_coverage_all_HIV_adults_upper"], -) - -# plt.savefig(make_graph_file_name("HIV_treatment_coverage")) - -plt.show() - -# ---------------------------------------------------------------------- # -# %%: DEATHS - GBD COMPARISON -# ---------------------------------------------------------------------- # -# get numbers of deaths from model runs -results_deaths = extract_results( - results_folder, - module="tlo.methods.demography", - key="death", - custom_generate_series=( - lambda df: df.assign(year=df["date"].dt.year).groupby( - ["year", "cause"])["person_id"].count() - ), - do_scaling=True, -) - -results_deaths = results_deaths.reset_index() - -# results_deaths.columns.get_level_values(1) -# Index(['', '', 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2], dtype='object', name='run') -# -# results_deaths.columns.get_level_values(0) # this is higher level -# Index(['year', 'cause', 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3], dtype='object', name='draw') - -# AIDS deaths -# select cause of death -tmp = results_deaths.loc[ - (results_deaths.cause == "AIDS_TB") | (results_deaths.cause == "AIDS_non_TB") - ] -# select draw - drop columns where draw != 0, but keep year and cause -tmp2 = tmp.loc[ - :, ("draw" == draw) - ].copy() # selects only columns for draw=0 (removes year/cause) -# join year and cause back to df - needed for groupby -frames = [tmp["year"], tmp["cause"], tmp2] -tmp3 = pd.concat(frames, axis=1) - -# create new column names, dependent on number of runs in draw -base_columns = ["year", "cause"] -run_columns = ["run" + str(x) for x in range(0, info["runs_per_draw"])] -base_columns.extend(run_columns) -tmp3.columns = base_columns -tmp3 = tmp3.set_index("year") - -# sum rows for each year (2 entries) -# for each run need to combine deaths in each year, may have different numbers of runs -model_deaths_AIDS = pd.DataFrame(tmp3.groupby(["year"]).sum()) - -# double check all columns are float64 or quantile argument will fail -model_2010_median = model_deaths_AIDS.iloc[2].quantile(0.5) -model_2015_median = model_deaths_AIDS.iloc[5].quantile(0.5) -model_2010_low = model_deaths_AIDS.iloc[2].quantile(0.025) -model_2015_low = model_deaths_AIDS.iloc[5].quantile(0.025) -model_2010_high = model_deaths_AIDS.iloc[2].quantile(0.975) -model_2015_high = model_deaths_AIDS.iloc[5].quantile(0.975) - -# get GBD estimates from any log_filepath -outputpath = Path("./outputs") # folder for convenience of storing outputs -list_of_paths = outputpath.glob('*.log') # gets latest log file -latest_path = max(list_of_paths, key=lambda p: p.stat().st_ctime) -death_compare = compare_number_of_deaths(latest_path, resourcefilepath) - - -# include all ages and both sexes -deaths2010 = death_compare.loc[("2010-2014", slice(None), slice(None), "AIDS")].sum() -deaths2015 = death_compare.loc[("2015-2019", slice(None), slice(None), "AIDS")].sum() - -# include all ages and both sexes -deaths2010_TB = death_compare.loc[("2010-2014", slice(None), slice(None), "TB (non-AIDS)")].sum() -deaths2015_TB = death_compare.loc[("2015-2019", slice(None), slice(None), "TB (non-AIDS)")].sum() - -x_vals = [1, 2, 3, 4] -labels = ["2010-2014", "2010-2014", "2015-2019", "2015-2019"] -col = ["mediumblue", "mediumseagreen", "mediumblue", "mediumseagreen"] -# handles for legend -blue_patch = mpatches.Patch(color="mediumblue", label="GBD") -green_patch = mpatches.Patch(color="mediumseagreen", label="TLO") - -# plot AIDS deaths -y_vals = [ - deaths2010["GBD_mean"], - model_2010_median, - deaths2015["GBD_mean"], - model_2015_median, -] -y_lower = [ - abs(deaths2010["GBD_lower"] - deaths2010["GBD_mean"]), - abs(model_2010_low - model_2010_median), - abs(deaths2015["GBD_lower"] - deaths2015["GBD_mean"]), - abs(model_2015_low - model_2015_median), -] -y_upper = [ - abs(deaths2010["GBD_upper"] - deaths2010["GBD_mean"]), - abs(model_2010_high - model_2010_median), - abs(deaths2015["GBD_upper"] - deaths2015["GBD_mean"]), - abs(model_2015_high - model_2015_median), -] -plt.bar(x_vals, y_vals, color=col) -plt.errorbar( - x_vals, y_vals, - yerr=[y_lower, y_upper], - ls="none", - marker="o", - markeredgecolor="red", - markerfacecolor="red", - ecolor="red", -) -plt.xticks(ticks=x_vals, labels=labels) -plt.title("Deaths per year due to AIDS") -plt.legend(handles=[blue_patch, green_patch]) -plt.tight_layout() -# plt.savefig(make_graph_file_name("AIDS_deaths_with_GBD")) -plt.show() - -# ------------------------------------------------------------------------------------- - -# TB deaths -# select cause of death -tmp = results_deaths.loc[(results_deaths.cause == "TB")] -# select draw - drop columns where draw != 0, but keep year and cause -tmp2 = tmp.loc[ - :, ("draw" == draw) - ].copy() # selects only columns for draw=0 (removes year/cause) -# join year and cause back to df - needed for groupby -frames = [tmp["year"], tmp["cause"], tmp2] -tmp3 = pd.concat(frames, axis=1) - -# create new column names, dependent on number of runs in draw -base_columns = ["year", "cause"] -run_columns = ["run" + str(x) for x in range(0, info["runs_per_draw"])] -base_columns.extend(run_columns) -tmp3.columns = base_columns -tmp3 = tmp3.set_index("year") - -# sum rows for each year (2 entries) -# for each run need to combine deaths in each year, may have different numbers of runs -model_deaths_TB = pd.DataFrame(tmp3.groupby(["year"]).sum()) - -# double check all columns are float64 or quantile argument will fail -model_2010_median = model_deaths_TB.iloc[2].quantile(0.5) -model_2015_median = model_deaths_TB.iloc[5].quantile(0.5) -model_2010_low = model_deaths_TB.iloc[2].quantile(0.025) -model_2015_low = model_deaths_TB.iloc[5].quantile(0.025) -model_2010_high = model_deaths_TB.iloc[2].quantile(0.975) -model_2015_high = model_deaths_TB.iloc[5].quantile(0.975) - -x_vals = [1, 2, 3, 4, 5, 6] -labels = ["2010-2014", "2010-2014", "2010-2014", "2015-2019", "2015-2019", "2015-2019"] -col = ["mediumblue", "mediumseagreen", "red", "mediumblue", "mediumseagreen", "red"] -# handles for legend -blue_patch = mpatches.Patch(color="mediumblue", label="GBD") -green_patch = mpatches.Patch(color="mediumseagreen", label="WHO") -red_patch = mpatches.Patch(color="red", label="TLO") - - -# plot TB deaths -y_vals = [ - deaths2010_TB["GBD_mean"], - deaths_2010_2014_average, - model_2010_median, - deaths2015_TB["GBD_mean"], - deaths_2015_2019_average, - model_2015_median, -] -y_lower = [ - abs(deaths2010_TB["GBD_lower"] - deaths2010_TB["GBD_mean"]), - deaths_2010_2014_average_low, - abs(model_2010_low - model_2010_median), - abs(deaths2015_TB["GBD_lower"] - deaths2015_TB["GBD_mean"]), - deaths_2015_2019_average_low, - abs(model_2015_low - model_2015_median), -] -y_upper = [ - abs(deaths2010_TB["GBD_upper"] - deaths2010_TB["GBD_mean"]), - deaths_2010_2014_average_high, - abs(model_2010_high - model_2010_median), - abs(deaths2015_TB["GBD_upper"] - deaths2015_TB["GBD_mean"]), - deaths_2015_2019_average_high, - abs(model_2015_high - model_2015_median), -] - -plt.bar(x_vals, y_vals, color=col) -plt.errorbar( - x_vals, y_vals, - yerr=[y_lower, y_upper], - ls="none", - marker="o", - markeredgecolor="lightskyblue", - markerfacecolor="lightskyblue", - ecolor="lightskyblue", -) -plt.xticks(ticks=x_vals, labels=labels) -plt.title("Deaths per year due to TB") -plt.legend(handles=[blue_patch, green_patch, red_patch]) -plt.tight_layout() -# plt.savefig(make_graph_file_name("TB_deaths_with_GBD")) -plt.show() diff --git a/src/scripts/impact_of_historical_changes_in_hr/analysis_historical_changes_in_hr.py b/src/scripts/impact_of_historical_changes_in_hr/analysis_historical_changes_in_hr.py new file mode 100644 index 0000000000..affa7c8603 --- /dev/null +++ b/src/scripts/impact_of_historical_changes_in_hr/analysis_historical_changes_in_hr.py @@ -0,0 +1,436 @@ +"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different +scenarios (scenario_impact_of_healthsystem.py)""" + +import argparse +import textwrap +from pathlib import Path +from typing import Tuple + +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt + +from scripts.impact_of_historical_changes_in_hr.scenario_historical_changes_in_hr import ( + HistoricalChangesInHRH, +) +from tlo import Date +from tlo.analysis.utils import extract_results, make_age_grp_lookup, summarize + + +def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None, the_target_period: Tuple[Date, Date] = None): + """Produce standard set of plots describing the effect of each TREATMENT_ID. + - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur. + - We estimate the draw on healthcare system resources as the FEWER appointments when that treatment does not occur. + """ + + TARGET_PERIOD = the_target_period + + # Definitions of general helper functions + make_graph_file_name = lambda stub: output_folder / f"{stub.replace('*', '_star_')}.png" # noqa: E731 + + _, age_grp_lookup = make_age_grp_lookup() + + def target_period() -> str: + """Returns the target period as a string of the form YYYY-YYYY""" + return "-".join(str(t.year) for t in TARGET_PERIOD) + + def get_parameter_names_from_scenario_file() -> Tuple[str]: + """Get the tuple of names of the scenarios from `Scenario` class used to create the results.""" + e = HistoricalChangesInHRH() + return tuple(e._scenarios.keys()) + + def get_num_deaths(_df): + """Return total number of Deaths (total within the TARGET_PERIOD)""" + return pd.Series(data=len(_df.loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)])) + + def get_num_dalys(_df): + """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD). + Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using + results from runs that crashed mid-way through the simulation. + """ + years_needed = [i.year for i in TARGET_PERIOD] + assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded." + return pd.Series( + data=_df + .loc[_df.year.between(*years_needed)] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum().sum() + ) + + def set_param_names_as_column_index_level_0(_df): + """Set the columns index (level 0) as the param_names.""" + ordered_param_names_no_prefix = {i: x for i, x in enumerate(param_names)} + names_of_cols_level0 = [ordered_param_names_no_prefix.get(col) for col in _df.columns.levels[0]] + assert len(names_of_cols_level0) == len(_df.columns.levels[0]) + _df.columns = _df.columns.set_levels(names_of_cols_level0, level=0) + return _df + + def find_difference_relative_to_comparison_series( + _ser: pd.Series, + comparison: str, + scaled: bool = False, + drop_comparison: bool = True, + ): + """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0) + within the runs (level 1), relative to where draw = `comparison`. + The comparison is `X - COMPARISON`.""" + return _ser \ + .unstack(level=0) \ + .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \ + .drop(columns=([comparison] if drop_comparison else [])) \ + .stack() + + def find_difference_relative_to_comparison_series_dataframe(_df: pd.DataFrame, **kwargs): + """Apply `find_difference_relative_to_comparison_series` to each row in a dataframe""" + return pd.concat({ + _idx: find_difference_relative_to_comparison_series(row, **kwargs) + for _idx, row in _df.iterrows() + }, axis=1).T + + def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False, put_labels_in_legend=True): + """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the + extent of the error bar.""" + + substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + yerr = np.array([ + (_df['mean'] - _df['lower']).values, + (_df['upper'] - _df['mean']).values, + ]) + + xticks = {(i + 0.5): k for i, k in enumerate(_df.index)} + + # Define colormap (used only with option `put_labels_in_legend=True`) + cmap = plt.get_cmap("tab20") + rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731 + colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None + + fig, ax = plt.subplots(figsize=(10, 5)) + ax.bar( + xticks.keys(), + _df['mean'].values, + yerr=yerr, + alpha=0.8, + ecolor='black', + color=colors, + capsize=10, + label=xticks.values(), + zorder=100, + ) + if annotations: + for xpos, ypos, text in zip(xticks.keys(), _df['upper'].values, annotations): + ax.text(xpos, ypos*1.15, text, horizontalalignment='center', rotation='vertical', fontsize='x-small') + ax.set_xticks(list(xticks.keys())) + + if put_labels_in_legend: + # Update xticks label with substitute labels + # Insert legend with updated labels that shows correspondence between substitute label and original label + xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())] + xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())] + h, _ = ax.get_legend_handles_labels() + ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5)) + ax.set_xticklabels(list(xtick_values)) + else: + if not xticklabels_horizontal_and_wrapped: + # xticklabels will be vertical and not wrapped + ax.set_xticklabels(list(xticks.values()), rotation=90) + else: + wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()] + ax.set_xticklabels(wrapped_labs) + + ax.grid(axis="y") + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + fig.tight_layout() + + return fig, ax + + # %% Define parameter names + param_names = get_parameter_names_from_scenario_file() + counterfactual_scenario = 'Counterfactual (No Scale-up)' + actual_scenario = 'Actual (Scale-up)' + + # %% Quantify the health gains associated with all interventions combined. + + # Absolute Number of Deaths and DALYs + num_deaths = extract_results( + results_folder, + module='tlo.methods.demography', + key='death', + custom_generate_series=get_num_deaths, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + num_dalys = extract_results( + results_folder, + module='tlo.methods.healthburden', + key='dalys_stacked', + custom_generate_series=get_num_dalys, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + # %% Charts of total numbers of deaths / DALYS + num_dalys_summarized = summarize(num_dalys).loc[0].unstack().reindex(param_names) + num_deaths_summarized = summarize(num_deaths).loc[0].unstack().reindex(param_names) + + name_of_plot = f'Deaths, {target_period()}' + fig, ax = do_bar_plot_with_ci(num_deaths_summarized / 1e6, xticklabels_horizontal_and_wrapped=True, put_labels_in_legend=False) + ax.set_title(name_of_plot) + ax.set_ylabel('(Millions)') + fig.tight_layout() + ax.axhline(num_deaths_summarized.loc[counterfactual_scenario, 'mean']/1e6, color='black', alpha=0.5) + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + name_of_plot = f'DALYs, {target_period()}' + fig, ax = do_bar_plot_with_ci(num_dalys_summarized / 1e6, xticklabels_horizontal_and_wrapped=True, put_labels_in_legend=False) + ax.set_title(name_of_plot) + ax.set_ylabel('(Millions)') + ax.axhline(num_dalys_summarized.loc[counterfactual_scenario, 'mean']/1e6, color='black', alpha=0.5) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + + # %% Deaths and DALYS averted relative to Counterfactual + num_deaths_averted = summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_deaths.loc[0], + comparison=counterfactual_scenario) + ).T + ).iloc[0].unstack().reindex(param_names).drop([counterfactual_scenario]) + + pc_deaths_averted = 100.0 * summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_deaths.loc[0], + comparison=counterfactual_scenario, + scaled=True) + ).T + ).iloc[0].unstack().reindex(param_names).drop([counterfactual_scenario]) + + num_dalys_averted = summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_dalys.loc[0], + comparison=counterfactual_scenario) + ).T + ).iloc[0].unstack().reindex(param_names).drop([counterfactual_scenario]) + + pc_dalys_averted = 100.0 * summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_dalys.loc[0], + comparison=counterfactual_scenario, + scaled=True) + ).T + ).iloc[0].unstack().reindex(param_names).drop([counterfactual_scenario]) + + # DEATHS + name_of_plot = f'Deaths Averted vs Counterfactual, {target_period()}' + fig, ax = do_bar_plot_with_ci( + num_deaths_averted.clip(lower=0.0), + annotations=None, + put_labels_in_legend=False, + xticklabels_horizontal_and_wrapped=True, + ) + annotation = (f"{int(round(num_deaths_averted.loc[actual_scenario,'mean'], -3))} ({int(round(num_deaths_averted.loc[actual_scenario, 'lower'], -3))} - {int(round(num_deaths_averted.loc[actual_scenario,'upper'], -3))})\n" + f"{round(pc_deaths_averted.loc[actual_scenario, 'mean'])} ({round(pc_deaths_averted.loc[actual_scenario,'lower'], 1)} - {round(pc_deaths_averted.loc[actual_scenario, 'upper'], 1)})% of that in Counterfactual" + ) + ax.set_title(f"{name_of_plot}\n{annotation}") + ax.set_ylabel('Deaths Averted vs Counterfactual') + fig.set_figwidth(5) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + # DALYS + name_of_plot = f'DALYs Averted vs Counterfactual, {target_period()}' + fig, ax = do_bar_plot_with_ci( + (num_dalys_averted / 1e6).clip(lower=0.0), + annotations=None, + put_labels_in_legend=False, + xticklabels_horizontal_and_wrapped=True, + ) + annotation = (f"{int(round(num_dalys_averted.loc[actual_scenario,'mean'], -4))} ({int(round(num_dalys_averted.loc[actual_scenario, 'lower'], -4))} - {int(round(num_dalys_averted.loc[actual_scenario,'upper'], -4))})\n" + f"{round(pc_dalys_averted.loc[actual_scenario, 'mean'])} ({round(pc_dalys_averted.loc[actual_scenario,'lower'], 1)} - {round(pc_dalys_averted.loc[actual_scenario, 'upper'], 1)})% of that in Counterfactual" + ) + ax.set_title(f"{name_of_plot}\n{annotation}") + ax.set_ylabel('DALYS Averted \n(Millions)') + fig.set_figwidth(5) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + # Graphs showing difference by disease (HTM/OTHER and split by age/sex) + def get_total_num_dalys_by_label_htm(_df): + """Return the total number of DALYS in the TARGET_PERIOD by wealth and cause label.""" + y = _df \ + .loc[_df['year'].between(*[d.year for d in TARGET_PERIOD])] \ + .drop(columns=['date', 'year', 'sex', 'age_range']) \ + .sum(axis=0) + + # define course cause mapper for HIV, TB, MALARIA and OTHER + causes = { + 'AIDS': 'HIV/AIDS', + 'TB (non-AIDS)': 'TB', + 'Malaria': 'Malaria', + 'Lower respiratory infections': 'Lower respiratory infections', + 'Neonatal Disorders': 'Neonatal Disorders', + 'Maternal Disorders': 'Maternal Disorders', + '': 'Other', # defined in order to use this dict to determine ordering of the causes in output + } + causes_relabels = y.index.map(causes).fillna('Other') + + return y.groupby(by=causes_relabels).sum()[list(causes.values())] + + total_num_dalys_by_label_results = extract_results( + results_folder, + module="tlo.methods.healthburden", + key="dalys_stacked_by_age_and_time", + custom_generate_series=get_total_num_dalys_by_label_htm, + do_scaling=True, + ).pipe(set_param_names_as_column_index_level_0) + + total_num_dalys_by_label_results_averted_vs_baseline = summarize( + -1.0 * find_difference_relative_to_comparison_series_dataframe( + total_num_dalys_by_label_results, + comparison=counterfactual_scenario, + ), + only_mean=True + ) + + # Check that when we sum across the causes, we get the same total as calculated when we didn't split by cause. + assert ( + (total_num_dalys_by_label_results_averted_vs_baseline.sum(axis=0).sort_index() + - num_dalys_averted['mean'].sort_index() + ) < 1e-6 + ).all() + + yerr = np.array([ + (num_dalys_averted['mean'].values - num_dalys_averted['lower']).values, + (num_dalys_averted['upper'].values - num_dalys_averted['mean']).values, + ])/1e6 + + make_string_number = lambda row: f"{round(row['mean']/1e6,1)} ({round(row['lower']/1e6, 1)}-{round(row['upper']/1e6, 1)}) Million" # noqa: E731 + str_num_dalys_averted = f'{make_string_number(num_dalys_averted.loc[actual_scenario])}' + + make_string_percent = lambda row: f"{round(row['mean'], 1)} ({round(row['lower'], 1)}-{round(row['upper'], 1)})" # noqa: E731 + str_pc_dalys_averted = f'{make_string_percent(pc_dalys_averted.loc[actual_scenario])}% of DALYS in Counterfactual' # noqa: E731 + + def make_daly_split_by_cause_graph(df: pd.DataFrame, filename_suffix: str): + name_of_plot = f'DALYS Averted: Actual vs Counterfactual, {target_period()}' + fig, ax = plt.subplots() + (df.iloc[::-1] /1e6).T.plot.bar( + stacked=True, + ax=ax, + rot=0, + alpha=0.75, + zorder=3, + legend=False, + # color=['orange', 'teal', 'purple', 'red'] + ) + ax.errorbar(0, num_dalys_averted['mean'].values/1e6, yerr=yerr, fmt="o", color="black", zorder=4) + ax.set_title(name_of_plot + '\n' + str_num_dalys_averted + '\n' + str_pc_dalys_averted) + ax.set_ylabel('DALYs Averted\n(Millions)') + ax.set_xlabel('') + ax.set_xlim(-0.5, 0.65) + ax.set_ylim(bottom=0) + ax.get_xaxis().set_ticks([]) + wrapped_labs = ["\n".join(textwrap.wrap(_lab.get_text(), 20)) for _lab in ax.get_xticklabels()] + ax.set_xticklabels(wrapped_labs) + ax.grid(axis='y', zorder=0) + handles, labels = ax.get_legend_handles_labels() + ax.legend(handles[::-1], labels[::-1], title='Cause of DALYS', loc='center right') + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '') + filename_suffix)) + fig.show() + plt.close(fig) + + # Make graph - separating H/T/M/Other + make_daly_split_by_cause_graph(total_num_dalys_by_label_results_averted_vs_baseline, filename_suffix='_by_htm') + + # Make graph - separating HTM-Combined/Other + total_num_dalys_by_label_results_averted_vs_baseline_grouping_htm = total_num_dalys_by_label_results_averted_vs_baseline.groupby( + total_num_dalys_by_label_results_averted_vs_baseline.index == 'Other').sum().rename( + index={False: "H/T/M", True: "Other"}) + make_daly_split_by_cause_graph(total_num_dalys_by_label_results_averted_vs_baseline_grouping_htm, filename_suffix='_broad') + + # percent of DALYS averted in H/T/M + pc_dalys_averted_in_h_t_m = (100 * (total_num_dalys_by_label_results_averted_vs_baseline / total_num_dalys_by_label_results_averted_vs_baseline.sum())).round(0) + + def plot_table(df, name_of_plot): + fig, ax = plt.subplots(dpi=600) + ax.axis('off') + pd.plotting.table(ax, df, loc='center', cellLoc='center', colWidths=list([.2, .2])) + ax.set_title(name_of_plot) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + plot_table(pc_dalys_averted_in_h_t_m, name_of_plot=f'Breakdown of DALYS Averted: {target_period()}') + + # percent of DALYS averted in HTM (combined) + pc_dalys_averted_in_htm = 1.0 - (total_num_dalys_by_label_results_averted_vs_baseline.loc['Other'] / total_num_dalys_by_label_results_averted_vs_baseline.sum()) + print(f'pc_dalys_averted_in_htm ({the_target_period}): {pc_dalys_averted_in_htm[actual_scenario]}') + + + #%% Breakdown of causes: Find the top 5 causes averted other than HTM + def get_total_num_dalys_by_label_all_causes(_df): + """Return the total number of DALYS in the TARGET_PERIOD cause label.""" + return _df \ + .loc[_df['year'].between(*[d.year for d in TARGET_PERIOD])] \ + .drop(columns=['date', 'year', 'age_range', 'sex']) \ + .sum(axis=0) + + total_num_dalys_by_label_results_all_causes = extract_results( + results_folder, + module="tlo.methods.healthburden", + key="dalys_stacked_by_age_and_time", + custom_generate_series=get_total_num_dalys_by_label_all_causes, + do_scaling=True, + ).pipe(set_param_names_as_column_index_level_0) + + total_num_dalys_by_label_results_averted_vs_baseline_all_causes = summarize( + -1.0 * find_difference_relative_to_comparison_series_dataframe( + total_num_dalys_by_label_results_all_causes, + comparison=counterfactual_scenario, + ), + only_mean=True + ) + + top_three_causes = list(total_num_dalys_by_label_results_averted_vs_baseline_all_causes[actual_scenario] + .drop(index={'AIDS', 'TB (non-AIDS)', 'Malaria', 'Other'}) + .sort_values(ascending=False).head(3).keys()) + print(f"Top 3 causes of DALYS Averted in Other during {TARGET_PERIOD}: {top_three_causes}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("results_folder", type=Path) # outputs/horizontal_and_vertical_programs-2024-05-16 + args = parser.parse_args() + + # Produce results for short-term analysis - 2020 - 2024 (incl.) + apply( + results_folder=args.results_folder, + output_folder=args.results_folder, + resourcefilepath=Path('./resources'), + the_target_period=(Date(2020, 1, 1), Date(2024, 12, 31)) + ) + # Produce results for only later period 2025-2030 (incl.) + apply( + results_folder=args.results_folder, + output_folder=args.results_folder, + resourcefilepath=Path('./resources'), + the_target_period=(Date(2025, 1, 1), Date(2030, 12, 31)) + ) diff --git a/src/scripts/impact_of_historical_changes_in_hr/examining_data_historic_changes_in_hr.py b/src/scripts/impact_of_historical_changes_in_hr/examining_data_historic_changes_in_hr.py new file mode 100644 index 0000000000..b018fb3513 --- /dev/null +++ b/src/scripts/impact_of_historical_changes_in_hr/examining_data_historic_changes_in_hr.py @@ -0,0 +1,169 @@ +import datetime +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from scipy.optimize import curve_fit + +from tlo.analysis.utils import get_root_path + +# Path to shared folder +path_to_share = Path( # <-- point to the shared folder + '/Users/tbh03/Library/CloudStorage/OneDrive-SharedLibraries-ImperialCollegeLondon/TLOModel - WP - Documents/' +) + + +#%% Numbers employed in HRH (As provided by Dominic Nkhoma: Email 13/8/14) + +df = pd.read_excel( + path_to_share / '07 - Data' / 'Historical_Changes_in_HR' / '03' / 'Malawi MOH Yearly_Employees_Data_Updated.xlsx', + sheet_name='Sheet1' +) +num_employees = df.set_index(['District', 'Month', 'Year'])['Emp_Totals'] + +# Find number of employees each year, using the count in March. (This gives the identical values to that quoted +# by Dominic in his email; i.e., +# year_by_year = pd.Series({ +# 2017: 24863, +# 2018: 24156, +# 2019: 25994, +# 2020: 24763, +# 2021: 28737, +# 2022: 29570, +# 2023: 31304, +# 2024: 34486, +# }) +year_by_year = num_employees.loc[(slice(None), 'March', slice(None))].groupby(by='Year').sum().astype(int) + + +# Plot trend overall +fig, ax = plt.subplots() +year_by_year.plot(ax=ax, legend=False, marker='o') +ax.set_title('Trend in Healthcare Workers', fontweight='bold', fontsize=10) +ax.set_ylabel('Number of HCW') +ax.set_ylim(0, 40_000) +ax.set_xlim(2016, 2025) +fig.tight_layout() +fig.show() + +# difference vs 2017 +diff_since_2017 = year_by_year - year_by_year.at[2017] + + +# Plot trend for the different districts +fig, ax = plt.subplots(figsize=(6, 4), layout='constrained') +num_employees.groupby(by=['Year', 'District']).mean().unstack().plot(ax=ax, legend=False, marker='.') +ax.set_title('Trend in Healthcare Workers by District', fontweight='bold', fontsize=10) +ax.set_ylabel('Number of HCW') +ax.set_ylim([0, 5_000]) +fig.legend(loc="outside lower center", ncols=5, fontsize='small') +fig.show() + + +# %% Curve-fitting to the scale-up + +def func(y, beta, ystart): + return np.exp(beta * (y - ystart - 2017).clip(0.0)) + +popt, pcov = curve_fit(func, + year_by_year.index.to_numpy(), + year_by_year.to_numpy() / year_by_year[2017], + ) + +plt.figure() +plt.plot(year_by_year.index.to_numpy(), year_by_year.to_numpy() / year_by_year[2017], marker='o', label='Historical data') +plt.plot(year_by_year.index.to_numpy(), func(year_by_year.index.to_numpy(), *popt), label='fit') +plt.show() + + +#%% Plot to explain setup of Scenario + +to_plot = pd.DataFrame(index=pd.Index(range(2017, 2031), name='year')) + +to_plot['Data'] = year_by_year / year_by_year[2017] # data is the year-on-year trend, normalised to 2017 + +# Assign the date of mid-year to the data points +to_plot['mid-year_date'] = pd.to_datetime(dict(year=to_plot.index, month=7, day=1)).dt.date.values + +# Define scale-up pattern: fitted line +to_plot['Scale-up'] = pd.Series(index=year_by_year.index.to_numpy(), data=func(year_by_year.index.to_numpy(), *popt)) + +# Define counterfactual scenario +to_plot['No Scale-up'] = 1.0 + +# Actual and Counterfactual are held to the last level achieved in the data when we go forward +to_plot['Scale-up'] = to_plot['Scale-up'].ffill() +to_plot['No Scale-up'] = to_plot['No Scale-up'].ffill() + +# For plotting the scenarios, we'll show that the changes happen at the start of the year. +step_dates = [datetime.date(y, 1, 1) for y in to_plot.index] + [datetime.date(to_plot.index.max() + 1, 1, 1)] + +for xlim in (datetime.date(2025, 1, 1), datetime.date(2031, 1, 1)): + fig, ax = plt.subplots() + ax.stairs( # line for the actual scenario + values=to_plot['Scale-up'], + edges=step_dates, baseline=None, + label='Scale-up Actual Scenario', + color='r', + zorder=2, + linewidth=3) + ax.stairs( # the shading between the actual and counterfactual scenarios + values=to_plot['Scale-up'], + edges=step_dates, + baseline=1.0, + label=None, + color='r', + zorder=2, + fill=True, + alpha=0.3) + ax.stairs( # line for the counterfactual scenario + values=to_plot['No Scale-up'], + edges=step_dates, baseline=None, + label='No Scale-up Counterfactual', + color='g', + zorder=3, + linewidth=3) + ax.plot( # the data + to_plot['mid-year_date'], + to_plot['Data'], + marker='o', + linestyle='--', + label='Data') + ax.set_title('Change in the Number of Healthcare Workers') + ax.set_ylabel('Number of Staff\n(Normalised to 2017)') + ax.legend(loc='upper left') + ax.grid() + ax.set_ylim(0.95, 1.6) + ax.set_xlabel('Date') + xtickrange = pd.date_range(datetime.date(2017, 1, 1), xlim, freq='YS', inclusive='both') + ax.set_xlim(xtickrange.min(), xtickrange.max()) + ax.set_xticks(xtickrange) + ax.set_xticklabels(xtickrange.year, rotation=90) + fig.tight_layout() + fig.show() + + + +#%% Save this as a scale-up scenario + +# Work-out the annual multipliers that will give the desired scale-up pattern +scale_up_multipliers = dict() +scale_up_multipliers[2010] = 1.0 +for idx, val in to_plot['Scale-up'].sort_index().items(): + if idx-1 > to_plot['Scale-up'].index[0]: + scale_up_multipliers[idx] = val / to_plot.loc[idx-1, 'Scale-up'] + + +scale_up_scenario = pd.DataFrame({'dynamic_HR_scaling_factor': pd.Series(scale_up_multipliers)}) +scale_up_scenario['scale_HR_by_popsize'] = ["FALSE"] * len(scale_up_scenario) +scale_up_scenario = scale_up_scenario.reset_index() +scale_up_scenario = scale_up_scenario.rename(columns={'index': 'year'}) +scale_up_scenario['year'] = scale_up_scenario['year'].astype(int) +scale_up_scenario.sort_values('year', inplace=True) + +# Add (or over-write) a sheet called 'historical_scaling' with the scale-up pattern to the relevant ResourceFile +target_file = get_root_path() / 'resources' / 'healthsystem' / 'human_resources' / 'scaling_capabilities' / 'ResourceFile_dynamic_HR_scaling.xlsx' + +with pd.ExcelWriter(target_file, engine='openpyxl', mode='a', if_sheet_exists="replace") as writer: + scale_up_scenario.to_excel(writer, sheet_name='historical_scaling', index=False) diff --git a/src/scripts/impact_of_historical_changes_in_hr/scenario_historical_changes_in_hr.py b/src/scripts/impact_of_historical_changes_in_hr/scenario_historical_changes_in_hr.py new file mode 100644 index 0000000000..0f936a0c33 --- /dev/null +++ b/src/scripts/impact_of_historical_changes_in_hr/scenario_historical_changes_in_hr.py @@ -0,0 +1,110 @@ +"""This Scenario file run the model under different assumptions for the historical changes in Human Resources for Health + +Run on the batch system using: +``` +tlo batch-submit src/scripts/impact_of_historical_changes_in_hr/scenario_historical_changes_in_hr.py +``` + +""" + +from pathlib import Path +from typing import Dict + +from tlo import Date, logging +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.methods.fullmodel import fullmodel +from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher +from tlo.scenario import BaseScenario + + +class HistoricalChangesInHRH(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2031, 1, 1) # <-- End at the end of year 2030 + self.pop_size = 20_000 + self._scenarios = self._get_scenarios() + self.number_of_draws = len(self._scenarios) + self.runs_per_draw = 10 + + def log_configuration(self): + return { + 'filename': 'historical_changes_in_hr', + 'directory': Path('./outputs'), + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.demography.detail': logging.WARNING, + 'tlo.methods.healthburden': logging.INFO, + 'tlo.methods.healthsystem': logging.WARNING, + 'tlo.methods.healthsystem.summary': logging.INFO, + } + } + + def modules(self): + return ( + fullmodel(resourcefilepath=self.resources) + + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + ) + + def draw_parameters(self, draw_number, rng): + if draw_number < len(self._scenarios): + return list(self._scenarios.values())[draw_number] + + def _get_scenarios(self) -> Dict[str, Dict]: + """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario.""" + + return { + "Actual (Scale-up)": + mix_scenarios( + self._common_baseline(), + { + "HealthSystem": { + # SCALE-UP IN HRH + 'yearly_HR_scaling_mode': 'historical_scaling', + # Scale-up pattern defined from examining the data + } + } + ), + + "Counterfactual (No Scale-up)": + mix_scenarios( + self._common_baseline(), + { + "HealthSystem": { + # NO CHANGE IN HRH EVER + 'yearly_HR_scaling_mode': 'no_scaling', + } + } + ), + } + + def _common_baseline(self) -> Dict: + return mix_scenarios( + get_parameters_for_status_quo(), + { + "HealthSystem": { + "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration + "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH + "scale_to_effective_capabilities": True, # <-- Transition into Mode2 with the effective capabilities in HRH 'revealed' in Mode 1 + "year_mode_switch": 2020, # <-- transition happens at start of 2020 when HRH starts to grow + + # Normalize the behaviour of Mode 2 + "policy_name": "EHP_III", # -- *For the alternative scenario of efficient implementation of EHP, otherwise use 'naive'* -- + "tclose_overwrite": 1, + "tclose_days_offset_overwrite": 7, + }, + # -- *For the alternative scenario of increased demand and improved clinician performance* -- + 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { + 'max_healthcare_seeking': [False, True], # <-- switch from False to True mid-way + 'max_healthsystem_function': [False, True], + 'year_of_switch': 2020, + } + }, + ) + + +if __name__ == '__main__': + from tlo.cli import scenario_run + scenario_run([__file__]) diff --git a/src/scripts/malaria/analysis_malaria.py b/src/scripts/malaria/analysis_malaria.py index b2b4217dc6..f74cab548f 100644 --- a/src/scripts/malaria/analysis_malaria.py +++ b/src/scripts/malaria/analysis_malaria.py @@ -35,7 +35,7 @@ start_date = Date(2010, 1, 1) end_date = Date(2014, 1, 1) -popsize = 100 +popsize = 1000 # set up the log config diff --git a/src/scripts/profiling/run_profiling.py b/src/scripts/profiling/run_profiling.py index 180a7571ab..caca37ed50 100644 --- a/src/scripts/profiling/run_profiling.py +++ b/src/scripts/profiling/run_profiling.py @@ -306,7 +306,6 @@ def run_profiling( timeline=False, color=True, flat=True, - flat_time="total", processor_options={"show_regex": ".*/tlo/.*", "hide_regex": ".*/pandas/.*", "filter_threshold": 1e-3} ) converter = Ansi2HTMLConverter(title=output_name) diff --git a/src/scripts/tb/analysis_tb.py b/src/scripts/tb/analysis_tb.py index fd89fdc196..7d4ddfaef2 100644 --- a/src/scripts/tb/analysis_tb.py +++ b/src/scripts/tb/analysis_tb.py @@ -137,7 +137,7 @@ # # ------------------------------------- DATA ------------------------------------- # # # import HIV data # aidsInfo_data = pd.read_excel( -# Path(resourcefilepath) / "ResourceFile_HIV.xlsx", sheet_name="aids_info", +# Path(resourcefilepath) / "ResourceFile_HIV/parameters.csv", sheet_name="aids_info", # ) # # data_years = pd.to_datetime(aidsInfo_data.year, format="%Y") diff --git a/src/scripts/tb/output_plots_tb.py b/src/scripts/tb/output_plots_tb.py index c86aafa59f..7d875e3fe9 100644 --- a/src/scripts/tb/output_plots_tb.py +++ b/src/scripts/tb/output_plots_tb.py @@ -77,7 +77,7 @@ def make_plot(model=None, data_mid=None, data_low=None, data_high=None, title_st data_tb_ntp = data_tb_ntp.drop(columns=["year"]) # HIV resourcefile -xls = pd.ExcelFile(resourcefilepath / "ResourceFile_HIV.xlsx") +xls = pd.ExcelFile(resourcefilepath / "ResourceFile_HIV/parameters.csv") # HIV UNAIDS data data_hiv_unaids = pd.read_excel(xls, sheet_name="unaids_infections_art2021") diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index fa6b1b17fc..ca37283014 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -11,7 +11,7 @@ from collections.abc import Mapping from pathlib import Path from types import MappingProxyType -from typing import Callable, Dict, Iterable, List, Optional, TextIO, Tuple, Union +from typing import Callable, Dict, Iterable, List, Literal, Optional, TextIO, Tuple, Union import git import matplotlib.colors as mcolors @@ -230,8 +230,18 @@ def load_pickled_dataframes(results_folder: Path, draw=0, run=0, name=None) -> d return output +def extract_draw_names(results_folder: Path) -> dict[int, str]: + """Returns dict keyed by the draw-number giving the 'draw-name' declared for that draw in the Scenario at + draw_names().""" + draws = [f for f in os.scandir(results_folder) if f.is_dir()] + return { + int(d.name): + load_pickled_dataframes(results_folder, d.name, 0, name="tlo.scenario")["tlo.scenario"]["draw_name"]["draw_name"].values[0] + for d in draws + } -def extract_params(results_folder: Path) -> Optional[pd.DataFrame]: + +def extract_params(results_folder: Path, use_draw_names: bool = False) -> Optional[pd.DataFrame]: """Utility function to get overridden parameters from scenario runs Returns dateframe summarizing parameters that change across the draws. It produces a dataframe with index of draw @@ -258,6 +268,11 @@ def extract_params(results_folder: Path) -> Optional[pd.DataFrame]: params.index.name = 'draw' params = params.rename(columns={'new_value': 'value'}) params = params.sort_index() + + if use_draw_names: + # use draw_names instead of draw_number in the index + draw_names = extract_draw_names(results_folder) + params.index = params.index.map(draw_names) return params except KeyError: @@ -343,36 +358,57 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: return _concat -def summarize(results: pd.DataFrame, only_mean: bool = False, collapse_columns: bool = False) -> pd.DataFrame: +def compute_summary_statistics( + results: pd.DataFrame, + central_measure: Literal["mean", "median"] = "median", + width_of_range: float = 0.95, + only_central: bool = False, + collapse_columns: bool = False, +) -> pd.DataFrame: """Utility function to compute summary statistics - Finds mean value and 95% interval across the runs for each draw. + Finds a central value and a specified interval across the runs for each draw. By default, this uses a central + measure of the median and a 95% interval range. + + :param results: The dataframe of results to compute summary statistics of. + :param central_measure: The name of the central measure to use - either 'mean' or 'median'. + :param width_of_range: The width of the range to compute the statistics (e.g. 0.95 for the 95% interval). + :param collapse_columns: Whether to simplify the columnar index if there is only one run (cannot be done otherwise). + :param only_central: Whether to only report the central value (dropping the range). + :return: A dataframe with computed summary statistics. + """ + stats = dict() + grouped_results = results.groupby(axis=1, by='draw', sort=False) - summary = pd.concat( - { - 'mean': results.groupby(axis=1, by='draw', sort=False).mean(), - 'lower': results.groupby(axis=1, by='draw', sort=False).quantile(0.025), - 'upper': results.groupby(axis=1, by='draw', sort=False).quantile(0.975), - }, - axis=1 - ) + if central_measure == 'mean': + stats['central'] = grouped_results.mean() + elif central_measure == 'median': + stats['central'] = grouped_results.median() + else: + raise ValueError(f"Unknown stat: {central_measure}") + + lower_quantile = (1. - width_of_range) / 2. + stats["lower"] = grouped_results.quantile(lower_quantile) + stats["upper"] = grouped_results.quantile(1 - lower_quantile) + + summary = pd.concat(stats, axis=1) summary.columns = summary.columns.swaplevel(1, 0) summary.columns.names = ['draw', 'stat'] - summary = summary.sort_index(axis=1) + summary = summary.sort_index(axis=1).reindex(columns=['lower', 'central', 'upper'], level=1) - if only_mean and (not collapse_columns): - # Remove other metrics and simplify if 'only_mean' across runs for each draw is required: - om: pd.DataFrame = summary.loc[:, (slice(None), "mean")] - om.columns = [c[0] for c in om.columns.to_flat_index()] - om.columns.name = 'draw' - return om + if only_central and (not collapse_columns): + # Remove other metrics and simplify if 'only_central' across runs for each draw is required: + oc: pd.DataFrame = summary.loc[:, (slice(None), "central")] + oc.columns = [c[0] for c in oc.columns.to_flat_index()] + oc.columns.name = 'draw' + return oc elif collapse_columns and (len(summary.columns.levels[0]) == 1): # With 'collapse_columns', if number of draws is 1, then collapse columns multi-index: summary_droppedlevel = summary.droplevel('draw', axis=1) - if only_mean: - return summary_droppedlevel['mean'] + if only_central: + return summary_droppedlevel['central'] else: return summary_droppedlevel @@ -380,6 +416,36 @@ def summarize(results: pd.DataFrame, only_mean: bool = False, collapse_columns: return summary +def summarize( + results: pd.DataFrame, + only_mean: bool = False, + collapse_columns: bool = False +): + """Utility function to compute summary statistics + + Finds mean value and 95% interval across the runs for each draw. + + NOTE: This provides the legacy functionality of `summarize` that is hard-wired to use `means` (the kwarg is + `only_mean` and the name of the column in the output is `mean`). Please move to using the new and more flexible + version of `summarize` that allows the use of medians and is flexible to allow other forms of summary measure in + the future. + """ + warnings.warn( + "This function uses MEAN as the central measure. We now recommend using MEDIAN instead. " + "This can be done by using the function `compute_summary_statistics`." + "" + ) + output = compute_summary_statistics( + results=results, + central_measure='mean', + only_central=only_mean, + collapse_columns=collapse_columns, + ) + if output.columns.nlevels > 1: + output = output.rename(columns={'central': 'mean'}, level=1) # rename 'central' to 'mean' + return output + + def get_grid(params: pd.DataFrame, res: pd.Series): """Utility function to create the arrays needed to plot a heatmap. @@ -1203,6 +1269,51 @@ def get_parameters_for_standard_mode2_runs() -> Dict: } +def get_parameters_for_hrh_historical_scaling_and_rescaling_for_mode2() -> Dict: + """ + Returns a dictionary of parameters and their updated values to indicate + scenario runs that involve: + mode switch from 1 to 2 in 2020, + rescaling hrh capabilities to effective capabilities in the end of 2019 (the previous year of mode switch), + hrh historical scaling from 2020 to 2024. + + The return dict is in the form: + e.g. { + 'Depression': { + 'pr_assessed_for_depression_for_perinatal_female': 1.0, + 'pr_assessed_for_depression_in_generic_appt_level1': 1.0, + }, + 'Hiv': { + 'prob_start_art_or_vs': 1.0, + } + } + """ + + return { + "SymptomManager": { + "spurious_symptoms": True, + }, + "HealthSystem": { + 'Service_Availability': ['*'], + "use_funded_or_actual_staffing": "actual", + "mode_appt_constraints": 1, + "mode_appt_constraints_postSwitch": 2, + "year_mode_switch": 2020, # <-- Given that the data in HRH capabilities resource file are for year 2019 + # and that the model has been calibrated to data by 2019, we want the rescaling to effective capabilities + # to happen in the end of year 2019, which should be the previous year of mode switch to mode 2. + "scale_to_effective_capabilities": True, + 'yearly_HR_scaling_mode': 'historical_scaling', # <-- for 5 years of 2020-2024; the yearly historical + # scaling factor are stored in the sheet "historical_scaling" in ResourceFile_dynamic_HR_scaling. + "tclose_overwrite": 1, # <-- In most of our runs in mode 2, we chose to overwrite tclose + "tclose_days_offset_overwrite": 7, # <-- and usually set it to 7. + "cons_availability": "default", + "beds_availability": "default", + "equip_availability": "all", # <--- NB. Existing calibration is assuming all equipment is available + "policy_name": 'Naive', + }, + } + + def get_parameters_for_improved_healthsystem_and_healthcare_seeking( resourcefilepath: Path, max_healthsystem_function: Optional[bool] = False, diff --git a/src/tlo/cli.py b/src/tlo/cli.py index 6824dd1045..1404088d76 100644 --- a/src/tlo/cli.py +++ b/src/tlo/cli.py @@ -8,6 +8,7 @@ import tempfile from collections import defaultdict from pathlib import Path +from shutil import copytree from typing import Dict import click @@ -40,6 +41,7 @@ def cli(ctx, config_file, verbose): * submit scenarios to batch system * query batch system about job and tasks * download output results for completed job + * combine runs from multiple batch jobs with same draws """ ctx.ensure_object(dict) ctx.obj["config_file"] = config_file @@ -844,5 +846,69 @@ def add_tasks(batch_service_client, user_identity, job_id, batch_service_client.task.add_collection(job_id, tasks) -if __name__ == '__main__': +@cli.command() +@click.argument( + "output_results_directory", + type=click.Path(exists=True, file_okay=False, writable=True, path_type=Path), +) +@click.argument( + "additional_result_directories", + nargs=-1, + type=click.Path(exists=True, file_okay=False, path_type=Path), +) +def combine_runs(output_results_directory: Path, additional_result_directories: tuple[Path]) -> None: + """Combine runs from multiple batch jobs locally. + + Merges runs from each draw in one or more additional results directories in + to corresponding draws in output results directory. + + All results directories must contain same draw numbers and the draw numbers + must be consecutive integers starting from 0. All run numbers in the output + result directory draw directories must be consecutive integers starting + from 0. + """ + if len(additional_result_directories) == 0: + msg = "One or more additional results directories to merge must be specified" + raise click.UsageError(msg) + results_directories = (output_results_directory,) + additional_result_directories + draws_per_directory = [ + sorted( + int(draw_directory.name) + for draw_directory in results_directory.iterdir() + if draw_directory.is_dir() + ) + for results_directory in results_directories + ] + for draws in draws_per_directory: + if not draws == list(range(len(draws_per_directory[0]))): + msg = ( + "All results directories must contain same draws, " + "consecutively numbered from 0." + ) + raise click.UsageError(msg) + draws = draws_per_directory[0] + runs_per_draw = [ + sorted( + int(run_directory.name) + for run_directory in (output_results_directory / str(draw)).iterdir() + if run_directory.is_dir() + ) + for draw in draws + ] + for runs in runs_per_draw: + if not runs == list(range(len(runs))): + msg = "All runs in output directory must be consecutively numbered from 0." + raise click.UsageError(msg) + for results_directory in additional_result_directories: + for draw in draws: + run_counter = len(runs_per_draw[draw]) + for source_path in sorted((results_directory / str(draw)).iterdir()): + if not source_path.is_dir(): + continue + destination_path = output_results_directory / str(draw) / str(run_counter) + run_counter = run_counter + 1 + copytree(source_path, destination_path) + + +if __name__ == "__main__": cli(obj={}) diff --git a/src/tlo/lm.py b/src/tlo/lm.py index e099714850..a7538a1a7a 100644 --- a/src/tlo/lm.py +++ b/src/tlo/lm.py @@ -385,7 +385,7 @@ def predict( rng: Optional[np.random.RandomState] = None, squeeze_single_row_output=True, **kwargs - ) -> pd.Series: + ) -> Union[pd.Series, np.bool_]: """Evaluate linear model output for a given set of input data. :param df: The input ``DataFrame`` containing the input data to evaluate the @@ -396,7 +396,8 @@ def predict( output directly returned. :param squeeze_single_row_output: If ``rng`` argument is not ``None`` and this argument is set to ``True``, the output for a ``df`` input with a single-row - will be a scalar boolean value rather than a boolean ``Series``. + will be a scalar boolean value rather than a boolean ``Series``, if set to + ``False``, the output will always be a ``Series``. :param **kwargs: Values for any external variables included in model predictors. """ diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 69ce038299..dabe61e884 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -12,6 +12,7 @@ from tlo.methods.labour import LabourOnsetEvent from tlo.methods.malaria import HSI_MalariaIPTp from tlo.methods.tb import HSI_Tb_ScreeningAndRefer +from tlo.util import read_csv_files logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -176,8 +177,8 @@ def __init__(self, name=None, resourcefilepath=None): } def read_parameters(self, data_folder): - parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_AntenatalCare.xlsx', - sheet_name='parameter_values') + parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_AntenatalCare', + files='parameter_values') self.load_parameters_from_dataframe(parameter_dataframe) def initialise_population(self, population): diff --git a/src/tlo/methods/chronicsyndrome.py b/src/tlo/methods/chronicsyndrome.py index 0ae6599939..5a16e2b3ec 100644 --- a/src/tlo/methods/chronicsyndrome.py +++ b/src/tlo/methods/chronicsyndrome.py @@ -213,12 +213,12 @@ def initialise_simulation(self, sim): outreach_event = ChronicSyndrome_LaunchOutreachEvent(self) self.sim.schedule_event(outreach_event, self.sim.date + DateOffset(months=6)) - # Schedule the occurance of a population wide change in risk that goes through the health system: - popwide_hsi_event = HSI_ChronicSyndrome_PopulationWideBehaviourChange(self) - self.sim.modules['HealthSystem'].schedule_hsi_event( - popwide_hsi_event, priority=1, topen=self.sim.date, tclose=None + # Schedule the occurrence of a population wide change in risk: + popwide_event = ChronicSyndrome_PopulationWideBehaviourChange(self) + self.sim.schedule_event( + popwide_event, self.sim.date ) - logger.debug(key='debug', data='The population wide HSI event has been scheduled successfully!') + logger.debug(key='debug', data='The population wide event has been scheduled successfully!') def on_birth(self, mother_id, child_id): """Initialise our properties for a newborn individual. @@ -513,9 +513,9 @@ def did_not_run(self): pass -class HSI_ChronicSyndrome_PopulationWideBehaviourChange(HSI_Event, PopulationScopeEventMixin): +class ChronicSyndrome_PopulationWideBehaviourChange(Event, PopulationScopeEventMixin): """ - This is a Population-Wide Health System Interaction Event - will change the variables to do with risk for + This is a Population-Wide Event - will change the variables to do with risk for ChronicSyndrome """ @@ -523,11 +523,8 @@ def __init__(self, module): super().__init__(module) assert isinstance(module, ChronicSyndrome) - # Define the necessary information for a Population level HSI - self.TREATMENT_ID = 'ChronicSyndrome_PopulationWideBehaviourChange' - - def apply(self, population, squeeze_factor): - logger.debug(key='debug', data='This is HSI_ChronicSyndrome_PopulationWideBehaviourChange') + def apply(self, population): + logger.debug(key='debug', data='This is ChronicSyndrome_PopulationWideBehaviourChange') # As an example, we will reduce the chance of acquisition per year (due to behaviour change) self.module.parameters['p_acquisition_per_year'] = self.module.parameters['p_acquisition_per_year'] * 0.5 diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 09cb394804..76c401e3cb 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -8,7 +8,7 @@ from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods.hsi_event import HSI_Event -from tlo.util import random_date, sample_outcome, transition_states +from tlo.util import random_date, read_csv_files, sample_outcome, transition_states logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -164,7 +164,8 @@ def read_parameters(self, data_folder): """Import the relevant sheets from the ResourceFile (excel workbook) and declare values for other parameters (CSV ResourceFile). """ - workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', sheet_name=None) + workbook = read_csv_files(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception', + files=None) # Import selected sheets from the workbook as the parameters sheet_names = [ @@ -1350,10 +1351,10 @@ def __init__(self, *args): super().__init__(name='Labour') def read_parameters(self, *args): - parameter_dataframe = pd.read_excel(self.sim.modules['Contraception'].resourcefilepath / + parameter_dataframe = read_csv_files(self.sim.modules['Contraception'].resourcefilepath / 'contraception' / - 'ResourceFile_Contraception.xlsx', - sheet_name='simplified_labour_parameters') + 'ResourceFile_Contraception', + files='simplified_labour_parameters') self.load_parameters_from_dataframe(parameter_dataframe) def initialise_population(self, population): diff --git a/src/tlo/methods/epi.py b/src/tlo/methods/epi.py index 0ad0c75c1f..4aae66dc5e 100644 --- a/src/tlo/methods/epi.py +++ b/src/tlo/methods/epi.py @@ -7,6 +7,7 @@ from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata from tlo.methods.hsi_event import HSI_Event +from tlo.util import read_csv_files logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -71,9 +72,7 @@ def __init__(self, name=None, resourcefilepath=None): def read_parameters(self, data_folder): p = self.parameters - workbook = pd.read_excel( - Path(self.resourcefilepath) / 'ResourceFile_EPI_WHO_estimates.xlsx', sheet_name=None - ) + workbook = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_EPI_WHO_estimates', files=None) self.load_parameters_from_dataframe(workbook["parameters"]) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 5c6b2022e1..a8ad554a40 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1403,35 +1403,37 @@ def check_hsi_event_is_valid(self, hsi_event): # Check that non-empty treatment ID specified assert hsi_event.TREATMENT_ID != '' + + # Check that the target of the HSI is not the entire population + assert not isinstance(hsi_event.target, tlo.population.Population) - if not isinstance(hsi_event.target, tlo.population.Population): - # This is an individual-scoped HSI event. - # It must have EXPECTED_APPT_FOOTPRINT, BEDDAYS_FOOTPRINT and ACCEPTED_FACILITY_LEVELS. + # This is an individual-scoped HSI event. + # It must have EXPECTED_APPT_FOOTPRINT, BEDDAYS_FOOTPRINT and ACCEPTED_FACILITY_LEVELS. - # Correct formatted EXPECTED_APPT_FOOTPRINT - assert self.appt_footprint_is_valid(hsi_event.EXPECTED_APPT_FOOTPRINT), \ - f"the incorrectly formatted appt_footprint is {hsi_event.EXPECTED_APPT_FOOTPRINT}" + # Correct formatted EXPECTED_APPT_FOOTPRINT + assert self.appt_footprint_is_valid(hsi_event.EXPECTED_APPT_FOOTPRINT), \ + f"the incorrectly formatted appt_footprint is {hsi_event.EXPECTED_APPT_FOOTPRINT}" - # That it has an acceptable 'ACCEPTED_FACILITY_LEVEL' attribute - assert hsi_event.ACCEPTED_FACILITY_LEVEL in self._facility_levels, \ - f"In the HSI with TREATMENT_ID={hsi_event.TREATMENT_ID}, the ACCEPTED_FACILITY_LEVEL (=" \ - f"{hsi_event.ACCEPTED_FACILITY_LEVEL}) is not recognised." + # That it has an acceptable 'ACCEPTED_FACILITY_LEVEL' attribute + assert hsi_event.ACCEPTED_FACILITY_LEVEL in self._facility_levels, \ + f"In the HSI with TREATMENT_ID={hsi_event.TREATMENT_ID}, the ACCEPTED_FACILITY_LEVEL (=" \ + f"{hsi_event.ACCEPTED_FACILITY_LEVEL}) is not recognised." - self.bed_days.check_beddays_footprint_format(hsi_event.BEDDAYS_FOOTPRINT) + self.bed_days.check_beddays_footprint_format(hsi_event.BEDDAYS_FOOTPRINT) - # Check that this can accept the squeeze argument - assert _accepts_argument(hsi_event.run, 'squeeze_factor') + # Check that this can accept the squeeze argument + assert _accepts_argument(hsi_event.run, 'squeeze_factor') - # Check that the event does not request an appointment at a facility - # level which is not possible - appt_type_to_check_list = hsi_event.EXPECTED_APPT_FOOTPRINT.keys() - facility_appt_types = self._appt_type_by_facLevel[ - hsi_event.ACCEPTED_FACILITY_LEVEL - ] - assert facility_appt_types.issuperset(appt_type_to_check_list), ( - f"An appointment type has been requested at a facility level for " - f"which it is not possible: TREATMENT_ID={hsi_event.TREATMENT_ID}" - ) + # Check that the event does not request an appointment at a facility + # level which is not possible + appt_type_to_check_list = hsi_event.EXPECTED_APPT_FOOTPRINT.keys() + facility_appt_types = self._appt_type_by_facLevel[ + hsi_event.ACCEPTED_FACILITY_LEVEL + ] + assert facility_appt_types.issuperset(appt_type_to_check_list), ( + f"An appointment type has been requested at a facility level for " + f"which it is not possible: TREATMENT_ID={hsi_event.TREATMENT_ID}" + ) @staticmethod def is_treatment_id_allowed(treatment_id: str, service_availability: list) -> bool: @@ -1705,33 +1707,22 @@ def _match(_this_officer, facility_ids: List[int], officer_type: str): def record_hsi_event(self, hsi_event, actual_appt_footprint=None, squeeze_factor=None, did_run=True, priority=None): """ Record the processing of an HSI event. - If this is an individual-level HSI_Event, it will also record the actual appointment footprint + It will also record the actual appointment footprint. :param hsi_event: The HSI_Event (containing the initial expectations of footprints) :param actual_appt_footprint: The actual Appointment Footprint (if individual event) :param squeeze_factor: The squeeze factor (if individual event) """ - if isinstance(hsi_event.target, tlo.population.Population): - # Population HSI-Event (N.B. This is not actually logged.) - log_info = dict() - log_info['TREATMENT_ID'] = hsi_event.TREATMENT_ID - log_info['Number_By_Appt_Type_Code'] = 'Population' # remove the appt-types with zeros - log_info['Person_ID'] = -1 # Junk code - log_info['Squeeze_Factor'] = 0 - log_info['did_run'] = did_run - log_info['priority'] = priority - - else: - # Individual HSI-Event - _squeeze_factor = squeeze_factor if squeeze_factor != np.inf else 100.0 - self.write_to_hsi_log( - event_details=hsi_event.as_namedtuple(actual_appt_footprint), - person_id=hsi_event.target, - facility_id=hsi_event.facility_info.id, - squeeze_factor=_squeeze_factor, - did_run=did_run, - priority=priority, - ) + # HSI-Event + _squeeze_factor = squeeze_factor if squeeze_factor != np.inf else 100.0 + self.write_to_hsi_log( + event_details=hsi_event.as_namedtuple(actual_appt_footprint), + person_id=hsi_event.target, + facility_id=hsi_event.facility_info.id, + squeeze_factor=_squeeze_factor, + did_run=did_run, + priority=priority, + ) def write_to_hsi_log( self, @@ -1989,14 +1980,6 @@ def on_end_of_year(self) -> None: self._write_hsi_event_counts_to_log_and_reset() self._write_never_ran_hsi_event_counts_to_log_and_reset() - def run_population_level_events(self, _list_of_population_hsi_event_tuples: List[HSIEventQueueItem]) -> None: - """Run a list of population level events.""" - while len(_list_of_population_hsi_event_tuples) > 0: - pop_level_hsi_event_tuple = _list_of_population_hsi_event_tuples.pop() - pop_level_hsi_event = pop_level_hsi_event_tuple.hsi_event - pop_level_hsi_event.run(squeeze_factor=0) - self.record_hsi_event(hsi_event=pop_level_hsi_event) - def run_individual_level_events_in_mode_0_or_1(self, _list_of_individual_hsi_event_tuples: List[HSIEventQueueItem]) -> List: @@ -2220,10 +2203,8 @@ def _get_events_due_today(self,) -> Tuple[List, List]: """Interrogate the HSI_EVENT queue object to remove from it the events due today, and to return these in two lists: * list_of_individual_hsi_event_tuples_due_today - * list_of_population_hsi_event_tuples_due_today """ _list_of_individual_hsi_event_tuples_due_today = list() - _list_of_population_hsi_event_tuples_due_today = list() _list_of_events_not_due_today = list() # To avoid repeated dataframe accesses in subsequent loop, assemble set of alive @@ -2237,7 +2218,7 @@ def _get_events_due_today(self,) -> Tuple[List, List]: self.sim.population.props.index[self.sim.population.props.is_alive].to_list() ) - # Traverse the queue and split events into the three lists (due-individual, due-population, not_due) + # Traverse the queue and split events into the two lists (due-individual, not_due) while len(self.module.HSI_EVENT_QUEUE) > 0: next_event_tuple = hp.heappop(self.module.HSI_EVENT_QUEUE) @@ -2252,11 +2233,8 @@ def _get_events_due_today(self,) -> Tuple[List, List]: priority=next_event_tuple.priority ) - elif not ( - isinstance(event.target, tlo.population.Population) - or event.target in alive_persons - ): - # if individual level event and the person who is the target is no longer alive, do nothing more, + elif event.target not in alive_persons: + # if the person who is the target is no longer alive, do nothing more, # i.e. remove from heapq pass @@ -2266,38 +2244,28 @@ def _get_events_due_today(self,) -> Tuple[List, List]: else: # The event is now due to run today and the person is confirmed to be still alive - # Add it to the list of events due today (individual or population level) + # Add it to the list of events due today # NB. These list is ordered by priority and then due date - - is_pop_level_hsi_event = isinstance(event.target, tlo.population.Population) - if is_pop_level_hsi_event: - _list_of_population_hsi_event_tuples_due_today.append(next_event_tuple) - else: - _list_of_individual_hsi_event_tuples_due_today.append(next_event_tuple) + _list_of_individual_hsi_event_tuples_due_today.append(next_event_tuple) # add events from the _list_of_events_not_due_today back into the queue while len(_list_of_events_not_due_today) > 0: hp.heappush(self.module.HSI_EVENT_QUEUE, hp.heappop(_list_of_events_not_due_today)) - return _list_of_individual_hsi_event_tuples_due_today, _list_of_population_hsi_event_tuples_due_today + return _list_of_individual_hsi_event_tuples_due_today def process_events_mode_0_and_1(self, hold_over: List[HSIEventQueueItem]) -> None: while True: # Get the events that are due today: ( - list_of_individual_hsi_event_tuples_due_today, - list_of_population_hsi_event_tuples_due_today + list_of_individual_hsi_event_tuples_due_today ) = self._get_events_due_today() if ( (len(list_of_individual_hsi_event_tuples_due_today) == 0) - and (len(list_of_population_hsi_event_tuples_due_today) == 0) ): break - # Run the list of population-level HSI events - self.module.run_population_level_events(list_of_population_hsi_event_tuples_due_today) - # For each individual level event, check whether the equipment it has already declared is available. If it # is not, then call the HSI's never_run function, and do not take it forward for running; if it is then # add it to the list of events to run. @@ -2334,7 +2302,6 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: self.sim.population.props.index[self.sim.population.props.is_alive].to_list() ) - list_of_population_hsi_event_tuples_due_today = list() list_of_events_not_due_today = list() # Traverse the queue and run events due today until have capabilities still available @@ -2359,11 +2326,8 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: priority=next_event_tuple.priority ) - elif not ( - isinstance(event.target, tlo.population.Population) - or event.target in alive_persons - ): - # if individual level event and the person who is the target is no longer alive, + elif event.target not in alive_persons: + # if the person who is the target is no longer alive, # do nothing more, i.e. remove from heapq pass @@ -2379,133 +2343,128 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: else: # The event is now due to run today and the person is confirmed to be still alive. - # Add it to the list of events due today if at population level. - # Otherwise, run event immediately. - is_pop_level_hsi_event = isinstance(event.target, tlo.population.Population) - if is_pop_level_hsi_event: - list_of_population_hsi_event_tuples_due_today.append(next_event_tuple) + # Run event immediately. + + # Retrieve officers&facility required for HSI + original_call = next_event_tuple.hsi_event.expected_time_requests + _priority = next_event_tuple.priority + # In this version of mode_appt_constraints = 2, do not have access to squeeze + # based on queue information, and we assume no squeeze ever takes place. + squeeze_factor = 0. + + # Check if any of the officers required have run out. + out_of_resources = False + for officer, call in original_call.items(): + # If any of the officers are not available, then out of resources + if officer not in set_capabilities_still_available: + out_of_resources = True + # If officers still available, run event. Note: in current logic, a little + # overtime is allowed to run last event of the day. This seems more realistic + # than medical staff leaving earlier than + # planned if seeing another patient would take them into overtime. + + if out_of_resources: + + # Do not run, + # Call did_not_run for the hsi_event + rtn_from_did_not_run = event.did_not_run() + + # If received no response from the call to did_not_run, or a True signal, then + # add to the hold-over queue. + # Otherwise (disease module returns "FALSE") the event is not rescheduled and + # will not run. + + if rtn_from_did_not_run is not False: + # reschedule event + # Add the event to the queue: + hp.heappush(hold_over, next_event_tuple) + + # Log that the event did not run + self.module.record_hsi_event( + hsi_event=event, + actual_appt_footprint=event.EXPECTED_APPT_FOOTPRINT, + squeeze_factor=squeeze_factor, + did_run=False, + priority=_priority + ) + + # Have enough capabilities left to run event else: + # Notes-to-self: Shouldn't this be done after checking the footprint? + # Compute the bed days that are allocated to this HSI and provide this + # information to the HSI + if sum(event.BEDDAYS_FOOTPRINT.values()): + event._received_info_about_bed_days = \ + self.module.bed_days.issue_bed_days_according_to_availability( + facility_id=self.module.bed_days.get_facility_id_for_beds( + persons_id=event.target), + footprint=event.BEDDAYS_FOOTPRINT + ) + + # Check that a facility has been assigned to this HSI + assert event.facility_info is not None, \ + f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." - # Retrieve officers&facility required for HSI - original_call = next_event_tuple.hsi_event.expected_time_requests - _priority = next_event_tuple.priority - # In this version of mode_appt_constraints = 2, do not have access to squeeze - # based on queue information, and we assume no squeeze ever takes place. - squeeze_factor = 0. - - # Check if any of the officers required have run out. - out_of_resources = False - for officer, call in original_call.items(): - # If any of the officers are not available, then out of resources - if officer not in set_capabilities_still_available: - out_of_resources = True - # If officers still available, run event. Note: in current logic, a little - # overtime is allowed to run last event of the day. This seems more realistic - # than medical staff leaving earlier than - # planned if seeing another patient would take them into overtime. - - if out_of_resources: - - # Do not run, - # Call did_not_run for the hsi_event - rtn_from_did_not_run = event.did_not_run() - - # If received no response from the call to did_not_run, or a True signal, then - # add to the hold-over queue. - # Otherwise (disease module returns "FALSE") the event is not rescheduled and - # will not run. - - if rtn_from_did_not_run is not False: - # reschedule event - # Add the event to the queue: - hp.heappush(hold_over, next_event_tuple) - - # Log that the event did not run - self.module.record_hsi_event( + # Check if equipment declared is available. If not, call `never_ran` and do not run the + # event. (`continue` returns flow to beginning of the `while` loop) + if not event.is_all_declared_equipment_available: + self.module.call_and_record_never_ran_hsi_event( hsi_event=event, - actual_appt_footprint=event.EXPECTED_APPT_FOOTPRINT, - squeeze_factor=squeeze_factor, - did_run=False, - priority=_priority + priority=next_event_tuple.priority ) + continue - # Have enough capabilities left to run event - else: - # Notes-to-self: Shouldn't this be done after checking the footprint? - # Compute the bed days that are allocated to this HSI and provide this - # information to the HSI - if sum(event.BEDDAYS_FOOTPRINT.values()): - event._received_info_about_bed_days = \ - self.module.bed_days.issue_bed_days_according_to_availability( - facility_id=self.module.bed_days.get_facility_id_for_beds( - persons_id=event.target), - footprint=event.BEDDAYS_FOOTPRINT - ) - - # Check that a facility has been assigned to this HSI - assert event.facility_info is not None, \ - f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." + # Expected appt footprint before running event + _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT + # Run event & get actual footprint + actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) - # Check if equipment declared is available. If not, call `never_ran` and do not run the - # event. (`continue` returns flow to beginning of the `while` loop) - if not event.is_all_declared_equipment_available: - self.module.call_and_record_never_ran_hsi_event( - hsi_event=event, - priority=next_event_tuple.priority - ) - continue + # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call + if actual_appt_footprint is not None: - # Expected appt footprint before running event - _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT - # Run event & get actual footprint - actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) + # check its formatting: + assert self.module.appt_footprint_is_valid(actual_appt_footprint) - # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call - if actual_appt_footprint is not None: + # Update call that will be used to compute capabilities used + updated_call = self.module.get_appt_footprint_as_time_request( + facility_info=event.facility_info, + appt_footprint=actual_appt_footprint + ) + else: + actual_appt_footprint = _appt_footprint_before_running + updated_call = original_call + + # Recalculate call on officers based on squeeze factor. + for k in updated_call.keys(): + updated_call[k] = updated_call[k]/(squeeze_factor + 1.) + + # Subtract this from capabilities used so-far today + capabilities_monitor.subtract(updated_call) + + # If any of the officers have run out of time by performing this hsi, + # remove them from list of available officers. + for officer, call in updated_call.items(): + if capabilities_monitor[officer] <= 0: + if officer in set_capabilities_still_available: + set_capabilities_still_available.remove(officer) + else: + logger.warning( + key="message", + data=(f"{event.TREATMENT_ID} actual_footprint requires different" + f"officers than expected_footprint.") + ) - # check its formatting: - assert self.module.appt_footprint_is_valid(actual_appt_footprint) + # Update today's footprint based on actual call and squeeze factor + self.module.running_total_footprint.update(updated_call) - # Update call that will be used to compute capabilities used - updated_call = self.module.get_appt_footprint_as_time_request( - facility_info=event.facility_info, - appt_footprint=actual_appt_footprint - ) - else: - actual_appt_footprint = _appt_footprint_before_running - updated_call = original_call - - # Recalculate call on officers based on squeeze factor. - for k in updated_call.keys(): - updated_call[k] = updated_call[k]/(squeeze_factor + 1.) - - # Subtract this from capabilities used so-far today - capabilities_monitor.subtract(updated_call) - - # If any of the officers have run out of time by performing this hsi, - # remove them from list of available officers. - for officer, call in updated_call.items(): - if capabilities_monitor[officer] <= 0: - if officer in set_capabilities_still_available: - set_capabilities_still_available.remove(officer) - else: - logger.warning( - key="message", - data=(f"{event.TREATMENT_ID} actual_footprint requires different" - f"officers than expected_footprint.") - ) - - # Update today's footprint based on actual call and squeeze factor - self.module.running_total_footprint.update(updated_call) - - # Write to the log - self.module.record_hsi_event( - hsi_event=event, - actual_appt_footprint=actual_appt_footprint, - squeeze_factor=squeeze_factor, - did_run=True, - priority=_priority - ) + # Write to the log + self.module.record_hsi_event( + hsi_event=event, + actual_appt_footprint=actual_appt_footprint, + squeeze_factor=squeeze_factor, + did_run=True, + priority=_priority + ) # Don't have any capabilities at all left for today, no # point in going through the queue to check what's left to do today. @@ -2541,11 +2500,8 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: priority=next_event_tuple.priority ) - elif not ( - isinstance(event.target, tlo.population.Population) - or event.target in alive_persons - ): - # if individual level event and the person who is the target is no longer alive, + elif event.target not in alive_persons: + # if the person who is the target is no longer alive, # do nothing more, i.e. remove from heapq pass @@ -2556,45 +2512,37 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: hp.heappush(list_of_events_not_due_today, next_event_tuple) else: - # Add it to the list of events due today if at population level. - # Otherwise, run event immediately. - is_pop_level_hsi_event = isinstance(event.target, tlo.population.Population) - if is_pop_level_hsi_event: - list_of_population_hsi_event_tuples_due_today.append(next_event_tuple) - else: - # In previous iteration, have already run all the events for today that could run - # given capabilities available, so put back any remaining events due today to the - # hold_over queue as it would not be possible to run them today. - - # Do not run, - # Call did_not_run for the hsi_event - rtn_from_did_not_run = event.did_not_run() - - # If received no response from the call to did_not_run, or a True signal, then - # add to the hold-over queue. - # Otherwise (disease module returns "FALSE") the event is not rescheduled and - # will not run. - - if rtn_from_did_not_run is not False: - # reschedule event - # Add the event to the queue: - hp.heappush(hold_over, next_event_tuple) - - # Log that the event did not run - self.module.record_hsi_event( - hsi_event=event, - actual_appt_footprint=event.EXPECTED_APPT_FOOTPRINT, - squeeze_factor=0, - did_run=False, - priority=next_event_tuple.priority - ) + # In previous iteration, have already run all the events for today that could run + # given capabilities available, so put back any remaining events due today to the + # hold_over queue as it would not be possible to run them today. + + # Do not run, + # Call did_not_run for the hsi_event + rtn_from_did_not_run = event.did_not_run() + + # If received no response from the call to did_not_run, or a True signal, then + # add to the hold-over queue. + # Otherwise (disease module returns "FALSE") the event is not rescheduled and + # will not run. + + if rtn_from_did_not_run is not False: + # reschedule event + # Add the event to the queue: + hp.heappush(hold_over, next_event_tuple) + + # Log that the event did not run + self.module.record_hsi_event( + hsi_event=event, + actual_appt_footprint=event.EXPECTED_APPT_FOOTPRINT, + squeeze_factor=0, + did_run=False, + priority=next_event_tuple.priority + ) # add events from the list_of_events_not_due_today back into the queue while len(list_of_events_not_due_today) > 0: hp.heappush(self.module.HSI_EVENT_QUEUE, hp.heappop(list_of_events_not_due_today)) - # Run the list of population-level HSI events - self.module.run_population_level_events(list_of_population_hsi_event_tuples_due_today) def apply(self, population): diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index a8c621b6c1..4e03286a23 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -25,7 +25,6 @@ """ from __future__ import annotations -import os from typing import TYPE_CHECKING, List import numpy as np @@ -40,7 +39,7 @@ from tlo.methods.hsi_event import HSI_Event from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom -from tlo.util import create_age_range_lookup +from tlo.util import create_age_range_lookup, read_csv_files if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import HSIEventScheduler @@ -339,6 +338,11 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False): Types.REAL, "Probability that a male will be circumcised, if HIV-negative, following testing", ), + "increase_in_prob_circ_2019": Parameter( + Types.REAL, + "increase in probability that a male will be circumcised, if HIV-negative, following testing" + "from 2019 onwards", + ), "prob_circ_for_child_before_2020": Parameter( Types.REAL, "Probability that a male aging <15 yrs will be circumcised before year 2020", @@ -410,6 +414,10 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False): Types.DATA_FRAME, "the parameters and values changed in scenario analysis" ), + "interval_for_viral_load_measurement_months": Parameter( + Types.REAL, + " the interval for viral load monitoring in months" + ), } def read_parameters(self, data_folder): @@ -423,10 +431,7 @@ def read_parameters(self, data_folder): # Shortcut to parameters dict p = self.parameters - workbook = pd.read_excel( - os.path.join(self.resourcefilepath, "ResourceFile_HIV.xlsx"), - sheet_name=None, - ) + workbook = read_csv_files(self.resourcefilepath/'ResourceFile_HIV', files=None) self.load_parameters_from_dataframe(workbook["parameters"]) # Load data on HIV prevalence @@ -597,6 +602,10 @@ def _build_linear_models(self): p["prob_circ_after_hiv_test"], Predictor("hv_inf").when(False, 1.0).otherwise(0.0), Predictor("sex").when("M", 1.0).otherwise(0.0), + Predictor("year", + external=True, + conditions_are_mutually_exclusive=True).when("<2019", 1) + .otherwise(p["increase_in_prob_circ_2019"]) ) # Linear model for circumcision for male and aging <15 yrs who spontaneously presents for VMMC @@ -2418,7 +2427,8 @@ def apply(self, person_id, squeeze_factor): # If person is a man, and not circumcised, then consider referring to VMMC if (person["sex"] == "M") & (~person["li_is_circ"]): x = self.module.lm["lm_circ"].predict( - df.loc[[person_id]], self.module.rng + df.loc[[person_id]], self.module.rng, + year=self.sim.date.year, ) if x: self.sim.modules["HealthSystem"].schedule_hsi_event( @@ -2493,6 +2503,14 @@ def apply(self, person_id, squeeze_factor): if not person["is_alive"]: return + # get confirmatory test + test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( + dx_tests_to_run="hiv_rapid_test", hsi_event=self + ) + if test_result is not None: + df.at[person_id, "hv_number_tests"] += 1 + df.at[person_id, "hv_last_test_date"] = self.sim.date + # if person not circumcised, perform the procedure if not person["li_is_circ"]: # Check/log use of consumables, if materials available, do circumcision and schedule follow-up appts @@ -2830,6 +2848,15 @@ def do_at_initiation(self, person_id): # ART is first item in drugs_available dict if drugs_available.get('art', False): + + # get confirmatory test + test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( + dx_tests_to_run="hiv_rapid_test", hsi_event=self + ) + if test_result is not None: + df.at[person_id, "hv_number_tests"] += 1 + df.at[person_id, "hv_last_test_date"] = self.sim.date + # Assign person to be suppressed or un-suppressed viral load # (If person is VL suppressed This will prevent the Onset of AIDS, or an AIDS death if AIDS has already # onset) @@ -2860,13 +2887,15 @@ def do_at_continuation(self, person_id): df = self.sim.population.props person = df.loc[person_id] + p = self.module.parameters # default to person stopping cotrimoxazole df.at[person_id, "hv_on_cotrimoxazole"] = False # Viral Load Monitoring # NB. This does not have a direct effect on outcomes for the person. - _ = self.get_consumables(item_codes=self.module.item_codes_for_consumables_required['vl_measurement']) + if self.module.rng.random_sample(size=1) < p['dispensation_period_months'] / p['interval_for_viral_load_measurement_months']: + _ = self.get_consumables(item_codes=self.module.item_codes_for_consumables_required['vl_measurement']) # Check if drugs are available, and provide drugs: drugs_available = self.get_drugs(age_of_person=person["age_years"]) diff --git a/src/tlo/methods/hiv_tb_calibration.py b/src/tlo/methods/hiv_tb_calibration.py index bd09f54d96..c138523187 100644 --- a/src/tlo/methods/hiv_tb_calibration.py +++ b/src/tlo/methods/hiv_tb_calibration.py @@ -60,7 +60,7 @@ def read_data_files(self): """Make a dict of all data to be used in calculating calibration score""" # # HIV read in resource files for data - xls = pd.ExcelFile(self.resourcefilepath / "ResourceFile_HIV.xlsx") + xls = pd.ExcelFile(self.resourcefilepath / "ResourceFile_HIV/parameters.csv") # MPHIA HIV data - age-structured data_hiv_mphia_inc = pd.read_excel(xls, sheet_name="MPHIA_incidence2015") diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index c252a40974..26fcd6d880 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -7,7 +7,6 @@ from tlo import Date, logging from tlo.events import Event -from tlo.population import Population if TYPE_CHECKING: from tlo import Module, Simulation @@ -72,7 +71,7 @@ class HSIEventQueueItem(NamedTuple): class HSI_Event: """Base HSI event class, from which all others inherit. - Concrete subclasses should also inherit from one of the EventMixin classes + Concrete subclasses should also inherit from `IndividualScopeEventMixin` defined in `src/tlo/events.py`, and implement at least an `apply` and `did_not_run` method. """ @@ -170,13 +169,13 @@ def post_apply_hook(self) -> None: def _run_after_hsi_event(self) -> None: """ Do things following the event's `apply` and `post_apply_hook` functions running. - * Impose the bed-days footprint (if target of the HSI is a person_id) + * Impose the bed-days footprint * Record the equipment that has been added before and during the course of the HSI Event. """ - if isinstance(self.target, int): - self.healthcare_system.bed_days.impose_beddays_footprint( - person_id=self.target, footprint=self.bed_days_allocated_to_this_event - ) + + self.healthcare_system.bed_days.impose_beddays_footprint( + person_id=self.target, footprint=self.bed_days_allocated_to_this_event + ) if self.facility_info is not None: # If there is a facility_info (e.g., healthsystem not running in disabled mode), then record equipment used @@ -323,28 +322,28 @@ def initialise(self) -> None: # Over-write ACCEPTED_FACILITY_LEVEL to to redirect all '1b' appointments to '2' self._adjust_facility_level_to_merge_1b_and_2() - if not isinstance(self.target, Population): - self.facility_info = health_system.get_facility_info(self) - - # If there are bed-days specified, add (if needed) the in-patient admission and in-patient day Appointment - # Types. - # (HSI that require a bed for one or more days always need such appointments, but this may have been - # missed in the declaration of the `EXPECTED_APPT_FOOTPRINT` in the HSI.) - # NB. The in-patient day Appointment time is automatically applied on subsequent days. - if sum(self.BEDDAYS_FOOTPRINT.values()): - self.EXPECTED_APPT_FOOTPRINT = ( - health_system.bed_days.add_first_day_inpatient_appts_to_footprint( - self.EXPECTED_APPT_FOOTPRINT - ) + self.facility_info = health_system.get_facility_info(self) + + # If there are bed-days specified, add (if needed) the in-patient admission and in-patient day Appointment + # Types. + # (HSI that require a bed for one or more days always need such appointments, but this may have been + # missed in the declaration of the `EXPECTED_APPT_FOOTPRINT` in the HSI.) + # NB. The in-patient day Appointment time is automatically applied on subsequent days. + if sum(self.BEDDAYS_FOOTPRINT.values()): + self.EXPECTED_APPT_FOOTPRINT = ( + health_system.bed_days.add_first_day_inpatient_appts_to_footprint( + self.EXPECTED_APPT_FOOTPRINT ) + ) - # Write the time requirements for staff of the appointments to the HSI: - self.expected_time_requests = ( - health_system.get_appt_footprint_as_time_request( - facility_info=self.facility_info, - appt_footprint=self.EXPECTED_APPT_FOOTPRINT, - ) + # Write the time requirements for staff of the appointments to the HSI: + self.expected_time_requests = ( + health_system.get_appt_footprint_as_time_request( + facility_info=self.facility_info, + appt_footprint=self.EXPECTED_APPT_FOOTPRINT, ) + ) + # Do checks self._check_if_appt_footprint_can_run() @@ -353,20 +352,19 @@ def _check_if_appt_footprint_can_run(self) -> bool: """Check that event (if individual level) is able to run with this configuration of officers (i.e. check that this does not demand officers that are _never_ available), and issue warning if not. """ - if not isinstance(self.target, Population): - if self.healthcare_system._officers_with_availability.issuperset( - self.expected_time_requests.keys() - ): - return True - else: - logger.debug( - key="message", - data=( - f"The expected footprint of {self.TREATMENT_ID} is not possible with the configuration of " - f"officers." - ), - ) - return False + if self.healthcare_system._officers_with_availability.issuperset( + self.expected_time_requests.keys() + ): + return True + else: + logger.debug( + key="message", + data=( + f"The expected footprint of {self.TREATMENT_ID} is not possible with the configuration of " + f"officers." + ), + ) + return False @staticmethod def _return_item_codes_in_dict( @@ -449,11 +447,9 @@ def run(self): # Check that the person is still alive (this check normally happens in the HealthSystemScheduler and silently # do not run the HSI event) - if isinstance(self.hsi_event.target, Population) or ( - self.hsi_event.module.sim.population.props.at[ + if self.hsi_event.module.sim.population.props.at[ self.hsi_event.target, "is_alive" - ] - ): + ]: if self.run_hsi: # Run the event (with 0 squeeze_factor) and ignore the output diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 35081b7d27..876259e020 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -17,7 +17,7 @@ from tlo.methods.hsi_event import HSI_Event from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.postnatal_supervisor import PostnatalWeekOneMaternalEvent -from tlo.util import BitsetHandler +from tlo.util import BitsetHandler, read_csv_files if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import HSIEventScheduler @@ -620,9 +620,9 @@ def __init__(self, name=None, resourcefilepath=None): } def read_parameters(self, data_folder): - parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_LabourSkilledBirth' - 'Attendance.xlsx', - sheet_name='parameter_values') + parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_LabourSkilledBirth' + 'Attendance', + files='parameter_values') self.load_parameters_from_dataframe(parameter_dataframe) def initialise_population(self, population): diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index 0eee59980b..aa6cc0e33d 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -20,7 +20,7 @@ from tlo.methods.hsi_event import HSI_Event from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom -from tlo.util import random_date +from tlo.util import random_date, read_csv_files if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler @@ -228,7 +228,8 @@ def __init__(self, name=None, resourcefilepath=None): } def read_parameters(self, data_folder): - workbook = pd.read_excel(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', sheet_name=None) + # workbook = pd.read_excel(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', sheet_name=None) + workbook = read_csv_files(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria', files=None) self.load_parameters_from_dataframe(workbook['parameters']) p = self.parameters @@ -628,7 +629,7 @@ def initialise_simulation(self, sim): self.item_codes_for_consumables_required['paracetamol'] = get_item_code('Paracetamol 500mg_1000_CMST') # malaria treatment complicated - same consumables for adults and children - self.item_codes_for_consumables_required['malaria_complicated'] = get_item_code('Injectable artesunate') + self.item_codes_for_consumables_required['malaria_complicated_artesunate'] = get_item_code('Injectable artesunate') self.item_codes_for_consumables_required['malaria_complicated_optional_items'] = [ get_item_code('Malaria test kit (RDT)'), @@ -640,7 +641,7 @@ def initialise_simulation(self, sim): # malaria IPTp for pregnant women self.item_codes_for_consumables_required['malaria_iptp'] = get_item_code( - 'Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg' + 'Fansidar (sulphadoxine / pyrimethamine tab)' ) def update_parameters_for_program_scaleup(self): @@ -869,6 +870,7 @@ def do_at_generic_first_appt_emergency( event, priority=0, topen=self.sim.date ) + class MalariaPollingEventDistrict(RegularEvent, PopulationScopeEventMixin): """ this calls functions to assign new malaria infections @@ -1231,26 +1233,32 @@ def get_drugs(self, age_of_person): # non-complicated malaria if age_of_person < 5: # Formulation for young children + # 5–14kg: 1 tablet(120mg Lumefantrine / 20mg Artemether) every 12 hours for 3 days + # paracetamol syrup in 1ml doses, 10ml 4x per day, 3 days drugs_available = self.get_consumables( - item_codes=self.module.item_codes_for_consumables_required['malaria_uncomplicated_young_children'], - optional_item_codes=[self.module.item_codes_for_consumables_required['paracetamol_syrup'], - self.module.item_codes_for_consumables_required['malaria_rdt']] + item_codes={self.module.item_codes_for_consumables_required['malaria_uncomplicated_young_children']: 6}, + optional_item_codes={self.module.item_codes_for_consumables_required['paracetamol_syrup']: 120, + self.module.item_codes_for_consumables_required['malaria_rdt']: 1} ) elif 5 <= age_of_person <= 15: # Formulation for older children + # 35–44 kg: 4 tablets every 12 hours for 3 days + # paracetamol syrup in 1ml doses, 15ml 4x per day, 3 days drugs_available = self.get_consumables( - item_codes=self.module.item_codes_for_consumables_required['malaria_uncomplicated_older_children'], - optional_item_codes=[self.module.item_codes_for_consumables_required['paracetamol_syrup'], - self.module.item_codes_for_consumables_required['malaria_rdt']] + item_codes={self.module.item_codes_for_consumables_required['malaria_uncomplicated_older_children']: 24}, + optional_item_codes={self.module.item_codes_for_consumables_required['paracetamol_syrup']: 180, + self.module.item_codes_for_consumables_required['malaria_rdt']: 1} ) else: # Formulation for adults + # 4 tablets every 12 hours for 3 day + # paracetamol in 1 mg doses, 4g per day for 3 days drugs_available = self.get_consumables( - item_codes=self.module.item_codes_for_consumables_required['malaria_uncomplicated_adult'], - optional_item_codes=[self.module.item_codes_for_consumables_required['paracetamol'], - self.module.item_codes_for_consumables_required['malaria_rdt']] + item_codes={self.module.item_codes_for_consumables_required['malaria_uncomplicated_adult']: 24}, + optional_item_codes={self.module.item_codes_for_consumables_required['paracetamol']: 12_000, + self.module.item_codes_for_consumables_required['malaria_rdt']: 1} ) return drugs_available @@ -1287,8 +1295,11 @@ def apply(self, person_id, squeeze_factor): data=f'HSI_Malaria_Treatment_Complicated: requesting complicated malaria treatment for ' f' {person_id}') + # dosage in 60mg artesunate ampoules + # First dose: 2.4 mg/kg × 25 kg = 60 mg (administered IV or IM). + # Repeat 60 mg after 12 hours and then again at 24 hours. if self.get_consumables( - item_codes=self.module.item_codes_for_consumables_required['malaria_complicated'], + item_codes={self.module.item_codes_for_consumables_required['malaria_complicated_artesunate']: 3}, optional_item_codes=self.module.item_codes_for_consumables_required[ 'malaria_complicated_optional_items'] ): @@ -1317,6 +1328,12 @@ def apply(self, person_id, squeeze_factor): ) logger.info(key='rdt_log', data=person_details_for_test) + # schedule ACT to follow inpatient care, this is delivered through outpatient facility + continue_to_treat = HSI_Malaria_Treatment(self.module, person_id=person_id) + self.sim.modules['HealthSystem'].schedule_hsi_event( + continue_to_treat, priority=1, topen=self.sim.date, tclose=None + ) + def did_not_run(self): logger.debug(key='message', data='HSI_Malaria_Treatment_Complicated: did not run') @@ -1351,6 +1368,7 @@ def apply(self, person_id, squeeze_factor): data=f'HSI_MalariaIPTp: requesting IPTp for person {person_id}') # request the treatment + # dosage is one tablet if self.get_consumables(self.module.item_codes_for_consumables_required['malaria_iptp']): logger.debug(key='message', data=f'HSI_MalariaIPTp: giving IPTp for person {person_id}') diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py index 39f9828860..585aadf511 100644 --- a/src/tlo/methods/measles.py +++ b/src/tlo/methods/measles.py @@ -1,7 +1,6 @@ from __future__ import annotations import math -import os from typing import TYPE_CHECKING, List import pandas as pd @@ -13,7 +12,7 @@ from tlo.methods.hsi_event import HSI_Event from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom -from tlo.util import random_date +from tlo.util import random_date, read_csv_files if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import HSIEventScheduler @@ -102,11 +101,7 @@ def __init__(self, name=None, resourcefilepath=None): def read_parameters(self, data_folder): """Read parameter values from file """ - - workbook = pd.read_excel( - os.path.join(self.resourcefilepath, "ResourceFile_Measles.xlsx"), - sheet_name=None, - ) + workbook = read_csv_files(self.resourcefilepath/'ResourceFile_Measles', files=None) self.load_parameters_from_dataframe(workbook["parameters"]) self.parameters["symptom_prob"] = workbook["symptoms"] diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index 3691bc6003..0caf207499 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -10,7 +10,7 @@ from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event from tlo.methods.postnatal_supervisor import PostnatalWeekOneNeonatalEvent -from tlo.util import BitsetHandler +from tlo.util import BitsetHandler, read_csv_files logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -311,8 +311,8 @@ def __init__(self, name=None, resourcefilepath=None): def read_parameters(self, data_folder): - parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes.xlsx', - sheet_name='parameter_values') + parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes', + files='parameter_values') self.load_parameters_from_dataframe(parameter_dataframe) # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 25bce6013f..739b747e88 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -9,6 +9,7 @@ from tlo.methods import Metadata, postnatal_supervisor_lm, pregnancy_helper_functions from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event +from tlo.util import read_csv_files logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -230,8 +231,8 @@ def __init__(self, name=None, resourcefilepath=None): } def read_parameters(self, data_folder): - parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PostnatalSupervisor.xlsx', - sheet_name='parameter_values') + parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_PostnatalSupervisor', + files='parameter_values') self.load_parameters_from_dataframe(parameter_dataframe) def initialise_population(self, population): diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 7dd8819ab6..b25a935d9e 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -27,7 +27,7 @@ ) from tlo.methods.causes import Cause from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin -from tlo.util import BitsetHandler +from tlo.util import BitsetHandler, read_csv_files if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import HSIEventScheduler @@ -434,8 +434,8 @@ def __init__(self, name=None, resourcefilepath=None): def read_parameters(self, data_folder): # load parameters from the resource file - parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor.xlsx', - sheet_name='parameter_values') + parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor', + files='parameter_values') self.load_parameters_from_dataframe(parameter_dataframe) # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module. diff --git a/src/tlo/methods/schisto.py b/src/tlo/methods/schisto.py index 0e9735286a..385b8cd77d 100644 --- a/src/tlo/methods/schisto.py +++ b/src/tlo/methods/schisto.py @@ -14,7 +14,7 @@ from tlo.methods.hsi_event import HSI_Event from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom -from tlo.util import random_date +from tlo.util import random_date, read_csv_files if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import HSIEventScheduler @@ -123,7 +123,7 @@ def read_parameters(self, data_folder): """Read parameters and register symptoms.""" # Load parameters - workbook = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_Schisto.xlsx', sheet_name=None) + workbook = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_Schisto', files=None) self.parameters = self._load_parameters_from_workbook(workbook) for _spec in self.species.values(): self.parameters.update(_spec.load_parameters_from_workbook(workbook)) diff --git a/src/tlo/methods/simplified_births.py b/src/tlo/methods/simplified_births.py index 50408f21e9..4daa8c3805 100644 --- a/src/tlo/methods/simplified_births.py +++ b/src/tlo/methods/simplified_births.py @@ -111,7 +111,7 @@ def read_parameters(self, data_folder): self.parameters['months_between_pregnancy_and_delivery'] = 9 # Breastfeeding status for newborns (importing from the Newborn resourcefile) - rf = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes.xlsx') + rf = pd.read_csv(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes/parameter_values.csv') param_as_string = rf.loc[rf.parameter_name == 'prob_breastfeeding_type']['value'].iloc[0] parameter = json.loads(param_as_string)[0] self.parameters['prob_breastfeeding_type'] = parameter diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index c067a78929..7436fb5e5f 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -3,8 +3,6 @@ It schedules TB treatment and follow-up appointments along with preventive therapy for eligible people (HIV+ and paediatric contacts of active TB cases """ - -import os from functools import reduce import pandas as pd @@ -17,7 +15,7 @@ from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event from tlo.methods.symptommanager import Symptom -from tlo.util import random_date +from tlo.util import parse_csv_values_for_columns_with_mixed_datatypes, random_date, read_csv_files logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -399,9 +397,7 @@ def read_parameters(self, data_folder): """ # 1) Read the ResourceFiles - workbook = pd.read_excel( - os.path.join(self.resourcefilepath, "ResourceFile_TB.xlsx"), sheet_name=None - ) + workbook = read_csv_files(self.resourcefilepath/"ResourceFile_TB", files=None) self.load_parameters_from_dataframe(workbook["parameters"]) p = self.parameters @@ -782,6 +778,21 @@ def get_consumables_for_dx_and_tx(self): ) ) + # TB Culture + self.item_codes_for_consumables_required['culture_test'] = \ + hs.get_item_code_from_item_name("MGIT960 Culture and DST") + + # sensitivity/specificity set for smear status of cases + self.sim.modules["HealthSystem"].dx_manager.register_dx_test( + tb_culture_test=DxTest( + property="tb_inf", + target_categories=["active"], + sensitivity=1.0, + specificity=1.0, + item_codes=self.item_codes_for_consumables_required['culture_test'] + ) + ) + # 4) -------- Define the treatment options -------- # treatment supplied as full kits for duration of treatment # adult treatment - primary @@ -912,6 +923,10 @@ def update_parameters_for_program_scaleup(self): """ options for program scale-up are 'target' or 'max' """ p = self.parameters scaled_params_workbook = p["scaleup_parameters"] + for col in scaled_params_workbook.columns: + scaled_params_workbook[col] = scaled_params_workbook[col].apply( + parse_csv_values_for_columns_with_mixed_datatypes + ) if p['type_of_scaleup'] == 'target': scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict() @@ -1458,6 +1473,7 @@ def __init__(self, module): def apply(self, population): self.module.update_parameters_for_program_scaleup() + # note also culture test used in target/max scale-up in place of clinical dx class TbActiveEvent(RegularEvent, PopulationScopeEventMixin): @@ -1729,6 +1745,8 @@ def apply(self, person_id, squeeze_factor): # check if patient has: cough, fever, night sweat, weight loss # if none of the above conditions are present, no further action persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id) + person_has_tb_symptoms = all(symptom in persons_symptoms for symptom in self.module.symptom_list) + if not any(x in self.module.symptom_list for x in persons_symptoms): return self.make_appt_footprint({}) @@ -1944,6 +1962,27 @@ def apply(self, person_id, squeeze_factor): tclose=None, ) + # ------------------------- Culture testing if program scale-up ------------------------- # + # under program scale-up, if a person tests negative but still has symptoms + # indicative of TB, they are referred for culture test which has perfect sensitivity + # this has the effect to reduce false negatives + if not test_result and person_has_tb_symptoms: + if p['type_of_scaleup'] != 'none' and self.sim.date.year >= p['scaleup_start_year']: + logger.debug( + key="message", + data=f"HSI_Tb_ScreeningAndRefer: scheduling culture for person {person_id}", + ) + + culture_event = HSI_Tb_Culture( + self.module, person_id=person_id + ) + self.sim.modules["HealthSystem"].schedule_hsi_event( + culture_event, + priority=0, + topen=now, + tclose=None, + ) + # Return the footprint. If it should be suppressed, return a blank footprint. if self.suppress_footprint: return self.make_appt_footprint({}) @@ -1979,6 +2018,7 @@ def apply(self, person_id, squeeze_factor): df = self.sim.population.props now = self.sim.date person = df.loc[person_id] + p = self.module.parameters test_result = None # If the person is dead or already diagnosed, do nothing do not occupy any resources @@ -2021,6 +2061,79 @@ def apply(self, person_id, squeeze_factor): tclose=None, priority=0, ) + # ------------------------- Culture testing if program scale-up ------------------------- # + # under program scale-up, if a person tests negative but still has all symptoms + # indicative of TB, they are referred for culture test which has perfect sensitivity + # this has the effect to reduce false negatives + person_has_tb_symptoms = all(symptom in persons_symptoms for symptom in self.module.symptom_list) + + if not test_result and person_has_tb_symptoms: + if p['type_of_scaleup'] != 'none' and self.sim.date.year >= p['scaleup_start_year']: + + logger.debug( + key="message", + data=f"HSI_Tb_ClinicalDiagnosis: scheduling culture for person {person_id}", + ) + + culture_event = HSI_Tb_Culture( + self.module, person_id=person_id + ) + self.sim.modules["HealthSystem"].schedule_hsi_event( + culture_event, + priority=0, + topen=now, + tclose=None, + ) + + +class HSI_Tb_Culture(HSI_Event, IndividualScopeEventMixin): + """ + This the TB culture HSI used for microbiological diagnosis of TB + results (MGIT) are available after 2-6 weeks + will return drug-susceptibility + 100% sensitivity in smear-positive + 80-90% sensitivity in smear-negative + if this test is not available, not further action as this is + already preceded by a sequence of tests + """ + + def __init__(self, module, person_id): + super().__init__(module, person_id=person_id) + assert isinstance(module, Tb) + + self.TREATMENT_ID = "Tb_Test_Culture" + self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({"LabTBMicro": 1}) + self.ACCEPTED_FACILITY_LEVEL = '1b' + + def apply(self, person_id, squeeze_factor): + + df = self.sim.population.props + + if not df.at[person_id, "is_alive"] or df.at[person_id, "tb_diagnosed"]: + return self.sim.modules["HealthSystem"].get_blank_appt_footprint() + + test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( + dx_tests_to_run="tb_culture_test", hsi_event=self) + + # todo equipment required: MGIT instrument, MGIT tube, reagent kit included in consumables + if test_result is not None: + self.add_equipment({'Autoclave', 'Blood culture incubator', 'Vortex mixer', + 'Dispensing pumps for culture media preparation', 'Biosafety Cabinet (Class II)', + 'Centrifuge'}) + + # if test returns positive result, refer for appropriate treatment + if test_result: + df.at[person_id, "tb_diagnosed"] = True + df.at[person_id, "tb_date_diagnosed"] = self.sim.date + + self.sim.modules["HealthSystem"].schedule_hsi_event( + HSI_Tb_StartTreatment( + person_id=person_id, module=self.module, facility_level="1a" + ), + topen=self.sim.date, + tclose=None, + priority=0, + ) class HSI_Tb_Xray_level1b(HSI_Event, IndividualScopeEventMixin): @@ -2483,6 +2596,7 @@ def apply(self, person_id, squeeze_factor): self.number_of_occurrences += 1 df = self.sim.population.props # shortcut to the dataframe + now = self.sim.date person = df.loc[person_id] @@ -2494,6 +2608,19 @@ def apply(self, person_id, squeeze_factor): ): return + # refer for HIV testing: all ages + # do not run if already HIV diagnosed or had test in last week + if not person["hv_diagnosed"] or (person["hv_last_test_date"] >= (now - DateOffset(days=7))): + self.sim.modules["HealthSystem"].schedule_hsi_event( + hsi_event=hiv.HSI_Hiv_TestAndRefer( + person_id=person_id, + module=self.sim.modules["Hiv"], + referred_from="Tb", + ), + priority=1, + topen=now, + tclose=None, + ) # if currently have symptoms of TB, refer for screening/testing persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id) if any(x in self.module.symptom_list for x in persons_symptoms): diff --git a/src/tlo/scenario.py b/src/tlo/scenario.py index d2b23c8646..0abb8b5b75 100644 --- a/src/tlo/scenario.py +++ b/src/tlo/scenario.py @@ -243,6 +243,11 @@ def draw_parameters(self, draw_number, rng): """ return None + def draw_name(self, draw_number) -> str: + """Returns the name of the draw corresponding to the given draw number. This is offered for convenience so that + the logfile contain a 'user-friendly' label for the draw.""" + return str(draw_number) + def get_log_config(self, override_output_directory=None): """Returns the log configuration for the scenario, with some post_processing.""" log_config = self.log_configuration() @@ -328,6 +333,7 @@ def get_draw(self, draw_number): return { "draw_number": draw_number, "parameters": self.scenario.draw_parameters(draw_number, self.scenario.rng), + "draw_name": self.scenario.draw_name(draw_number), } def get_run_config(self, scenario_path): @@ -419,6 +425,7 @@ def run_sample_by_number(self, output_directory, draw_number, sample_number): log_config=log_config, ) sim.register(*self.scenario.modules()) + logger.info(key="draw_name", data={'draw_name': draw['draw_name']}, description="The draw name") if sample["parameters"] is not None: self.override_parameters(sim, sample["parameters"]) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 0f42b2d851..f14ed2ebea 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -10,6 +10,7 @@ from tlo import Date, DateOffset, Module, Property, Simulation, Types, logging from tlo.analysis.utils import ( colors_in_matplotlib, + compute_summary_statistics, flatten_multi_index_series_into_dict_for_logging, get_coarse_appt_type, get_color_cause_of_death_or_daly_label, @@ -573,7 +574,7 @@ def check_parameters(self) -> None: sim.simulate(end_date=Date(year_of_change + 2, 1, 1)) -def test_summarize(): +def test_compute_summary_statistics(): """Check that the summarize utility function works as expected.""" results_multiple_draws = pd.DataFrame( @@ -604,10 +605,10 @@ def test_summarize(): columns=pd.MultiIndex.from_tuples( [ ("DrawA", "lower"), - ("DrawA", "mean"), + ("DrawA", "central"), ("DrawA", "upper"), ("DrawB", "lower"), - ("DrawB", "mean"), + ("DrawB", "central"), ("DrawB", "upper"), ], names=("draw", "stat"), @@ -620,7 +621,7 @@ def test_summarize(): ] ), ), - summarize(results_multiple_draws), + compute_summary_statistics(results_multiple_draws, central_measure='mean'), ) # Without collapsing and only mean @@ -630,19 +631,32 @@ def test_summarize(): index=["TimePoint0", "TimePoint1"], data=np.array([[10.0, 1500.0], [10.0, 1500.0]]), ), - summarize(results_multiple_draws, only_mean=True), + compute_summary_statistics(results_multiple_draws, central_measure='mean', only_central=True), ) # With collapsing (as only one draw) pd.testing.assert_frame_equal( pd.DataFrame( - columns=pd.Index(["lower", "mean", "upper"], name="stat"), + columns=pd.Index(["lower", "central", "upper"], name="stat"), index=["TimePoint0", "TimePoint1"], data=np.array([[0.5, 10.0, 19.5], [0.5, 10.0, 19.5], ]), ), - summarize(results_one_draw, collapse_columns=True), + compute_summary_statistics(results_one_draw, central_measure='mean', collapse_columns=True), ) + # Check that summarize() produces the expected legacy behaviour (i.e., uses mean) + pd.testing.assert_frame_equal( + compute_summary_statistics(results_multiple_draws, central_measure='mean').rename(columns={'central': 'mean'}, level=1), + summarize(results_multiple_draws) + ) + pd.testing.assert_frame_equal( + compute_summary_statistics(results_multiple_draws, central_measure='mean', only_central=True), + summarize(results_multiple_draws, only_mean=True) + ) + pd.testing.assert_frame_equal( + compute_summary_statistics(results_one_draw, central_measure='mean', collapse_columns=True), + summarize(results_one_draw, collapse_columns=True) + ) def test_control_loggers_from_same_module_independently(seed, tmpdir): """Check that detailed/summary loggers in the same module can configured independently.""" diff --git a/tests/test_contraception.py b/tests/test_contraception.py index 388b834393..6847165043 100644 --- a/tests/test_contraception.py +++ b/tests/test_contraception.py @@ -12,6 +12,7 @@ from tlo.methods import contraception, demography, enhanced_lifestyle, healthsystem, symptommanager from tlo.methods.contraception import HSI_Contraception_FamilyPlanningAppt from tlo.methods.hiv import DummyHivModule +from tlo.util import read_csv_files def run_sim(tmpdir, @@ -918,8 +919,8 @@ def test_input_probs_sum(): # Import relevant sheets from the workbook resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' - workbook = pd.read_excel(Path(resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', - sheet_name=None) + workbook = read_csv_files(Path(resourcefilepath) / 'contraception' / 'ResourceFile_Contraception', + files=None) sheet_names = [ 'Initiation_ByMethod', 'Interventions_Pop', diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index 6eeabc4995..875e3e03d4 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -1743,9 +1743,7 @@ def initialise_simulation(self, sim): tclose=None, priority=sim.modules['DummyModule'].rng.randint(0, 3)) - (list_of_individual_hsi_event_tuples_due_today, - list_of_population_hsi_event_tuples_due_today - ) = sim.modules['HealthSystem'].healthsystemscheduler._get_events_due_today() + list_of_individual_hsi_event_tuples_due_today = sim.modules['HealthSystem'].healthsystemscheduler._get_events_due_today() # Check that HealthSystemScheduler is recovering the correct number of events for today assert len(list_of_individual_hsi_event_tuples_due_today) == Ntoday diff --git a/tests/test_hiv.py b/tests/test_hiv.py index 5a27cf2c33..b3d61a25b8 100644 --- a/tests/test_hiv.py +++ b/tests/test_hiv.py @@ -35,6 +35,7 @@ HSI_Hiv_StartOrContinueTreatment, HSI_Hiv_TestAndRefer, ) +from tlo.util import read_csv_files try: resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' @@ -1210,8 +1211,7 @@ def test_baseline_hiv_prevalence(seed): # get data on 2010 prevalence # HIV resourcefile - xls = pd.ExcelFile(resourcefilepath / "ResourceFile_HIV.xlsx") - prev_data = pd.read_excel(xls, sheet_name="DHS_prevalence") + prev_data = read_csv_files(resourcefilepath / 'ResourceFile_HIV', files="DHS_prevalence") adult_prev_1549_data = prev_data.loc[ (prev_data.Year == 2010, "HIV prevalence among general population 15-49")].values[0] / 100 diff --git a/tests/test_htm_scaleup.py b/tests/test_htm_scaleup.py index fcb538f19c..95ee6e9509 100644 --- a/tests/test_htm_scaleup.py +++ b/tests/test_htm_scaleup.py @@ -19,6 +19,7 @@ symptommanager, tb, ) +from tlo.util import parse_csv_values_for_columns_with_mixed_datatypes, read_csv_files resourcefilepath = Path(os.path.dirname(__file__)) / "../resources" @@ -54,7 +55,8 @@ def get_sim(seed): def check_initial_params(sim): - original_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name='parameters') + original_params = read_csv_files(resourcefilepath / 'ResourceFile_HIV', files='parameters') + original_params.value = original_params.value.apply(parse_csv_values_for_columns_with_mixed_datatypes) # check initial parameters assert sim.modules["Hiv"].parameters["beta"] == \ @@ -73,8 +75,9 @@ def test_hiv_scale_up(seed): """ test hiv program scale-up changes parameters correctly and on correct date """ - original_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="parameters") - new_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="scaleup_parameters") + original_params = read_csv_files(resourcefilepath / 'ResourceFile_HIV', files="parameters") + original_params.value = original_params.value.apply(parse_csv_values_for_columns_with_mixed_datatypes) + new_params = read_csv_files(resourcefilepath / 'ResourceFile_HIV', files="scaleup_parameters") popsize = 100 @@ -104,10 +107,12 @@ def test_hiv_scale_up(seed): new_params.parameter == "prob_circ_after_hiv_test", "target_value"].values[0] # check malaria parameters unchanged - mal_original_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', - sheet_name="parameters") - mal_rdt_testing = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', - sheet_name="WHO_TestData2023") + mal_original_params = read_csv_files(resourcefilepath / 'malaria' / 'ResourceFile_malaria', + files="parameters") + mal_original_params.value = mal_original_params.value.apply(parse_csv_values_for_columns_with_mixed_datatypes) + + mal_rdt_testing = read_csv_files(resourcefilepath / 'malaria' / 'ResourceFile_malaria', + files="WHO_TestData2023") assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == mal_original_params.loc[ mal_original_params.parameter_name == "prob_malaria_case_tests", "value"].values[0] @@ -121,8 +126,9 @@ def test_hiv_scale_up(seed): mal_original_params.parameter_name == "itn", "value"].values[0] # check tb parameters unchanged - tb_original_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="parameters") - tb_testing = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="NTP2019") + tb_original_params = read_csv_files(resourcefilepath / 'ResourceFile_TB', files="parameters") + tb_original_params.value = tb_original_params.value.apply(parse_csv_values_for_columns_with_mixed_datatypes) + tb_testing = read_csv_files(resourcefilepath / 'ResourceFile_TB', files="NTP2019") pd.testing.assert_series_equal(sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"], tb_testing["treatment_coverage"]) @@ -143,8 +149,9 @@ def test_htm_scale_up(seed): and on correct date """ # Load data on HIV prevalence - original_hiv_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="parameters") - new_hiv_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="scaleup_parameters") + original_hiv_params = read_csv_files(resourcefilepath / 'ResourceFile_HIV', files="parameters") + original_hiv_params.value = original_hiv_params.value.apply(parse_csv_values_for_columns_with_mixed_datatypes) + new_hiv_params = read_csv_files(resourcefilepath / 'ResourceFile_HIV', files="scaleup_parameters") popsize = 100 @@ -178,8 +185,8 @@ def test_htm_scale_up(seed): new_hiv_params.parameter == "prob_circ_after_hiv_test", "target_value"].values[0] # check malaria parameters changed - new_mal_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', - sheet_name="scaleup_parameters") + new_mal_params = read_csv_files(resourcefilepath / 'malaria' / 'ResourceFile_malaria', + files="scaleup_parameters") assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == new_mal_params.loc[ new_mal_params.parameter == "prob_malaria_case_tests", "target_value"].values[0] @@ -193,7 +200,8 @@ def test_htm_scale_up(seed): new_mal_params.parameter == "itn", "target_value"].values[0] # check tb parameters changed - new_tb_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="scaleup_parameters") + new_tb_params = read_csv_files(resourcefilepath / 'ResourceFile_TB', files="scaleup_parameters") + new_tb_params.target_value = new_tb_params.target_value.apply(parse_csv_values_for_columns_with_mixed_datatypes) assert sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"].eq(new_tb_params.loc[ new_tb_params.parameter == "tb_treatment_coverage", "target_value"].values[0]).all()