project.yaml

version: '3.0'

expectations:
  population_size: 10000

actions:

  ####################
  # Curation
  ####################
  generate_study_population_curation:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_curation --output-format=csv.gz --output-dir output/curation --index-date-range "2019-03-01 to 2019-03-01 by month"
    outputs:
      highly_sensitive:
        cohort: output/curation/input_curation_*.csv.gz

  generate_dataset_report_curation:
    run: >
      python:latest python analysis/dataset_report.py
        --input-files output/curation/input_curation_*.csv.gz
        --output-dir output/curation/
    needs: [generate_study_population_curation]
    outputs:
      moderately_sensitive:
        dataset_report: output/curation/input_curation_*.html

  # To count exclusions
  generate_study_population_exclusions:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_exclusions --output-format=csv.gz --output-dir output/curation/exclusions --index-date-range "2022-10-01 to 2022-10-01 by month"
    outputs:
      highly_sensitive:
        cohort: output/curation/exclusions/input_exclusions_*.csv.gz

  generate_measures_exclusions:
      run: cohortextractor:latest generate_measures --study-definition study_definition_exclusions --output-dir=output/curation/exclusions
      needs: [generate_study_population_exclusions]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/curation/exclusions/measure_*_rate.csv

  join_measures_exclusions:
      run: python:latest python analysis/join_and_round.py
           --input-files output/curation/exclusions/measure_*rate.csv
           --output-dir output/curation/exclusions
           --output-name "exclusions.csv"
      needs: [generate_measures_exclusions]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/curation/exclusions/exclusions.csv

  # For comparison with openprescribing
  generate_study_population_num_matches_2018_2019:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_num_matches --output-format=csv.gz --output-dir output/curation --index-date-range "2018-01-01 to 2019-12-31 by month"
    outputs:
      highly_sensitive:
        cohort: output/curation/input_num_matches_201*.csv.gz

  generate_study_population_num_matches_2020_2021:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_num_matches --output-format=csv.gz --output-dir output/curation --index-date-range "2020-01-01 to 2021-12-31 by month"
    outputs:
      highly_sensitive:
        cohort: output/curation/input_num_matches_20*.csv.gz

  generate_study_population_num_matches_2022_2023:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_num_matches --output-format=csv.gz --output-dir output/curation --index-date-range "2022-01-01 to 2023-03-01 by month"
    outputs:
      highly_sensitive:
        cohort: output/curation/input_num_matches_2*.csv.gz

  generate_measures_num_matches:
      run: cohortextractor:latest generate_measures --study-definition study_definition_num_matches --output-dir=output/curation
      needs: [generate_study_population_num_matches_2018_2019, generate_study_population_num_matches_2020_2021, generate_study_population_num_matches_2022_2023]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/curation/measure_*_rate.csv

  join_measures_num_matches:
      run: python:latest python analysis/join_and_round.py
           --input-files output/curation/measure_antidepressant_any_all_total_events_rate.csv
           --output-dir output/curation/
           --output-name "measure_events.csv"
      needs: [generate_measures_num_matches]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/curation/measure_events.csv

  # To compute prevalence
  generate_study_population_prevalence:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_prevalence --output-format=csv.gz --output-dir output/curation --index-date-range "2021-03-01"
    outputs:
      highly_sensitive:
        cohort: output/curation/input_prevalence_2021-03-01.csv.gz

  generate_measures_prevalence:
      run: cohortextractor:latest generate_measures --study-definition study_definition_prevalence --output-dir=output/curation
      needs: [generate_study_population_prevalence]
      outputs:
        moderately_sensitive:
          measure_csv: output/curation/measure_antidepressant_any_learning_disability_total_prevalence.csv

  join_measures_prevalence:
      run: python:latest python analysis/join_and_round.py
           --input-files output/curation/measure_antidepressant_any_learning_disability_total_prevalence.csv
           --output-dir output/curation/
           --output-name "measure_prevalence.csv"
      needs: [generate_measures_prevalence]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/curation/measure_prevalence.csv


  ####################
  # Cohort Generation
  ####################

  # Since this runs on everyone, we can reuse for both studies 
  generate_study_population_ethnicity:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_ethnicity --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_ethnicity.csv.gz

  # Generate dataset report ethnicity
  generate_dataset_report_ethnicity:
    run: >
      python:latest python analysis/dataset_report.py
        --input-files output/input_*.csv.gz
        --output-dir output/
    needs: [generate_study_population_ethnicity]
    outputs:
      moderately_sensitive:
        dataset_report: output/input_ethnicity.html


  # Generate prescription variables by month
  generate_study_population_lda_2018_01:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_lda --index-date-range "2018-01-01 to 2018-01-01 by month" --output-format=csv.gz --output-dir output/lda/codelist_update
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/input_lda_2018-01*.csv.gz

  generate_study_population_lda_2018:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_lda --index-date-range "2018-02-01 to 2018-12-01 by month" --output-format=csv.gz --output-dir output/lda/codelist_update
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/input_lda_2018*.csv.gz

  # Generate prescription variables by month
  generate_study_population_lda_2019:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_lda --index-date-range "2019-01-01 to 2019-12-01 by month" --output-format=csv.gz --output-dir output/lda/codelist_update
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/input_lda_2019*.csv.gz

  generate_study_population_lda_2020:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_lda --index-date-range "2020-01-01 to 2020-12-01 by month" --output-format=csv.gz --output-dir output/lda/codelist_update
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/input_lda_2020*.csv.gz

  generate_study_population_lda_2021:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_lda --index-date-range "2021-01-01 to 2021-12-01 by month" --output-format=csv.gz --output-dir output/lda/codelist_update
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/input_lda_2021*.csv.gz

  generate_study_population_lda_2022:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_lda --index-date-range "2022-01-01 to 2022-12-01 by month" --output-format=csv.gz --output-dir output/lda/codelist_update
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/input_lda_2022*.csv.gz

  # Generate dataset report lda
  generate_dataset_report_lda:
    run: >
      python:latest python analysis/dataset_report.py
        --input-files output/lda/codelist_update/input_*.csv.gz
        --output-dir output/lda/codelist_update
    needs: [generate_study_population_lda_2018_01]
    outputs:
      moderately_sensitive:
        dataset_report: output/lda/codelist_update/input_*.html

  # Count prescription metrics
  test_lda:
    run: >
      python:latest python analysis/test_lda.py
        --input-files output/lda/codelist_update/input_*.csv.gz
        --output-dir output/lda/codelist_update
    needs: [generate_study_population_lda_2018_01, generate_study_population_lda_2018, generate_study_population_lda_2019, generate_study_population_lda_2020, generate_study_population_lda_2021, generate_study_population_lda_2022]
    outputs:
      moderately_sensitive:
        dataset_report: output/lda/codelist_update/test_*.csv

  # Count prescription metrics
  test_lda_cohort_sample:
    run: >
      python:latest python analysis/test_lda_study.py
        --input-files output/lda/codelist_update/input_*.csv.gz
        --output-dir output/lda/codelist_update
        --cohort-size 1000
    needs: [generate_study_population_lda_2018_01, generate_study_population_lda_2018, generate_study_population_lda_2019, generate_study_population_lda_2020, generate_study_population_lda_2021, generate_study_population_lda_2022]
    outputs:
      moderately_sensitive:
        dataset_report: output/lda/codelist_update/test_study_period.csv

  ####################
  # Join ethnicity to all generated input files
  # Efficiency fix https://github.com/opensafely/research-template
  # BUT BEWARE STALE DATA
  ###################
  join_cohorts_lda:
    run: >
      cohort-joiner:v0.0.56
        --lhs output/lda/codelist_update/input_*.csv.gz
        --rhs output/input_ethnicity.csv.gz
        --output-dir output/lda/codelist_update/joined
    needs: [generate_study_population_ethnicity, generate_study_population_lda_2018_01, generate_study_population_lda_2018, generate_study_population_lda_2019, generate_study_population_lda_2020, generate_study_population_lda_2021, generate_study_population_lda_2022]
    outputs:
      highly_sensitive:
        cohort: output/lda/codelist_update/joined/input_*.csv.gz

  ####################
  # Measures
  ####################
  generate_measures_lda:
      run: cohortextractor:latest generate_measures --study-definition study_definition_lda --output-dir=output/lda/codelist_update/joined/
      needs: [join_cohorts_lda]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/lda/codelist_update/joined/measure_*_rate.csv
          count_csv: output/lda/codelist_update/joined/measure_*_count.csv

  join_measures_lda:
      run: python:latest python analysis/join_and_round.py
           --input-files output/lda/codelist_update/joined/measure_*.csv
           --output-dir output/lda/codelist_update/joined/summary
           --output-name "measure_lda.csv"
      needs: [generate_measures_lda]
      outputs:
        moderately_sensitive:
          # Only output the single summary file
          measure_csv: output/lda/codelist_update/joined/summary/measure_lda.csv

  #############################
  # Tables and Figures
  #############################
  # Table 1
  generate_table1:
    run: >
            python:v1 python analysis/table1.py
            --input-file output/lda/codelist_update/joined/summary/measure_lda.csv
            --output-dir output/lda/codelist_update/joined/summary
            --measures-pattern "antidepressant_any_all_breakdown_*"
            --measures-pattern "antidepressant_any_autism_breakdown_*"
            --measures-pattern "antidepressant_any_learning_disability_breakdown_*"
            --include-denominator
            --include-rate
            --column-names "all" "learning_disability" "autism"
            --output-name "table1.html"
            --start-date "2022-10-01"
    needs: [join_measures_lda]
    outputs:
      moderately_sensitive:
        cohort: output/lda/codelist_update/joined/summary/table1.html

  # Table 2
  generate_prescription_table:
    run: >
            python:v1 python analysis/table1.py
            --input-file output/lda/codelist_update/joined/summary/measure_lda.csv
            --output-dir output/lda/codelist_update/joined/summary
            --measures-pattern "antidepressant_any_all_breakdown_prescription_count"
            --measures-pattern "antidepressant_any_autism_breakdown_prescription_count"
            --measures-pattern "antidepressant_any_learning_disability_breakdown_prescription_count"
            --column-names "all" "learning_disability" "autism"
            --output-name "prescription.html"
            --exclude-missing
            --start-date "2022-10-01"
    needs: [join_measures_lda]
    outputs:
      moderately_sensitive:
        cohort: output/lda/codelist_update/joined/summary/prescription.html

  # Interrupted Time Series Analysis
  # NOTE: we use statsmodels.tsa.deterministic.Fourier
  # Available statsmodels 12+, so cannot currently run server/ OS python image
  #run_itsa:
  #  run: >
  #          python:v1 python analysis/its.py
  #          --input-file output/lda/codelist_update/joined/summary/measure_lda.csv
  #          --output-dir output/lda/codelist_update/joined/summary
  #  needs: [join_measures_lda]
  #  outputs:
  #    moderately_sensitive:
  #      pngs: output/lda/codelist_update/joined/summary/*.png
  #      htmls: output/lda/codelist_update/joined/summary/*.html
  #      csvs: output/lda/codelist_update/joined/summary/*.csv

  # Supplemental Table 1
  generate_exclusions_table:
    run: >
            python:v1 python analysis/table1.py
            --input-file output/curation/exclusions/exclusions.csv
            --output-dir output/curation/exclusions
            --measures-pattern "antidepressant_any*"
            --column-names "all" "learning_disability" "autism"
            --output-name "exclusions_table.csv"
            --start-date "2022-10-01"
            --output-type "csv"
            --combine
            --no-overall
            --include-denominator
    needs: [join_measures_exclusions]
    outputs:
      moderately_sensitive:
        cohort: output/curation/exclusions/exclusions_table.csv