Skip to content

Commit

Permalink
Merge pull request #108 from USEPA/matchTADA
Browse files Browse the repository at this point in the history 
flag functions
  • Loading branch information
@cristinamullin
cristinamullin authored Sep 15, 2023
2 parents 9c0dd44 + 4e597e8 commit 3ef9139
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 14 deletions.
5 changes: 5 additions & 0 deletions R/utils_flag_functions.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ applyFlags <- function(in_table, orgs) {
# QC rep/blank
out <- TADA::TADA_FindQCActivities(out, clean = FALSE)

# Result is flagged as suspect by data submitter
out <- TADA::TADA_FlagMeasureQualifierCode(out, clean = FALSE)

# Invalid analytical method
out <- TADA::TADA_FlagMethod(out, clean = FALSE)

Expand Down Expand Up @@ -126,11 +129,13 @@ applyFlags <- function(in_table, orgs) {
# Convert depth height units - THIS ONE ONLY GETS RUN WHEN USER RUNS THE CLEANING
# FILTER AFTER MAKING ALL DECISIONS, AND SUMMARY COUNTS BASED ON UNIQUE UNITS IN
# DEPTH HEIGHT COLUMNS
# CM - this is already done via TADA autoclean 9/12/23, simply add note somewhere for users?
# out <-
# TADA::TADA_ConvertDepthUnits(out, unit = 'ft', transform = TRUE) # input$depthunit is dummy variable that would connect to the drop down

# Convert time zones - no flag function to run beforehand. This one might be
# tricky to implement - acts on ActivityStartTime.Time?
# CM - this is already done via dataRetrieval 9/12/23, simply add note somewhere for users?
#out = out

# Invalid coordinates - not included in mock up page?
Expand Down
17 changes: 9 additions & 8 deletions inst/flag_prompts.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ Order,Level,Prompt,flagType
5,Recommended,Invalid characteristic and fraction combination,Invalid characteristic and fraction combination
6,Recommended,Invalid units for a given characteristic and media combination,Invalid units for a given characteristic and media combination
7,Recommended,"Measurement activity type code indicates it is a QC replicate, duplicate, or blank","Measurement activity type code indicates it is a QC replicate, duplicate, or blank"
8,Optional,Metadata indicates duplicative uploads of the same results within a single organization,Metadata indicates duplicative uploads of the same results within a single organization
9,Optional,Metadata indicates duplicative uploads of the same results by multiple organizations,Metadata indicates duplicative uploads of the same results by multiple organizations
10,Optional,Uncommon analytical methods,Uncommon analytical methods
11,Optional,"QAPP approval column is ""N"" (indicating not approved) or NA","QAPP approval column is ""N"" (indicating not approved) or NA"
12,Optional,Quality Assurance Project Plan URL is missing,Quality Assurance Project Plan URL is missing
13,Optional,"Metadata indicates result(s) are aggregate, high-frequency continuous sensor results, not discrete samples","Metadata indicates result(s) are aggregate, high-frequency continuous sensor results, not discrete samples"
14,Optional,"Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)","Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)"
15,Optional,"Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)","Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)"
8,Recommended,MeasureQualifierCode indicates result value is suspect,MeasureQualifierCode indicates result value is suspect
9,Optional,Metadata indicates duplicative uploads of the same results within a single organization,Metadata indicates duplicative uploads of the same results within a single organization
10,Optional,Metadata indicates duplicative uploads of the same results by multiple organizations,Metadata indicates duplicative uploads of the same results by multiple organizations
11,Optional,Uncommon analytical methods,Uncommon analytical methods
12,Optional,"QAPP approval column is ""N"" (indicating not approved) or NA","QAPP approval column is ""N"" (indicating not approved) or NA"
13,Optional,Quality Assurance Project Plan URL is missing,Quality Assurance Project Plan URL is missing
14,Optional,"Metadata indicates result(s) are aggregate, high-frequency continuous sensor results, not discrete samples","Metadata indicates result(s) are aggregate, high-frequency continuous sensor results, not discrete samples"
15,Optional,"Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)","Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)"
16,Optional,"Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)","Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)"
19 changes: 13 additions & 6 deletions inst/flag_tests.csv
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
flagType,columnName,flagValue,keep,remove
"Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)",TADA.ResultValueAboveUpperThreshold.Flag,Y,0,1
"Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)",TADA.ResultValueAboveUpperThreshold.Flag,No threshold avilable,0,0
Uncommon analytical methods,TADA.AnalyticalMethod.Flag,Nonstandardized,0,0
"Result value(s) outside of the national upper range for a given characteristic, possibly indicating non-sensical value(s)",TADA.ResultValueAboveUpperThreshold.Flag,Not Reviewed,0,0
Uncommon analytical methods,TADA.AnalyticalMethod.Flag,Not Reviewed,0,0
Uncommon analytical methods,TADA.AnalyticalMethod.Flag,Invalid,0,1
Uncommon analytical methods,TADA.AnalyticalMethod.Flag,Valid,0,0
"Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)",TADA.ResultValueBelowLowerThreshold.Flag,Y,0,1
"Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)",TADA.ResultValueBelowLowerThreshold.Flag,No threshold avilable,0,0
"Result value(s) outside of the national lower range for a given characteristic, possibly indicating non-sensical value(s)",TADA.ResultValueBelowLowerThreshold.Flag,Not Reviewed,0,0
Conflict between detection condition text and detection limit type or detection limit type is not in WQX domain tables (likely USGS/NWIS-specific) ,TADA.CensoredData.Flag,Detection condition or detection limit is not documented in TADA reference tables.,0,1
Conflict between detection condition text and detection limit type or detection limit type is not in WQX domain tables (likely USGS/NWIS-specific) ,TADA.CensoredData.Flag,Detection condition is missing and required for censored data ID.,0,1
Conflict between detection condition text and detection limit type or detection limit type is not in WQX domain tables (likely USGS/NWIS-specific) ,TADA.CensoredData.Flag,Conflict between Condition and Limit,0,1
Expand All @@ -15,16 +16,16 @@ Conflict between detection condition text and detection limit type or detection
"Metadata indicates result(s) are aggregate, high-frequency continuous sensor results, not discrete samples",TADA.AggregatedContinuousData.Flag,Y,0,1
Sample media is not water,TADA.ActivityMediaName,WATER,1,1
Invalid speciation for associated characteristic,TADA.MethodSpeciation.Flag,Valid,0,0
Invalid speciation for associated characteristic,TADA.MethodSpeciation.Flag,Nonstandardized,0,0
Invalid speciation for associated characteristic,TADA.MethodSpeciation.Flag,Invalid,0,1
Invalid speciation for associated characteristic,TADA.MethodSpeciation.Flag,Not Reviewed,0,0
"QAPP approval column is ""N"" (indicating not approved) or NA",QAPPApprovedIndicator,NA,0,1
"QAPP approval column is ""N"" (indicating not approved) or NA",QAPPApprovedIndicator,Y,0,0
"QAPP approval column is ""N"" (indicating not approved) or NA",QAPPApprovedIndicator,N,0,1
Quality Assurance Project Plan URL is missing,TADA.QAPPDocAvailable,Y_ProjectFileUrlProvided,0,0
Quality Assurance Project Plan URL is missing,TADA.QAPPDocAvailable,N,0,1
Invalid characteristic and fraction combination,TADA.SampleFraction.Flag,Valid,0,0
Invalid characteristic and fraction combination,TADA.SampleFraction.Flag,Nonstandardized,0,0
Invalid characteristic and fraction combination,TADA.SampleFraction.Flag,Invalid,0,1
Invalid characteristic and fraction combination,TADA.SampleFraction.Flag,Not Reviewed,0,0
Spatial Flags,TADA.InvalidCoordinates.Flag,LAT_OutsideUSA,0,1
Spatial Flags,TADA.InvalidCoordinates.Flag,LONG_OutsideUSA,0,1
Spatial Flags,TADA.InvalidCoordinates.Flag,Imprecise_Latincludes999,0,0
Expand All @@ -44,8 +45,9 @@ Result value is not numeric or result value is NA and no detection limit value i
Result value is not numeric or result value is NA and no detection limit value is provided,TADA.ResultMeasureValueDataTypes.Flag,Result Value/Unit Estimated from Detection Limit,0,0
Result value is not numeric or result value is NA and no detection limit value is provided,TADA.ResultMeasureValueDataTypes.Flag,Text,0,1
Invalid units for a given characteristic and media combination,TADA.ResultUnit.Flag,Valid,0,0
Invalid units for a given characteristic and media combination,TADA.ResultUnit.Flag,Nonstandardized,0,0
Invalid units for a given characteristic and media combination,TADA.ResultUnit.Flag,NonStandardized,0,0
Invalid units for a given characteristic and media combination,TADA.ResultUnit.Flag,Invalid,0,1
Invalid units for a given characteristic and media combination,TADA.ResultUnit.Flag,Not Reviewed,0,0
"Measurement activity type code indicates it is a QC replicate, duplicate, or blank",TADA.ActivityType.Flag,Non_QC,0,0
"Measurement activity type code indicates it is a QC replicate, duplicate, or blank",TADA.ActivityType.Flag,QC_replicate,0,1
"Measurement activity type code indicates it is a QC replicate, duplicate, or blank",TADA.ActivityType.Flag,QC_duplicate,0,1
Expand All @@ -54,3 +56,8 @@ Invalid units for a given characteristic and media combination,TADA.ResultUnit.F
"Measurement activity type code indicates it is a QC replicate, duplicate, or blank",TADA.ActivityType.Flag,QC_other,0,1
Metadata indicates duplicative uploads of the same results within a single organization,TADA.ResultSelectedSingleOrg,N,0,1
Metadata indicates duplicative uploads of the same results by multiple organizations,TADA.ResultSelectedMultipleOrgs,N,0,1
MeasureQualifierCode indicates result value is suspect,TADA.MeasureQualifierCode.Flag,Non-Detect,0,0
MeasureQualifierCode indicates result value is suspect,TADA.MeasureQualifierCode.Flag,Pass,0,0
MeasureQualifierCode indicates result value is suspect,TADA.MeasureQualifierCode.Flag,Suspect,0,1
MeasureQualifierCode indicates result value is suspect,TADA.MeasureQualifierCode.Flag,Over-Detect,0,0
MeasureQualifierCode indicates result value is suspect,TADA.MeasureQualifierCode.Flag,Not Reviewed,0,0

0 comments on commit 3ef9139

Please sign in to comment.