Switch to just using PLUTO manual research for corrections

.github/workflows/publish.yml

@@ -37,11 +37,11 @@ jobs:
           - dcp_mappluto_wi
           - dcp_pluto
           - dcp_mappluto_clipped
-          - pluto_corrections
+          # - pluto_corrections
     steps:
       - uses: actions/checkout@v2
       - uses: NYCPlanning/[email protected]
-        if: matrix.dataset != 'pluto_corrections'
+        # if: matrix.dataset != 'pluto_corrections'
         with:
           name: ${{ matrix.dataset }}
           s3: true
@@ -51,7 +51,7 @@ jobs:
           version: ${{ needs.version.outputs.version }}
 
       - uses: NYCPlanning/[email protected]
-        if: matrix.dataset == 'pluto_corrections'
+        # if: matrix.dataset == 'pluto_corrections'
         with:
           name: ${{ matrix.dataset }}
           s3: true

pluto_build/01_dataloading.sh

@@ -48,7 +48,7 @@ import_public dcp_zoningmapamendments &
 import_public dcp_zoningmapindex &
 
 # import_public other
-import_public pluto_corrections &
+# import_public pluto_corrections &
 import_public dpr_greenthumb &
 import_public dsny_frequencies &
 import_public lpc_historic_districts &

pluto_build/01_dataloading_minor.sh

@@ -60,7 +60,7 @@ import_public lpc_historic_districts $LPC_HISTORIC_DISTRICTS_VERSION &
 import_public lpc_landmarks $LPC_LANDMARKS_VESRSION
 
 # import pluto corrections
-import_public pluto_corrections $PLUTO_CORRECTIONS_VERSION &
+# import_public pluto_corrections $PLUTO_CORRECTIONS_VERSION &
 
 wait
 

pluto_build/06_export.sh

@@ -5,12 +5,12 @@ mkdir -p output &&
   (cd output 
     echo "version: $VERSION" > version.txt
     echo "date: $DATE" >> version.txt
-    CSV_export pluto_corrections
+    # CSV_export pluto_changes
     CSV_export pluto_removed_records
-    CSV_export pluto_corrections_not_applied
-    CSV_export pluto_corrections_applied
-    zip pluto_corrections.zip *
-    ls | grep -v pluto_corrections.zip | xargs rm
+    CSV_export pluto_changes_not_applied
+    CSV_export pluto_changes_applied
+    zip pluto_changes.zip *
+    ls | grep -v pluto_changes.zip | xargs rm
   )
 
 mkdir -p output &&

pluto_build/sql/corr_communitydistrict.sql

@@ -1,29 +1,14 @@
 -- Take the community district value from research table
--- Insert records into pluto_corrections
-INSERT INTO pluto_corrections
-SELECT DISTINCT a.bbl, 
-	'cd' as field, 
-	a.cd as old_value, 
-	b.new_value as new_value,
-	b.type as type,
-	b.reason as reason,
-	b.version as version
-FROM pluto a, pluto_input_research b
-WHERE a.bbl = b.bbl
-	AND a.cd=b.old_value
-	AND b.field = 'cd'
-	AND a.bbl NOT IN (SELECT bbl FROM pluto_corrections WHERE field = 'cd');
-
-INSERT INTO pluto_corrections_not_applied
+INSERT INTO pluto_changes_not_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='cd'
 	AND b.old_value <> a.cd;
 
-INSERT INTO pluto_corrections_applied
+INSERT INTO pluto_changes_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='cd' 
 	AND b.old_value = a.cd;
@@ -32,7 +17,7 @@ WHERE b.bbl=a.bbl
 UPDATE pluto a
 SET cd = b.new_value,
 	dcpedited = 't'
-FROM pluto_corrections b
+FROM pluto_input_research b
 WHERE a.bbl = b.bbl
 	AND b.field = 'cd'
 	AND a.cd=b.old_value;

pluto_build/sql/corr_create.sql

@@ -2,8 +2,8 @@
 ALTER TABLE pluto
 ADD COLUMN dcpedited text;
 
-DROP TABLE IF EXISTS pluto_corrections_not_applied;
-CREATE TABLE pluto_corrections_not_applied (LIKE pluto_corrections INCLUDING ALL);
+DROP TABLE IF EXISTS pluto_changes_not_applied;
+CREATE TABLE pluto_changes_not_applied (LIKE pluto_input_research INCLUDING ALL);
 
-DROP TABLE IF EXISTS pluto_corrections_applied;
-CREATE TABLE pluto_corrections_applied (LIKE pluto_corrections INCLUDING ALL);
+DROP TABLE IF EXISTS pluto_changes_applied;
+CREATE TABLE pluto_changes_applied (LIKE pluto_input_research INCLUDING ALL);

pluto_build/sql/corr_dropoldrecords.sql

@@ -1,3 +1,3 @@
 -- remove inactive records from the corrections file
-DELETE FROM pluto_corrections
-WHERE bbl NOT IN (SELECT bbl FROM pluto);
+-- DELETE FROM pluto_corrections
+-- WHERE bbl NOT IN (SELECT bbl FROM pluto);

pluto_build/sql/corr_lotarea.sql

@@ -2,7 +2,7 @@
 -- compute the lot area from the geometry in sq feet
 
 -- Insert old and new values into tracking table
-INSERT INTO pluto_corrections
+INSERT INTO pluto_changes_applied
 SELECT DISTINCT 
 	a.bbl as bbl, 
 	'lotarea' as field, 
@@ -14,7 +14,7 @@ SELECT DISTINCT
 FROM pluto a
 WHERE a.lotarea = '0' 
 	AND a.geom IS NOT NULL
-	AND a.bbl NOT IN (SELECT bbl FROM pluto_corrections WHERE field = 'lotarea');
+	AND a.bbl NOT IN (SELECT bbl FROM pluto_input_research WHERE field = 'lotarea');
 
 -- -- Redundant code	
 -- -- Apply correction
@@ -28,30 +28,16 @@ WHERE a.lotarea = '0'
 -- 	AND a.geom IS NOT NULL;
 
 -- Take researched lot area values from the research table
-INSERT INTO pluto_corrections
-SELECT DISTINCT a.bbl, 
-	'lotarea' as field, 
-	a.lotarea as old_value, 
-	b.new_value as new_value,
-	b.type as type,
-	b.reason as reason,
-	b.version as version
-FROM pluto a, pluto_input_research b
-WHERE a.bbl = b.bbl
-	AND a.lotarea::numeric=b.old_value::numeric
-	AND b.field = 'lotarea'
-	AND a.bbl NOT IN (SELECT bbl FROM pluto_corrections WHERE field = 'lotarea');
-
-INSERT INTO pluto_corrections_not_applied
+INSERT INTO pluto_changes_not_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='lotarea' 
 	AND b.old_value::numeric <> a.lotarea::numeric;
 
-INSERT INTO pluto_corrections_applied
+INSERT INTO pluto_changes_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='lotarea' 
 	AND b.old_value::numeric=a.lotarea::numeric;
@@ -60,36 +46,22 @@ WHERE b.bbl=a.bbl
 UPDATE pluto a
 SET lotarea = b.new_value,
 	dcpedited = 't'
-FROM pluto_corrections b
+FROM pluto_changes_applied b
 WHERE a.bbl = b.bbl
 	AND b.field = 'lotarea'
 	AND a.lotarea::numeric=b.old_value::numeric;
 
 -- Take researched building area values from the research table
-INSERT INTO pluto_corrections
-SELECT DISTINCT a.bbl, 
-	'bldgarea' as field, 
-	a.bldgarea as old_value, 
-	b.new_value as new_value,
-	b.type as type,
-	b.reason as reason,
-	b.version as version
-FROM pluto a, pluto_input_research b
-WHERE a.bbl = b.bbl
-	AND a.bldgarea::numeric=b.old_value::numeric
-	AND b.field = 'bldgarea'
-	AND a.bbl NOT IN (SELECT bbl FROM pluto_corrections WHERE field = 'bldgarea');
-
-INSERT INTO pluto_corrections_not_applied
+INSERT INTO pluto_changes_not_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='bldgarea' 
 	AND b.old_value::numeric <> a.bldgarea::numeric;
 
-INSERT INTO pluto_corrections_applied
+INSERT INTO pluto_changes_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='bldgarea' 
 	AND b.old_value::numeric=a.bldgarea::numeric;
@@ -98,7 +70,7 @@ WHERE b.bbl=a.bbl
 UPDATE pluto a
 SET bldgarea = b.new_value,
 	dcpedited = 't'
-FROM pluto_corrections b
+FROM pluto_input_research b
 WHERE a.bbl = b.bbl
 	AND b.field = 'bldgarea'
 	AND a.bldgarea::numeric=b.old_value::numeric;
@@ -107,4 +79,4 @@ WHERE a.bbl = b.bbl
 UPDATE pluto
 SET builtfar = round(bldgarea::numeric / lotarea::numeric, 2)
 WHERE lotarea <> '0' AND lotarea IS NOT NULL
-AND bbl IN (SELECT bbl FROM pluto_corrections WHERE field = 'lotarea' OR field = 'bldgarea');
+AND bbl IN (SELECT bbl FROM pluto_changes_applied WHERE field = 'lotarea' OR field = 'bldgarea');

pluto_build/sql/corr_numfloors.sql

@@ -1,30 +1,16 @@
 -- Take the numfloors value from research table
--- Insert records into pluto_corrections
-INSERT INTO pluto_corrections
-SELECT DISTINCT a.bbl, 
-	'numfloors' as field, 
-	a.numfloors as old_value, 
-	b.new_value as new_value,
-	b.type as type,
-	b.reason as reason,
-	b.version as version
-FROM pluto a, pluto_input_research b
-WHERE a.bbl = b.bbl
-	AND a.numfloors=b.old_value
-	AND b.field = 'numfloors'
-	AND a.bbl NOT IN (SELECT bbl FROM pluto_corrections WHERE field = 'numfloors');
 
 -- Take note of corrections not applied to pluto
-INSERT INTO pluto_corrections_not_applied
+INSERT INTO pluto_changes_not_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='numfloors' 
 	AND b.old_value <> a.numfloors;
 
-INSERT INTO pluto_corrections_applied
+INSERT INTO pluto_changes_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='numfloors' 
 	AND b.old_value = a.numfloors;
@@ -33,7 +19,7 @@ WHERE b.bbl=a.bbl
 UPDATE pluto a
 SET numfloors = b.new_value,
 	dcpedited = 't'
-FROM pluto_corrections b
+FROM pluto_input_research b
 WHERE a.bbl = b.bbl
 	AND b.field = 'numfloors'
 	AND a.numfloors=b.old_value;

pluto_build/sql/corr_ownername_city.sql

@@ -1,6 +1,6 @@
 -- Take the owner name from the normalized list
 -- Insert records into pluto_input_corrections
-INSERT INTO pluto_corrections
+INSERT INTO pluto_changes_applied
 SELECT DISTINCT a.bbl, 
 	'ownername' as field, 
 	a.ownername as old_value, 
@@ -10,27 +10,27 @@ SELECT DISTINCT a.bbl,
 	b.version as version
 FROM pluto a, pluto_input_research b
 WHERE a.ownername = b.old_value
-	AND a.bbl NOT IN (SELECT bbl FROM pluto_corrections WHERE field = 'ownername');
+	AND a.bbl NOT IN (SELECT bbl FROM pluto_changes_applied WHERE field = 'ownername');
 
-INSERT INTO pluto_corrections_not_applied
+INSERT INTO pluto_changes_not_applied
 SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
+FROM pluto_input_research b, pluto a
 WHERE b.bbl=a.bbl 
 	AND b.field='ownername' 
 	AND b.old_value <> a.ownername;
 
-INSERT INTO pluto_corrections_applied
-SELECT DISTINCT b.*
-FROM pluto_corrections b, pluto a
-WHERE b.bbl=a.bbl 
-	AND b.field='ownername' 
-	AND b.old_value = a.ownername;
+-- INSERT INTO pluto_corrections_applied
+-- SELECT DISTINCT b.*
+-- FROM pluto_corrections b, pluto a
+-- WHERE b.bbl=a.bbl 
+-- 	AND b.field='ownername' 
+-- 	AND b.old_value = a.ownername;
 
 -- Apply correction to PLUTO
 UPDATE pluto a
 SET ownername = b.new_value,
 	dcpedited = 't'
-FROM pluto_corrections b
+FROM pluto_input_research b
 WHERE a.ownername = b.old_value
 AND b.field = 'ownername';