From bcd48477d0810f6c81955d6744a03e5d22dd7961 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Wed, 22 May 2024 13:34:19 -0400 Subject: [PATCH] create data network configure a network the data flow etl can connect to configure the database before running migrations closes nycplanning/ae-data-flow#53 --- .github/workflows/staging.yml | 2 + README.md | 36 ++++++++++-------- compose.yaml | 6 ++- db/Dockerfile | 8 ---- db/query/copy.sql | 72 ----------------------------------- package.json | 1 - sample.env | 3 +- 7 files changed, 29 insertions(+), 99 deletions(-) delete mode 100644 db/query/copy.sql diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 41dfa4cf..a7d54b53 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -26,6 +26,8 @@ jobs: node-version: 18.x - name: Install dependencies run: npm ci --ignore-scripts + - name: Configure database + run: npm run pg:configure - name: Apply migrations run: npm run drizzle:migrate deploy: diff --git a/README.md b/README.md index ee22b2fb..d0dda5ef 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ docker compose up If you need to install docker compose, follow [these instructions](https://docs.docker.com/compose/install/). ### Configure PostGIS -Enable geospatial features by installing the postgis extension +Enable geospatial features by activating the postgis extension ```sh npm run pg:configure @@ -39,13 +39,6 @@ Structure the database tables with drizzle npm run drizzle:migrate ``` -### Data loading -Load the data into the tables. Under the hood, this uses the Postgres COPY command. - -```sh -npm run pg:copy -``` - ## Local development The OpenAPI documentation and the API implementation complement each other. The OpenAPI documentation is written first, defining expectations for endpoints. @@ -78,13 +71,6 @@ Generate these resources with: npm run kubb:generate ``` -The console may include the output below. It can be safely ignored. -```console -✖ Something went wrong - - Error: The "paths[0]" argument must be of type string. Received undefined -``` - Finally, to run the api locally: ```sh npm run dev @@ -104,6 +90,26 @@ If you make changes to any database schema in `src/schema`, it will be necessary npm run drizzle:generate ``` +### Data loading +This API manages its own database schemas (using the drizzle steps listed above). However, the data itself is loaded into the database through a separate ETL repository. The ETL repository is `NYCPlanning/ae-data-flow`. +For local development, the ETL communicates with the Zoning API through a shared docker network. The Zoning API docker compose creates this network, which the ETL repository then looks for and joins when it starts. + +#### Troubleshooting the docker network + +The docker network should be created automatically when bringing the Zoning API `up` and the ETL should also join it automatically. However, there are two common scenarios that may cause issues. + +The ETL cannot find the network: +- Check the configuration of the `compose.yaml` in this repository (NYCPlanning/ae-zoning-api). It should be configured to have a network called `data`. +- Configure the compose to create a `data` network, if it does not already. +- If it does already configure this network, attempt to rebuild the container with `docker compose up --build -d` + +The ETL cannot find the Zoning Database on the network: +- In this scenario, the network exists from a previous build. However, the Zoning API is failing to join it when it starts. +- First confirm the network exists. + - Run `docker network ls` + - Confirm there is a network called `ae-zoning-api_data` + - Connect the Zoning API database container to the data network by running `docker network connect ae-zoning-api_data ae-zoning-api-db-1` + ## Production builds Running a production version of the site is a two step process. First, generate production versions of the OpenAPI documentation and API. diff --git a/compose.yaml b/compose.yaml index f89472ce..f62e8c8d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,13 +2,15 @@ services: db: build: context: ./db - args: - - STORAGE_URL=${STORAGE_URL} environment: - POSTGRES_USER=${DATABASE_USER} - POSTGRES_PASSWORD=${DATABASE_PASSWORD} - POSTGRES_DB=${DATABASE_NAME} ports: - "8010:5432" + networks: + - data volumes: - ./db-volume:/var/lib/postgresql/data +networks: + data: diff --git a/db/Dockerfile b/db/Dockerfile index 5ea08d84..60d6b7a8 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -4,11 +4,3 @@ RUN apt update RUN apt install -y postgresql-15-postgis-3 WORKDIR /var/lib/postgresql - -ARG STORAGE_URL -ADD --chown=postgres $STORAGE_URL/tables/borough.csv borough.csv -ADD --chown=postgres $STORAGE_URL/tables/land_use.csv land_use.csv -ADD --chown=postgres $STORAGE_URL/tables/tax_lot.csv tax_lot.csv -ADD --chown=postgres $STORAGE_URL/tables/zoning_district.csv zoning_district.csv -ADD --chown=postgres $STORAGE_URL/tables/zoning_district_class.csv zoning_district_class.csv -ADD --chown=postgres $STORAGE_URL/tables/zoning_district_zoning_district_class.csv zoning_district_zoning_district_class.csv diff --git a/db/query/copy.sql b/db/query/copy.sql deleted file mode 100644 index 3066b21a..00000000 --- a/db/query/copy.sql +++ /dev/null @@ -1,72 +0,0 @@ -BEGIN; - COPY borough ("id", "title", "abbr") - FROM '../borough.csv' - DELIMITER ',' - CSV HEADER; - -COMMIT; - -BEGIN; - COPY land_use ("id", "description", "color") - FROM '../land_use.csv' - DELIMITER ',' - CSV HEADER; - -COMMIT; - -BEGIN; - COPY tax_lot ( - "bbl", - "borough_id", - "block", - "lot", - "address", - "land_use_id", - "wgs84", - "li_ft" - ) - FROM '../tax_lot.csv' - DELIMITER ',' - CSV HEADER; - -COMMIT; - - -BEGIN; - COPY zoning_district ( - "id", - "label", - "wgs84", - "li_ft" - ) - FROM '../zoning_district.csv' - DELIMITER ',' - CSV HEADER; - -COMMIT; - - -BEGIN; - COPY zoning_district_class ( - "id", - "category", - "description", - "url", - "color" - ) - FROM '../zoning_district_class.csv' - DELIMITER ',' - CSV HEADER; - -COMMIT; - -BEGIN; - COPY zoning_district_zoning_district_class ( - "zoning_district_id", - "zoning_district_class_id" - ) - FROM '../zoning_district_zoning_district_class.csv' - DELIMITER ',' - CSV HEADER; - -COMMIT; \ No newline at end of file diff --git a/package.json b/package.json index 694931da..789f9a1b 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,6 @@ "start:prod": "node dist/main", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "pg:configure": "ts-node ./db/query.ts configure", - "pg:copy": "ts-node ./db/query.ts copy", "drizzle:migrate": "drizzle-kit migrate", "drizzle:generate": "drizzle-kit generate", "redoc:build": "redocly build-docs openapi/openapi.yaml --output openapi/index.html", diff --git a/sample.env b/sample.env index be785c4d..38a155f7 100644 --- a/sample.env +++ b/sample.env @@ -4,4 +4,5 @@ DATABASE_NAME=zoning DATABASE_PORT=8010 DATABASE_HOST=localhost DATABASE_ENV=development -STORAGE_URL= + +STORAGE_URL= \ No newline at end of file