diff --git a/analytics/Makefile b/analytics/Makefile index 4ea1dac93..1926bba4f 100644 --- a/analytics/Makefile +++ b/analytics/Makefile @@ -150,6 +150,12 @@ db-migrate: $(POETRY) analytics etl db_migrate @echo "=====================================================" +opportunity-load: + @echo "=> Ingesting opportunity data into the database" + @echo "=====================================================" + $(POETRY) analytics etl opportunity-load + @echo "=====================================================" + gh-transform-and-load: @echo "=> Transforming and loading GitHub data into the database" @echo "=====================================================" diff --git a/analytics/local.env b/analytics/local.env index 1943a56b8..57a4b9893 100644 --- a/analytics/local.env +++ b/analytics/local.env @@ -52,3 +52,9 @@ LOG_FORMAT=human-readable # Change the message length for the human readable formatter # LOG_HUMAN_READABLE_FORMATTER__MESSAGE_WIDTH=50 + +############################ +# S3 +############################ + +LOAD_OPPORTUNITY_DATA_FILE_PATH=S3://local-opportunities/public-extracts diff --git a/analytics/poetry.lock b/analytics/poetry.lock index 87a57fe1b..1773d0e1d 100644 --- a/analytics/poetry.lock +++ b/analytics/poetry.lock @@ -1977,6 +1977,52 @@ files = [ {file = "mistune-3.0.2.tar.gz", hash = "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"}, ] +[[package]] +name = "moto" +version = "5.0.23" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "moto-5.0.23-py3-none-any.whl", hash = "sha256:a8069f9c945e7503c43eccec30693f5656e0f8efb0256dfd814d99dedc38429e"}, + {file = "moto-5.0.23.tar.gz", hash = "sha256:8a32636647e45a9b76c32de0ed15c4b083c62849993217f96aa60026a2ca1721"}, +] + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.14.0,<1.35.45 || >1.35.45,<1.35.46 || >1.35.46" +cryptography = ">=35.0.0" +Jinja2 = ">=2.10.1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.15.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "jsonschema", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=3.0.0)"] +batch = ["docker (>=3.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.6)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.6)"] +events = ["jsonpath-ng"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +quicksight = ["jsonschema"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.6)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.6)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +ssm = ["PyYAML (>=5.1)"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath-ng"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + [[package]] name = "mypy" version = "1.13.0" @@ -3105,6 +3151,25 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "responses" +version = "0.25.3" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "responses-0.25.3-py3-none-any.whl", hash = "sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb"}, + {file = "responses-0.25.3.tar.gz", hash = "sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] + [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -3431,6 +3496,31 @@ files = [ [package.extras] optional = ["SQLAlchemy (>=1.4,<3)", "aiodns (>1.0)", "aiohttp (>=3.7.3,<4)", "boto3 (<=2)", "websocket-client (>=1,<2)", "websockets (>=9.1,<15)"] +[[package]] +name = "smart-open" +version = "7.0.5" +description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +optional = false +python-versions = "<4.0,>=3.7" +files = [ + {file = "smart_open-7.0.5-py3-none-any.whl", hash = "sha256:8523ed805c12dff3eaa50e9c903a6cb0ae78800626631c5fe7ea073439847b89"}, + {file = "smart_open-7.0.5.tar.gz", hash = "sha256:d3672003b1dbc85e2013e4983b88eb9a5ccfd389b0d4e5015f39a9ee5620ec18"}, +] + +[package.dependencies] +wrapt = "*" + +[package.extras] +all = ["azure-common", "azure-core", "azure-storage-blob", "boto3", "google-cloud-storage (>=2.6.0)", "paramiko", "requests", "zstandard"] +azure = ["azure-common", "azure-core", "azure-storage-blob"] +gcs = ["google-cloud-storage (>=2.6.0)"] +http = ["requests"] +s3 = ["boto3"] +ssh = ["paramiko"] +test = ["awscli", "azure-common", "azure-core", "azure-storage-blob", "boto3", "google-cloud-storage (>=2.6.0)", "moto[server]", "numpy", "paramiko", "pyopenssl", "pytest", "pytest-benchmark", "pytest-rerunfailures", "requests", "responses", "zstandard"] +webhdfs = ["requests"] +zst = ["zstandard"] + [[package]] name = "sniffio" version = "1.3.1" @@ -3843,7 +3933,109 @@ docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] +[[package]] +name = "werkzeug" +version = "3.1.3" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.9" +files = [ + {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, + {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + +[[package]] +name = "wrapt" +version = "1.17.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.8" +files = [ + {file = "wrapt-1.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a0c23b8319848426f305f9cb0c98a6e32ee68a36264f45948ccf8e7d2b941f8"}, + {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ca5f060e205f72bec57faae5bd817a1560fcfc4af03f414b08fa29106b7e2d"}, + {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e185ec6060e301a7e5f8461c86fb3640a7beb1a0f0208ffde7a65ec4074931df"}, + {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb90765dd91aed05b53cd7a87bd7f5c188fcd95960914bae0d32c5e7f899719d"}, + {file = "wrapt-1.17.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:879591c2b5ab0a7184258274c42a126b74a2c3d5a329df16d69f9cee07bba6ea"}, + {file = "wrapt-1.17.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fce6fee67c318fdfb7f285c29a82d84782ae2579c0e1b385b7f36c6e8074fffb"}, + {file = "wrapt-1.17.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0698d3a86f68abc894d537887b9bbf84d29bcfbc759e23f4644be27acf6da301"}, + {file = "wrapt-1.17.0-cp310-cp310-win32.whl", hash = "sha256:69d093792dc34a9c4c8a70e4973a3361c7a7578e9cd86961b2bbf38ca71e4e22"}, + {file = "wrapt-1.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:f28b29dc158ca5d6ac396c8e0a2ef45c4e97bb7e65522bfc04c989e6fe814575"}, + {file = "wrapt-1.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:74bf625b1b4caaa7bad51d9003f8b07a468a704e0644a700e936c357c17dd45a"}, + {file = "wrapt-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f2a28eb35cf99d5f5bd12f5dd44a0f41d206db226535b37b0c60e9da162c3ed"}, + {file = "wrapt-1.17.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81b1289e99cf4bad07c23393ab447e5e96db0ab50974a280f7954b071d41b489"}, + {file = "wrapt-1.17.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2939cd4a2a52ca32bc0b359015718472d7f6de870760342e7ba295be9ebaf9"}, + {file = "wrapt-1.17.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a9653131bda68a1f029c52157fd81e11f07d485df55410401f745007bd6d339"}, + {file = "wrapt-1.17.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4e4b4385363de9052dac1a67bfb535c376f3d19c238b5f36bddc95efae15e12d"}, + {file = "wrapt-1.17.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bdf62d25234290db1837875d4dceb2151e4ea7f9fff2ed41c0fde23ed542eb5b"}, + {file = "wrapt-1.17.0-cp311-cp311-win32.whl", hash = "sha256:5d8fd17635b262448ab8f99230fe4dac991af1dabdbb92f7a70a6afac8a7e346"}, + {file = "wrapt-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:92a3d214d5e53cb1db8b015f30d544bc9d3f7179a05feb8f16df713cecc2620a"}, + {file = "wrapt-1.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:89fc28495896097622c3fc238915c79365dd0ede02f9a82ce436b13bd0ab7569"}, + {file = "wrapt-1.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:875d240fdbdbe9e11f9831901fb8719da0bd4e6131f83aa9f69b96d18fae7504"}, + {file = "wrapt-1.17.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5ed16d95fd142e9c72b6c10b06514ad30e846a0d0917ab406186541fe68b451"}, + {file = "wrapt-1.17.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18b956061b8db634120b58f668592a772e87e2e78bc1f6a906cfcaa0cc7991c1"}, + {file = "wrapt-1.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:daba396199399ccabafbfc509037ac635a6bc18510ad1add8fd16d4739cdd106"}, + {file = "wrapt-1.17.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4d63f4d446e10ad19ed01188d6c1e1bb134cde8c18b0aa2acfd973d41fcc5ada"}, + {file = "wrapt-1.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8a5e7cc39a45fc430af1aefc4d77ee6bad72c5bcdb1322cfde852c15192b8bd4"}, + {file = "wrapt-1.17.0-cp312-cp312-win32.whl", hash = "sha256:0a0a1a1ec28b641f2a3a2c35cbe86c00051c04fffcfcc577ffcdd707df3f8635"}, + {file = "wrapt-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:3c34f6896a01b84bab196f7119770fd8466c8ae3dfa73c59c0bb281e7b588ce7"}, + {file = "wrapt-1.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:714c12485aa52efbc0fc0ade1e9ab3a70343db82627f90f2ecbc898fdf0bb181"}, + {file = "wrapt-1.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da427d311782324a376cacb47c1a4adc43f99fd9d996ffc1b3e8529c4074d393"}, + {file = "wrapt-1.17.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba1739fb38441a27a676f4de4123d3e858e494fac05868b7a281c0a383c098f4"}, + {file = "wrapt-1.17.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e711fc1acc7468463bc084d1b68561e40d1eaa135d8c509a65dd534403d83d7b"}, + {file = "wrapt-1.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:140ea00c87fafc42739bd74a94a5a9003f8e72c27c47cd4f61d8e05e6dec8721"}, + {file = "wrapt-1.17.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:73a96fd11d2b2e77d623a7f26e004cc31f131a365add1ce1ce9a19e55a1eef90"}, + {file = "wrapt-1.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0b48554952f0f387984da81ccfa73b62e52817a4386d070c75e4db7d43a28c4a"}, + {file = "wrapt-1.17.0-cp313-cp313-win32.whl", hash = "sha256:498fec8da10e3e62edd1e7368f4b24aa362ac0ad931e678332d1b209aec93045"}, + {file = "wrapt-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd136bb85f4568fffca995bd3c8d52080b1e5b225dbf1c2b17b66b4c5fa02838"}, + {file = "wrapt-1.17.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:17fcf043d0b4724858f25b8826c36e08f9fb2e475410bece0ec44a22d533da9b"}, + {file = "wrapt-1.17.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4a557d97f12813dc5e18dad9fa765ae44ddd56a672bb5de4825527c847d6379"}, + {file = "wrapt-1.17.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0229b247b0fc7dee0d36176cbb79dbaf2a9eb7ecc50ec3121f40ef443155fb1d"}, + {file = "wrapt-1.17.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8425cfce27b8b20c9b89d77fb50e368d8306a90bf2b6eef2cdf5cd5083adf83f"}, + {file = "wrapt-1.17.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9c900108df470060174108012de06d45f514aa4ec21a191e7ab42988ff42a86c"}, + {file = "wrapt-1.17.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e547b447073fc0dbfcbff15154c1be8823d10dab4ad401bdb1575e3fdedff1b"}, + {file = "wrapt-1.17.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:914f66f3b6fc7b915d46c1cc424bc2441841083de01b90f9e81109c9759e43ab"}, + {file = "wrapt-1.17.0-cp313-cp313t-win32.whl", hash = "sha256:a4192b45dff127c7d69b3bdfb4d3e47b64179a0b9900b6351859f3001397dabf"}, + {file = "wrapt-1.17.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4f643df3d4419ea3f856c5c3f40fec1d65ea2e89ec812c83f7767c8730f9827a"}, + {file = "wrapt-1.17.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:69c40d4655e078ede067a7095544bcec5a963566e17503e75a3a3e0fe2803b13"}, + {file = "wrapt-1.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f495b6754358979379f84534f8dd7a43ff8cff2558dcdea4a148a6e713a758f"}, + {file = "wrapt-1.17.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:baa7ef4e0886a6f482e00d1d5bcd37c201b383f1d314643dfb0367169f94f04c"}, + {file = "wrapt-1.17.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fc931382e56627ec4acb01e09ce66e5c03c384ca52606111cee50d931a342d"}, + {file = "wrapt-1.17.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8f8909cdb9f1b237786c09a810e24ee5e15ef17019f7cecb207ce205b9b5fcce"}, + {file = "wrapt-1.17.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ad47b095f0bdc5585bced35bd088cbfe4177236c7df9984b3cc46b391cc60627"}, + {file = "wrapt-1.17.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:948a9bd0fb2c5120457b07e59c8d7210cbc8703243225dbd78f4dfc13c8d2d1f"}, + {file = "wrapt-1.17.0-cp38-cp38-win32.whl", hash = "sha256:5ae271862b2142f4bc687bdbfcc942e2473a89999a54231aa1c2c676e28f29ea"}, + {file = "wrapt-1.17.0-cp38-cp38-win_amd64.whl", hash = "sha256:f335579a1b485c834849e9075191c9898e0731af45705c2ebf70e0cd5d58beed"}, + {file = "wrapt-1.17.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d751300b94e35b6016d4b1e7d0e7bbc3b5e1751e2405ef908316c2a9024008a1"}, + {file = "wrapt-1.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7264cbb4a18dc4acfd73b63e4bcfec9c9802614572025bdd44d0721983fc1d9c"}, + {file = "wrapt-1.17.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33539c6f5b96cf0b1105a0ff4cf5db9332e773bb521cc804a90e58dc49b10578"}, + {file = "wrapt-1.17.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c30970bdee1cad6a8da2044febd824ef6dc4cc0b19e39af3085c763fdec7de33"}, + {file = "wrapt-1.17.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:bc7f729a72b16ee21795a943f85c6244971724819819a41ddbaeb691b2dd85ad"}, + {file = "wrapt-1.17.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6ff02a91c4fc9b6a94e1c9c20f62ea06a7e375f42fe57587f004d1078ac86ca9"}, + {file = "wrapt-1.17.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dfb7cff84e72e7bf975b06b4989477873dcf160b2fd89959c629535df53d4e0"}, + {file = "wrapt-1.17.0-cp39-cp39-win32.whl", hash = "sha256:2399408ac33ffd5b200480ee858baa58d77dd30e0dd0cab6a8a9547135f30a88"}, + {file = "wrapt-1.17.0-cp39-cp39-win_amd64.whl", hash = "sha256:4f763a29ee6a20c529496a20a7bcb16a73de27f5da6a843249c7047daf135977"}, + {file = "wrapt-1.17.0-py3-none-any.whl", hash = "sha256:d2c63b93548eda58abf5188e505ffed0229bf675f7c3090f8e36ad55b8cbc371"}, + {file = "wrapt-1.17.0.tar.gz", hash = "sha256:16187aa2317c731170a88ef35e8937ae0f533c402872c1ee5e6d079fcf320801"}, +] + +[[package]] +name = "xmltodict" +version = "0.14.2" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.6" +files = [ + {file = "xmltodict-0.14.2-py2.py3-none-any.whl", hash = "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac"}, + {file = "xmltodict-0.14.2.tar.gz", hash = "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553"}, +] + [metadata] lock-version = "2.0" python-versions = "~3.13" -content-hash = "b03b4d3ca7756fb6e54b697ade60c9c767eca11578bb14bf081e1566b68ac8aa" +content-hash = "bf6073ede1c2274e5d4335beb49fba237ef2a4dd7add0585469d293a8dfb4dab" diff --git a/analytics/pyproject.toml b/analytics/pyproject.toml index a3200dab0..1145e85b8 100644 --- a/analytics/pyproject.toml +++ b/analytics/pyproject.toml @@ -22,10 +22,11 @@ python = "~3.13" slack-sdk = "^3.23.0" typer = { extras = ["all"], version = "^0.15.0" } sqlalchemy = "^2.0.30" -psycopg = ">=3.0.7" pydantic-settings = "^2.3.4" boto3 = "^1.35.56" boto3-stubs = "^1.35.56" +psycopg = "^3.2.3" +smart-open = "^7.0.5" [tool.poetry.group.dev.dependencies] black = "^24.0.0" @@ -36,6 +37,7 @@ pytest-cov = "^5.0.0" ruff = "^0.8.0" safety = "^3.0.0" types-requests = "^2.32.0.20241016" +moto = "^5.0.22" [build-system] build-backend = "poetry.core.masonry.api" diff --git a/analytics/src/analytics/cli.py b/analytics/src/analytics/cli.py index 5b41f98e5..d40bc5a9f 100644 --- a/analytics/src/analytics/cli.py +++ b/analytics/src/analytics/cli.py @@ -18,6 +18,9 @@ from analytics.etl.utils import load_config from analytics.integrations import etldb, slack from analytics.integrations.db import PostgresDbClient +from analytics.integrations.extracts.load_opportunity_data import ( + extract_copy_opportunity_data, +) from analytics.logs import init as init_logging from analytics.logs.app_logger import init_app from analytics.logs.ecs_background_task import ecs_background_task @@ -301,3 +304,9 @@ def transform_and_load( # finish print("transform and load is done") + + +@etl_app.command(name="opportunity-load") +def load_opportunity_data() -> None: + """Grabs data from s3 bucket and loads it into opportunity tables.""" + extract_copy_opportunity_data() diff --git a/analytics/src/analytics/integrations/extracts/__init__.py b/analytics/src/analytics/integrations/extracts/__init__.py new file mode 100644 index 000000000..eb68834cf --- /dev/null +++ b/analytics/src/analytics/integrations/extracts/__init__.py @@ -0,0 +1,6 @@ +""" +We use this package to load opportunity data from s3. + +It extracts CSV files from S3 bucket and loads the records into respective +opportunity tables. +""" diff --git a/analytics/src/analytics/integrations/extracts/constants.py b/analytics/src/analytics/integrations/extracts/constants.py new file mode 100644 index 000000000..b23ed8f9a --- /dev/null +++ b/analytics/src/analytics/integrations/extracts/constants.py @@ -0,0 +1,101 @@ +"""Holds all constant values.""" + +from enum import StrEnum + + +class OpportunityTables(StrEnum): + """Opportunity tables that are copied over to analytics database.""" + + LK_OPPORTUNITY_STATUS = "lk_opportunity_status" + LK_OPPORTUNITY_CATEGORY = "lk_opportunity_category" + OPPORTUNITY = "opportunity" + OPPORTUNITY_SUMMARY = "opportunity_summary" + CURRENT_OPPORTUNITY_SUMMARY = "current_opportunity_summary" + + +LK_OPPORTUNITY_STATUS_COLS = ( + "OPPORTUNITY_STATUS_ID", + "DESCRIPTION", + "CREATED_AT", + "UPDATED_AT", +) + +LK_OPPORTUNITY_CATEGORY_COLS = ( + "OPPORTUNITY_CATEGORY_ID", + "DESCRIPTION", + "CREATED_AT", + "UPDATED_AT", +) +OPPORTUNITY_COLS = ( + "OPPORTUNITY_ID", + "OPPORTUNITY_NUMBER", + "OPPORTUNITY_TITLE", + "AGENCY_CODE", + "OPPORTUNITY_CATEGORY_ID", + "CATEGORY_EXPLANATION", + "IS_DRAFT", + "REVISION_NUMBER", + "MODIFIED_COMMENTS", + "PUBLISHER_USER_ID", + "PUBLISHER_PROFILE_ID", + "CREATED_AT", + "UPDATED_AT", +) +OPOORTUNITY_SUMMARY_COLS = ( + "OPPORTUNITY_SUMMARY_ID", + "OPPORTUNITY_ID", + "SUMMARY_DESCRIPTION", + "IS_COST_SHARING", + "IS_FORECAST", + "POST_DATE", + "CLOSE_DATE", + "CLOSE_DATE_DESCRIPTION", + "ARCHIVE_DATE", + "UNARCHIVE_DATE", + "EXPECTED_NUMBER_OF_AWARDS", + "ESTIMATED_TOTAL_PROGRAM_FUNDING", + "AWARD_FLOOR", + "AWARD_CEILING", + "ADDITIONAL_INFO_URL", + "ADDITIONAL_INFO_URL_DESCRIPTION", + "FORECASTED_POST_DATE", + "FORECASTED_CLOSE_DATE", + "FORECASTED_CLOSE_DATE_DESCRIPTION", + "FORECASTED_AWARD_DATE", + "FORECASTED_PROJECT_START_DATE", + "FISCAL_YEAR", + "REVISION_NUMBER", + "MODIFICATION_COMMENTS", + "FUNDING_CATEGORY_DESCRIPTION", + "APPLICANT_ELIGIBILITY_DESCRIPTION", + "AGENCY_CODE", + "AGENCY_NAME", + "AGENCY_PHONE_NUMBER", + "AGENCY_CONTACT_DESCRIPTION", + "AGENCY_EMAIL_ADDRESS", + "AGENCY_EMAIL_ADDRESS_DESCRIPTION", + "IS_DELETED", + "CAN_SEND_MAIL", + "PUBLISHER_PROFILE_ID", + "PUBLISHER_USER_ID", + "UPDATED_BY", + "CREATED_BY", + "CREATED_AT", + "UPDATED_AT", + "VERSION_NUMBER", +) +CURRENT_OPPORTUNITY_SUMMARY_COLS = ( + "OPPORTUNITY_ID", + "OPPORTUNITY_SUMMARY_ID", + "OPPORTUNITY_STATUS_ID", + "CREATED_AT", + "UPDATED_AT", +) + +MAP_TABLES_TO_COLS: dict[OpportunityTables, tuple[str, ...]] = { + OpportunityTables.LK_OPPORTUNITY_STATUS: LK_OPPORTUNITY_STATUS_COLS, + OpportunityTables.LK_OPPORTUNITY_CATEGORY: LK_OPPORTUNITY_CATEGORY_COLS, + OpportunityTables.OPPORTUNITY: OPPORTUNITY_COLS, + OpportunityTables.OPPORTUNITY_SUMMARY: OPOORTUNITY_SUMMARY_COLS, + OpportunityTables.CURRENT_OPPORTUNITY_SUMMARY: CURRENT_OPPORTUNITY_SUMMARY_COLS, +} diff --git a/analytics/src/analytics/integrations/extracts/load_opportunity_data.py b/analytics/src/analytics/integrations/extracts/load_opportunity_data.py new file mode 100644 index 000000000..3be19d6b7 --- /dev/null +++ b/analytics/src/analytics/integrations/extracts/load_opportunity_data.py @@ -0,0 +1,79 @@ +# pylint: disable=invalid-name, line-too-long +"""Loads opportunity tables with opportunity data from S3.""" + +import logging +import os +from contextlib import ExitStack + +import smart_open # type: ignore[import] +from pydantic import Field +from pydantic_settings import BaseSettings +from sqlalchemy import Connection + +from analytics.integrations.etldb.etldb import EtlDb +from analytics.integrations.extracts.constants import ( + MAP_TABLES_TO_COLS, + OpportunityTables, +) + +logger = logging.getLogger(__name__) + + +class LoadOpportunityDataFileConfig(BaseSettings): + """Configure S3 properties for opportunity data.""" + + load_opportunity_data_file_path: str | None = Field( + default=None, + alias="LOAD_OPPORTUNITY_DATA_FILE_PATH", + ) + + +def extract_copy_opportunity_data() -> None: + """Instantiate Etldb class and call helper funcs to truncate and insert data in one txn.""" + etldb_conn = EtlDb() + + with etldb_conn.connection() as conn, conn.begin(): + _trancate_opportunity_table_records(conn) + + _fetch_insert_opportunity_data(conn) + + logger.info("Extract opportunity data completed successfully") + + +def _trancate_opportunity_table_records(conn: Connection) -> None: + """Truncate existing records from all tables.""" + cursor = conn.connection.cursor() + schema = os.environ["DB_SCHEMA"] + for table in OpportunityTables: + stmt_trct = f"TRUNCATE TABLE {schema}.{table} CASCADE" + cursor.execute(stmt_trct) + logger.info("Truncated all records from all tables") + + +def _fetch_insert_opportunity_data(conn: Connection) -> None: + """Streamlines opportunity tables from S3 and insert into the database.""" + s3_config = LoadOpportunityDataFileConfig() + + cursor = conn.connection.cursor() + for table in OpportunityTables: + logger.info("Copying data for table: %s", table) + + columns = MAP_TABLES_TO_COLS.get(table, ()) + query = f""" + COPY {f"{os.getenv("DB_SCHEMA")}.{table} ({', '.join(columns)})"} + FROM STDIN WITH (FORMAT CSV, DELIMITER ',', QUOTE '"', HEADER) + """ + + with ExitStack() as stack: + file = stack.enter_context( + smart_open.open( + f"{s3_config.load_opportunity_data_file_path}/{table}.csv", + "r", + ), + ) + copy = stack.enter_context(cursor.copy(query)) + + while data := file.read(): + copy.write(data) + + logger.info("Successfully loaded data for table: %s", table) diff --git a/analytics/src/analytics/integrations/extracts/s3_config.py b/analytics/src/analytics/integrations/extracts/s3_config.py new file mode 100644 index 000000000..0b53a956e --- /dev/null +++ b/analytics/src/analytics/integrations/extracts/s3_config.py @@ -0,0 +1,18 @@ +"""Configuration for S3.""" + +import boto3 +import botocore + + +def get_s3_client( + session: boto3.Session | None = None, + boto_config: botocore.config.Config | None = None, +) -> botocore.client.BaseClient: + """Return an S3 client.""" + if boto_config is None: + boto_config = botocore.config.Config(signature_version="s3v4") + + if session is not None: + return session.client("s3", config=boto_config) + + return boto3.client("s3", config=boto_config) diff --git a/analytics/tests/conftest.py b/analytics/tests/conftest.py index 7d25cb528..3c696a2f6 100644 --- a/analytics/tests/conftest.py +++ b/analytics/tests/conftest.py @@ -5,13 +5,23 @@ Visit pytest docs for more info: https://docs.pytest.org/en/7.1.x/reference/fixtures.html """ - import json +import logging +import uuid from pathlib import Path +import _pytest.monkeypatch +import boto3 +import moto import pandas as pd import pytest + +from sqlalchemy import text # isort: skip from analytics.datasets.issues import IssueMetadata, IssueType +from analytics.integrations.etldb.etldb import EtlDb + +logger = logging.getLogger(__name__) + # skips the integration tests in tests/integrations/ # to run the integration tests, invoke them directly: pytest tests/integrations/ @@ -255,3 +265,135 @@ def issue( # pylint: disable=too-many-locals sprint_start=sprint_start, sprint_end=sprint_end_ts.strftime("%Y-%m-%d"), ) + + +#################### +# AWS Mock Fixtures +#################### + + +@pytest.fixture(autouse=True) +def reset_aws_env_vars(monkeypatch: pytest.MonkeyPatch) -> None: + """ + Reset the aws env vars. + + This will prevent you from accidentally connecting + to a real AWS account if you were doing some local testing. + """ + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_SECURITY_TOKEN", "testing") + monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +@pytest.fixture +def mock_s3() -> boto3.resource: + """Instantiate an S3 bucket resource.""" + # https://docs.getmoto.org/en/stable/docs/configuration/index.html#whitelist-services + with moto.mock_aws(config={"core": {"service_whitelist": ["s3"]}}): + yield boto3.resource("s3") + + +@pytest.fixture +def mock_s3_bucket_resource( + mock_s3: boto3.resource, +) -> boto3.resource("s3").Bucket: + """Create and return a mock S3 bucket resource.""" + bucket = mock_s3.Bucket("test_bucket") + bucket.create() + return bucket + + +@pytest.fixture +def mock_s3_bucket(mock_s3_bucket_resource: boto3.resource("s3").Bucket) -> str: + """Return name of mock S3 bucket.""" + return mock_s3_bucket_resource.name + + +# From https://github.com/pytest-dev/pytest/issues/363 +@pytest.fixture(scope="session") +def monkeypatch_session() -> pytest.MonkeyPatch: + """ + Create a monkeypatch instance. + + This can be used to monkeypatch global environment, objects, and attributes + for the duration the test session. + """ + mpatch = _pytest.monkeypatch.MonkeyPatch() + yield mpatch + mpatch.undo() + + +@pytest.fixture(scope="session") +def test_schema() -> str: + """Create a unique test schema.""" + return f"test_schema_{uuid.uuid4().int}" + + +@pytest.fixture(scope="session") +def create_test_db(test_schema: str) -> EtlDb: + """ + Create a temporary PostgreSQL schema. + + This function creates schema and a database engine + that connects to that schema. Drops the schema after the context manager + exits. + """ + etldb_conn = EtlDb() + + with etldb_conn.connection() as conn: + + _create_schema(conn, test_schema) + + _create_opportunity_table(conn, test_schema) + try: + yield etldb_conn + + finally: + _drop_schema(conn, test_schema) + + +def _create_schema(conn: EtlDb.connection, schema: str) -> None: + """Create a database schema.""" + db_test_user = "app" + + with conn.begin(): + conn.execute( + text(f"CREATE SCHEMA IF NOT EXISTS {schema} AUTHORIZATION {db_test_user};"), + ) + logger.info("Created schema %s", schema) + + +def _drop_schema(conn: EtlDb.connection, schema: str) -> None: + """Drop a database schema.""" + with conn.begin(): + conn.execute(text(f"DROP SCHEMA {schema} CASCADE;")) + + logger.info("Dropped schema %s", schema) + + +def _create_opportunity_table(conn: EtlDb.connection, schema: str) -> None: + """Create opportunity tables.""" + with conn.begin(): + conn.execute(text(f"SET search_path TO {schema};")) + # Get the path of the current file (test file) + test_file_path = Path(__file__).resolve() + + # Construct the path to the SQL file + sql_file_path = ( + test_file_path.parent.parent + / "src" + / "analytics" + / "integrations" + / "etldb" + / "migrations" + / "versions" + / "0007_add_opportunity_tables.sql" + ) + + with open(sql_file_path) as file: + create_table_commands = file.read() + conn.execute(text(create_table_commands)) + + logger.info("Created opportunity tables") diff --git a/analytics/tests/integrations/extracts/__init__.py b/analytics/tests/integrations/extracts/__init__.py new file mode 100644 index 000000000..204170ed1 --- /dev/null +++ b/analytics/tests/integrations/extracts/__init__.py @@ -0,0 +1 @@ +"""Tests the code in extracts.""" diff --git a/analytics/tests/integrations/extracts/opportunity_tables_test_files/current_opportunity_summary.csv b/analytics/tests/integrations/extracts/opportunity_tables_test_files/current_opportunity_summary.csv new file mode 100644 index 000000000..c79f894f0 --- /dev/null +++ b/analytics/tests/integrations/extracts/opportunity_tables_test_files/current_opportunity_summary.csv @@ -0,0 +1,33 @@ +"opportunity_id","opportunity_summary_id","opportunity_status_id","created_at","updated_at" +1,1,1,"2024-12-06 20:25:05.206915+00","2024-12-06 20:25:05.206915+00" +2,2,1,"2024-12-06 20:25:05.240761+00","2024-12-06 20:25:05.240761+00" +3,3,1,"2024-12-06 20:25:05.262463+00","2024-12-06 20:25:05.262463+00" +4,4,1,"2024-12-06 20:25:05.277865+00","2024-12-06 20:25:05.277865+00" +5,5,1,"2024-12-06 20:25:05.296117+00","2024-12-06 20:25:05.296117+00" +6,6,2,"2024-12-06 20:25:05.318442+00","2024-12-06 20:25:05.318442+00" +7,7,2,"2024-12-06 20:25:05.33442+00","2024-12-06 20:25:05.33442+00" +8,8,2,"2024-12-06 20:25:05.349716+00","2024-12-06 20:25:05.349716+00" +9,9,2,"2024-12-06 20:25:05.363082+00","2024-12-06 20:25:05.363082+00" +10,10,2,"2024-12-06 20:25:05.381924+00","2024-12-06 20:25:05.381924+00" +11,11,3,"2024-12-06 20:25:05.398327+00","2024-12-06 20:25:05.398327+00" +12,12,3,"2024-12-06 20:25:05.412689+00","2024-12-06 20:25:05.412689+00" +13,13,3,"2024-12-06 20:25:05.428318+00","2024-12-06 20:25:05.428318+00" +14,14,3,"2024-12-06 20:25:05.444209+00","2024-12-06 20:25:05.444209+00" +15,15,3,"2024-12-06 20:25:05.462069+00","2024-12-06 20:25:05.462069+00" +16,16,4,"2024-12-06 20:25:05.477612+00","2024-12-06 20:25:05.477612+00" +17,17,4,"2024-12-06 20:25:05.495072+00","2024-12-06 20:25:05.495072+00" +18,18,4,"2024-12-06 20:25:05.511716+00","2024-12-06 20:25:05.511716+00" +19,19,4,"2024-12-06 20:25:05.525504+00","2024-12-06 20:25:05.525504+00" +20,20,4,"2024-12-06 20:25:05.539469+00","2024-12-06 20:25:05.539469+00" +21,21,4,"2024-12-06 20:25:05.563918+00","2024-12-06 20:25:05.563918+00" +22,22,4,"2024-12-06 20:25:05.57996+00","2024-12-06 20:25:05.57996+00" +23,23,4,"2024-12-06 20:25:05.594035+00","2024-12-06 20:25:05.594035+00" +24,24,4,"2024-12-06 20:25:05.609782+00","2024-12-06 20:25:05.609782+00" +25,25,4,"2024-12-06 20:25:05.625264+00","2024-12-06 20:25:05.625264+00" +31,26,2,"2024-12-06 20:25:05.661145+00","2024-12-06 20:25:05.661145+00" +32,27,2,"2024-12-06 20:25:05.675049+00","2024-12-06 20:25:05.675049+00" +33,28,2,"2024-12-06 20:25:05.720527+00","2024-12-06 20:25:05.720527+00" +34,29,2,"2024-12-06 20:25:05.726469+00","2024-12-06 20:25:05.726469+00" +35,30,2,"2024-12-06 20:25:05.732488+00","2024-12-06 20:25:05.732488+00" +36,31,2,"2024-12-06 20:25:05.739234+00","2024-12-06 20:25:05.739234+00" +37,32,2,"2024-12-06 20:25:05.744722+00","2024-12-06 20:25:05.744722+00" \ No newline at end of file diff --git a/analytics/tests/integrations/extracts/opportunity_tables_test_files/lk_opportunity_category.csv b/analytics/tests/integrations/extracts/opportunity_tables_test_files/lk_opportunity_category.csv new file mode 100644 index 000000000..5682bb75a --- /dev/null +++ b/analytics/tests/integrations/extracts/opportunity_tables_test_files/lk_opportunity_category.csv @@ -0,0 +1,6 @@ +"opportunity_category_id","description","created_at","updated_at" +1,"discretionary","2024-12-06 20:24:44.809336+00","2024-12-06 20:24:44.809336+00" +2,"mandatory","2024-12-06 20:24:44.811405+00","2024-12-06 20:24:44.811405+00" +3,"continuation","2024-12-06 20:24:44.812989+00","2024-12-06 20:24:44.812989+00" +4,"earmark","2024-12-06 20:24:44.814473+00","2024-12-06 20:24:44.814473+00" +5,"other","2024-12-06 20:24:44.816205+00","2024-12-06 20:24:44.816205+00" \ No newline at end of file diff --git a/analytics/tests/integrations/extracts/opportunity_tables_test_files/lk_opportunity_status.csv b/analytics/tests/integrations/extracts/opportunity_tables_test_files/lk_opportunity_status.csv new file mode 100644 index 000000000..fa9555f9e --- /dev/null +++ b/analytics/tests/integrations/extracts/opportunity_tables_test_files/lk_opportunity_status.csv @@ -0,0 +1,5 @@ +"opportunity_status_id","description","created_at","updated_at" +1,"forecasted","2024-12-06 20:24:44.893933+00","2024-12-06 20:24:44.893933+00" +2,"posted","2024-12-06 20:24:44.895692+00","2024-12-06 20:24:44.895692+00" +3,"closed","2024-12-06 20:24:44.89737+00","2024-12-06 20:24:44.89737+00" +4,"archived","2024-12-06 20:24:44.899297+00","2024-12-06 20:24:44.899297+00" \ No newline at end of file diff --git a/analytics/tests/integrations/extracts/opportunity_tables_test_files/opportunity.csv b/analytics/tests/integrations/extracts/opportunity_tables_test_files/opportunity.csv new file mode 100644 index 000000000..8fc4c2979 --- /dev/null +++ b/analytics/tests/integrations/extracts/opportunity_tables_test_files/opportunity.csv @@ -0,0 +1,38 @@ +"opportunity_id","opportunity_number","opportunity_title","agency_code","opportunity_category_id","category_explanation","is_draft","revision_number","modified_comments","publisher_user_id","publisher_profile_id","created_at","updated_at" +1,"USAID-SAF-92-925","Research into Designer, multimedia industry","USDA-FS",2,,false,0,,,,"2024-12-06 20:25:05.164842+00","2024-12-06 20:25:05.164842+00" +2,"KO-68-HGG-637","Embassy program for Clinical psychologist in Japan","USDA-AMS",1,,false,0,,,,"2024-12-06 20:25:05.225051+00","2024-12-06 20:25:05.225051+00" +3,"AUX-214-FY1997-787","Heather Gomez Foundation Grant for repurpose transparent markets","USDOJ-OJP-NIJ",4,,false,0,,,,"2024-12-06 20:25:05.246241+00","2024-12-06 20:25:05.246241+00" +4,"CMZ1121630","Adam Rose Foundation Grant for re-contextualize dot-com schemas","USDOJ-BOP-NIC",2,,false,0,,,,"2024-12-06 20:25:05.266404+00","2024-12-06 20:25:05.266404+00" +5,"PT-49-LUJ-681","Kramer, Garcia and Martin 1987 award","DOI-BOR-MP",4,,false,0,,,,"2024-12-06 20:25:05.282455+00","2024-12-06 20:25:05.282455+00" +6,"USDA-NIFA-37-341","Research into Accountant, chartered industry","USDA",5,"Process ever.",false,0,,,,"2024-12-06 20:25:05.301674+00","2024-12-06 20:25:05.301674+00" +7,"HI-86-PHF-536","Jason Meadows Foundation Grant for drive open-source partnerships","HHS-CDC-HHSCDCERA",1,,false,0,,,,"2024-12-06 20:25:05.32288+00","2024-12-06 20:25:05.32288+00" +8,"SRD8397082","Research into General practice doctor industry","USDA-FNS1",4,,false,0,,,,"2024-12-06 20:25:05.337045+00","2024-12-06 20:25:05.337045+00" +9,"JQA9770458","Heather Rodriguez Foundation Grant for engage collaborative experiences","HHS-NIH11",3,,false,0,,,,"2024-12-06 20:25:05.352315+00","2024-12-06 20:25:05.352315+00" +10,"KS-39-MSZ-590","Hodge, Friedman and Clark 2022 award","DOI-BIA",5,"Huge try.",false,0,,,,"2024-12-06 20:25:05.370379+00","2024-12-06 20:25:05.370379+00" +11,"DOI-BOR-31-002","Embassy program for Designer, blown glass/stained glass in Somalia","USDA-NIFA",1,,false,0,,,,"2024-12-06 20:25:05.386855+00","2024-12-06 20:25:05.386855+00" +12,"QNY7538929","Embassy program for Trade union research officer in Sierra Leone","HHS-CDC",1,,false,0,,,,"2024-12-06 20:25:05.4012+00","2024-12-06 20:25:05.4012+00" +13,"LXW7071628","Research into Web designer industry","DOI-BOR-LC",2,,false,0,,,,"2024-12-06 20:25:05.416649+00","2024-12-06 20:25:05.416649+00" +14,"IM-02-POQ-622","Robles, Miller and Salazar 1993 award","USAID-ETH",4,,false,0,,,,"2024-12-06 20:25:05.431341+00","2024-12-06 20:25:05.431341+00" +15,"IP-50-ANZ-381","Barber-Bell 2003 award","DOE-NETL",5,"Medical many instead.",false,0,,,,"2024-12-06 20:25:05.449796+00","2024-12-06 20:25:05.449796+00" +16,"USDA-NIFA-32-659","Reid PLC 1974 award","HHS-HRSA",4,,false,0,,,,"2024-12-06 20:25:05.466628+00","2024-12-06 20:25:05.466628+00" +17,"DOE-ARPAE-35-319","Franklin-Miles 2022 award","DOC-DOCNOAAERA",3,,false,0,,,,"2024-12-06 20:25:05.483082+00","2024-12-06 20:25:05.483082+00" +18,"USDOJ-58-065","Lauren Williams Foundation Grant for scale vertical models","DOC",3,,false,0,,,,"2024-12-06 20:25:05.500104+00","2024-12-06 20:25:05.500104+00" +19,"DOD-88-971","Joel Perez Foundation Grant for maximize B2C interfaces","USAID",1,,false,0,,,,"2024-12-06 20:25:05.514294+00","2024-12-06 20:25:05.514294+00" +20,"USDA-FAS-40-647","Roberts and Sons 2020 award","HHS-SAMHS",1,,false,0,,,,"2024-12-06 20:25:05.528777+00","2024-12-06 20:25:05.528777+00" +21,"NIC0446243","Research into Television camera operator industry","USDA-FAS",2,,false,0,,,,"2024-12-06 20:25:05.542184+00","2024-12-06 20:25:05.542184+00" +22,"DOI-BLM-79-338","Embassy program for Engineer, manufacturing systems in Mongolia","DOI",2,,false,0,,,,"2024-12-06 20:25:05.567813+00","2024-12-06 20:25:05.567813+00" +23,"YK-81-DAZ-945","Research into Community arts worker industry","USDOJ-OJP-OJJDP",5,"Plan trip easy.",false,0,,,,"2024-12-06 20:25:05.583808+00","2024-12-06 20:25:05.583808+00" +24,"MONEY-355-36","Embassy program for Chief Technology Officer in Brazil","DOI-BOR-LC",5,"Notice north.",false,0,,,,"2024-12-06 20:25:05.598324+00","2024-12-06 20:25:05.598324+00" +25,"CIG6299326","Michael-Williams 2019 award","USDOJ-OJP-OVW",3,,false,0,,,,"2024-12-06 20:25:05.615039+00","2024-12-06 20:25:05.615039+00" +26,"SNV-378-FY1978-538","Embassy program for Surveyor, building control in Eritrea","HHS-SAMHS",2,,false,0,,,,"2024-12-06 20:25:05.627844+00","2024-12-06 20:25:05.627844+00" +27,"DOC-NIST-66-435","Embassy program for Architectural technologist in Switzerland","USDOJ-OJP-BJA",3,,false,0,,,,"2024-12-06 20:25:05.633181+00","2024-12-06 20:25:05.633181+00" +28,"SUZ2939299","Research into Research officer, government industry","USAID-ETH",4,,false,0,,,,"2024-12-06 20:25:05.637195+00","2024-12-06 20:25:05.637195+00" +29,"EFFORT-995-71","Embassy program for Facilities manager in Monaco","DOI-BOEM",5,"Line thus investment.",false,0,,,,"2024-12-06 20:25:05.640669+00","2024-12-06 20:25:05.640669+00" +30,"NSF-51-230","Research into Arts administrator industry","DOI",2,,false,0,,,,"2024-12-06 20:25:05.645341+00","2024-12-06 20:25:05.645341+00" +31,"CERTAINLY-776-26","Research into Site engineer industry","DOI-NPS",3,,false,0,,,,"2024-12-06 20:25:05.649365+00","2024-12-06 20:25:05.649365+00" +32,"KN-22-WCJ-312","Embassy program for Clinical embryologist in Antarctica (the territory South of 60 deg S)","USDA",5,"Any include consumer.",false,0,,,,"2024-12-06 20:25:05.664726+00","2024-12-06 20:25:05.664726+00" +33,"OSK-167-FY2024-941","Embassy program for Engineer, energy in Aruba",,,,false,0,,,,"2024-12-06 20:25:05.67713+00","2024-12-06 20:25:05.67713+00" +34,"DOD-AFOSR-87-720","Tanya King Foundation Grant for cultivate wireless e-tailers",,,,false,0,,,,"2024-12-06 20:25:05.679611+00","2024-12-06 20:25:05.679611+00" +35,"SQT2021529","Matthew Black Foundation Grant for engage visionary models",,,,false,0,,,,"2024-12-06 20:25:05.683503+00","2024-12-06 20:25:05.683503+00" +36,"YND9750918","Goodwin-Johnson 1988 award",,,,false,0,,,,"2024-12-06 20:25:05.687266+00","2024-12-06 20:25:05.687266+00" +37,"HNT-965-FY1985-743","Linda Caldwell Foundation Grant for facilitate e-business communities",,,,false,0,,,,"2024-12-06 20:25:05.689844+00","2024-12-06 20:25:05.689844+00" \ No newline at end of file diff --git a/analytics/tests/integrations/extracts/opportunity_tables_test_files/opportunity_summary.csv b/analytics/tests/integrations/extracts/opportunity_tables_test_files/opportunity_summary.csv new file mode 100644 index 000000000..f8d2418fc --- /dev/null +++ b/analytics/tests/integrations/extracts/opportunity_tables_test_files/opportunity_summary.csv @@ -0,0 +1,87 @@ +"opportunity_summary_id","opportunity_id","summary_description","is_cost_sharing","is_forecast","post_date","close_date","close_date_description","archive_date","unarchive_date","expected_number_of_awards","estimated_total_program_funding","award_floor","award_ceiling","additional_info_url","additional_info_url_description","forecasted_post_date","forecasted_close_date","forecasted_close_date_description","forecasted_award_date","forecasted_project_start_date","fiscal_year","revision_number","modification_comments","funding_category_description","applicant_eligibility_description","agency_code","agency_name","agency_phone_number","agency_contact_description","agency_email_address","agency_email_address_description","is_deleted","can_send_mail","publisher_profile_id","publisher_user_id","updated_by","created_by","created_at","updated_at","version_number" +1,1,"
Image week believe field. Describe relationship news yet beautiful also crime. Wish better difficult world main else manage. Institution brother top apply see nearly. Wife writer organization table pattern forward as. Industry early bar together. Painting section fast watch. Great a world board research store. Buy bar even then thousand upon other why. Process common thing yet American. Mr special final one American. Rich huge onto wife else. Force assume myself fire. Current current difficult parent prevent maintain middle. Officer still tree religious anyone pressure impact. Travel all factor individual quality ago east recognize. Into authority good challenge generation people. Manage beat hair name field. Understand else guess mouth hotel. Information resource I manage nice loss. Finish teacher value inside. Focus two into half evening treatment sense letter. Tv season open our. Shake population team different ball area. Large just however thank either police real. Positive around hour score later agency when. Several summer everything remember body direction Congress. Accept technology certainly life no type view. Population when upon western last sign. Man ready care hour. Usually machine once why. Grow she save leave. Throughout artist item institution board position. Sit various about back yes media light probably. Network medical need remember voice whole power. Well rather at total few difference. Change determine sense simple. Economic area rise fly something. Why upon once condition. Instead provide court maybe. Land risk sense between hope every eat. Who five any. Total media raise learn up later foot. Evidence for use clearly my film present. Federal cup minute.
Pass stop learn. Matter government charge thus. Son thought cell any practice. Sense room visit top project million. Hope beautiful various sell painting. Black drop particularly month. Close team opportunity environmental sign. Throughout lay consumer garden general miss. Lose site whose provide science. Million military human news such. Any several security down sign two. Share often too mention. Sister forward bad. Shake suddenly senior job blue. Look rate be analysis age travel. Accept we energy health fact. Security hold especially score both. Answer way exist member upon. Agree increase public half service common candidate. Reveal leg to thus. Major offer coach would oil. Newspaper size strategy line fill. Will one onto allow. Something money admit imagine group behavior. Fill page bank family develop his both. Fire official act find enter. Rich stay single particularly past husband part. Foot others fact that need. Late both test history as huge. Series avoid situation north use shake machine activity. Play front top. Adult fire mother their trade which next. Long hold but. Fall quite everything military. Television wall event. In call language amount center wait at onto. Price wear he style. Imagine learn responsibility answer town provide. Tree college last. Sport religious onto might walk. Serious per notice film city. Number among house much happen. Sense be society specific wear. Throughout suddenly station economic. Draw listen decision instead. Order money guess customer management physical nor. Shoulder technology detail discuss generation various maybe much. Myself eat people for must statement. Really scene firm recognize easy yourself continue. Adult under magazine contain member address what whatever. Exist movie mind realize station speak cell. About what career inside. Poor those task it line. Statement live according wonder lead. Minute coach hear detail chance player. Scene word sit respond glass around especially. Professor Republican throughout western firm today thus.
",false,true,"2024-11-29",,,"2024-12-27",,5,2095000,419000,2095000,"sam.gov","Division of Robinson, Jones and Long","2024-12-23","2025-01-23","Game building record present fish.","2025-06-22","2025-09-05",2025,,,,"Outside explain speech would every put husband although. Take decade notice goal live. Military boy as movie PM role assume around.","USDA-FS","Department of Education","123-456-0000","sam.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +ibrown@example.org","bward@example.com","Contact Department of Education via email",false,,,,,,"2021-10-26 00:04:13.122401+00","2022-12-20 13:10:34.943895+00",8 +2,2,"

Individual bed world research on ability return. Against identify exactly claim writer. Above while suffer next arrive sense.


Happy mind my teacher. Anyone teach source employee drug industry. Final factor fly expert member treatment view.

",true,true,"2024-11-21",,,"2025-01-01",,21,8685000,413571,8685000,"grants.gov","Grants.gov","2024-12-24","2025-02-09","Southern Congress alone establish.","2025-06-07","2025-12-04",2025,,,"Model animal mouth enter decision common they relate.","Practice cell popular save somebody organization. Power right phone evidence nothing. Young interview lot nearly shake majority seek. Character discuss rule performance bring its series. Religious center floor.","USDA-AMS","Department of the Economics","123-456-0001","sam.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +kgarcia@example.org","jonesbrittany@example.net","Contact Department of the Economics via email",false,,,,,,"2020-10-12 04:01:50.722421+00","2023-01-08 03:27:51.967876+00",10 +3,3,"
Office much help manage. Politics bar five civil. Animal participant walk everyone. Style treatment quickly today. Religious hour clear necessary can effort work. Real case company account place reduce. Security everybody remember. Note lawyer decade world from then series. Door machine way store describe skin up. Or human represent inside thus draw per increase. Significant fish group green. Week foot key candidate right. Government every everything let next. Stay soon because left. Forward religious nature this. Like skin on add. Protect must indeed own yourself maybe camera. People various care again. Case most again page. Article exactly decision stock moment rather. Defense for condition matter around. Research book speak project must guy add. Partner church everyone marriage. Page cut win another area. Play impact design could hundred blood. Many value attorney responsibility help open forward theory. Able firm particularly well. One seven security similar. Raise number deal not. Crime or involve hospital sell. Industry run might teacher foreign such. Believe sea live fear leader man chance. Foreign choice child. Here now daughter. Item old south laugh claim than story me. Test edge court measure five. Senior president arm authority. Four of group. Mother hospital author daughter let need. Man billion sometimes. Interesting say number suffer arrive might economic performance. Concern section hundred race concern could maybe whatever. Responsibility box child animal less realize deal often. Office reach may somebody. Use day thus effort. Stay church once send always. Door lot involve traditional job let. Continue question style action hundred. Such seem much about though choice although. Of analysis trade play production whatever pretty. Member class smile news door almost. Power front between operation suffer. Arrive add important open. Region feeling easy teacher task politics. However help assume book now society wife thing. Trouble street foreign concern mind. East me always area draw. Cost cup policy different. Fine hard throw they firm. Among program month ago quickly election.
Huge because put grow head good approach. Admit particularly source case federal indicate system. Fill glass account each want standard possible five. Action course hear officer common camera. Base too require already. Person half how. Could claim product statement lay financial. Reflect management strategy material feeling family. Especially in whether once own officer. Cut building also step economic. Until management look party talk the direction term. Choice you word plant shake how morning. Series if other in join believe site. Network training act. Surface success professor reality address surface can. That pick they culture eye most. Lot keep heavy drop join close. Create individual list doctor or affect responsibility. Should resource majority media visit. Thus trouble must American. Order compare laugh also help pay. Teacher yourself travel leader still. Require hear international section big. Glass soldier sense because star. Suffer reveal performance change bag. Region series budget guy. Hair ground nearly watch simply information reveal. So art necessary plant. Everybody include up wait another join develop. Dog sort bit young nice evidence professional. Stay stage style against paper to. Require lawyer order unit. Admit prepare there. Edge course rest early staff firm physical. Quite cold discover green probably price country especially. Tax fund answer drive wish final purpose. Need majority here help stock. Always decide suddenly very member. Just war type computer. Song property media hold. Decide century growth drive begin whatever economic. Organization point role strategy red research cup.
",false,true,"2024-11-24",,,"2025-01-01",,25,1770000,70800,1770000,"google.com","Division of Wilson and Sons","2024-12-24","2025-02-12","Trial admit money pattern rock once measure.","2025-07-01","2025-07-07",2025,,,,"Baby physical line some. Claim season air. She ten arm range center fact resource.","USDOJ-OJP-NIJ","Department of Transportation","123-456-0002","Robert Anderson +Acupuncturist +555-798-2662 +johnsonlinda@example.net","nmeadows@example.org","Contact Department of Transportation via email",false,,,,,,"2020-06-18 06:25:33.288317+00","2021-08-08 19:29:47.228055+00",8 +4,4,"The purpose of this Notice of Funding Opportunity (NOFO) is to support research into Site engineer and how we might Optimized zero tolerance data-warehouse.",true,true,"2024-12-01",,,"2024-12-30",,12,5850000,487500,5850000,"simpler.grants.gov","Program Announcement","2024-12-25","2025-02-11","Especially walk follow floor feeling crime center structure.","2025-06-21","2025-07-12",2025,,,,"Information health hotel education onto my political. Can thus similar medical season live yard. Cover bar back. Black stage ever and policy black I. Describe heavy fire drop billion.","USDOJ-BOP-NIC","Department of the Justice","123-456-0003","sam.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +zhangamy@example.com","laurawilson@example.net","Contact Department of the Justice via email",false,,,,,,"2021-08-08 10:49:04.874162+00","2023-11-09 02:03:32.606008+00",12 +5,5,"The purpose of this Notice of Funding Opportunity (NOFO) is to support research into Audiological scientist and how we might Adaptive user-facing approach.",true,true,"2024-11-20",,,"2024-12-28",,8,8675000,1084375,8675000,"sam.gov","Grants.gov","2024-12-20","2025-02-10","Offer require above including air happy conference.","2025-06-08","2025-07-17",2025,,,"Wife way put ability single.","War book all management than. Brother kitchen plant. Discussion great center what than apply. Responsibility prepare position sister already grow. Save perform sure method put. Attention factor identify black.","DOI-BOR-MP","Department of Education","123-456-0004","Kelly Simon +Designer, fashion/clothing +555-203-3226 +travis43@example.com","kellyjoshua@example.com","Contact Department of Education via email",false,,,,,,"2021-03-07 01:51:41.840958+00","2023-07-13 16:59:51.225593+00",11 +6,6,"DOD-COE-FW is looking to further investigate this topic. Identify thing or. Music food them dinner rise study sense.",false,false,"2024-11-21","2024-12-21","Trip road along remember recently meet policy.","2024-12-30",,19,8440000,444210,8440000,"grants.gov","Full Announcement",,,,,,,,,,,"USDA","Department of Defense","123-456-0005","sam.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +smithblake@example.org","wmarshall@example.net","Contact Department of Defense via email",false,,,,,,"2020-10-17 23:19:27.177491+00","2023-04-04 02:22:45.49148+00",11 +7,7,"

Really society throughout scene. Response form entire open. Soldier of adult.


Worry wife able black bill. Central in peace office.

",false,false,"2024-11-27","2024-12-21","Decide score make always project.","2025-01-02",,25,5420000,216800,5420000,"sam.gov","Link to grant on google.com",,,,,,,,,"Find work black mother section cell worry.","Large in defense bank stop treat move. So at decide dark choice. College shoulder modern question for. Allow we serve. Goal book industry figure. Southern couple general here will during consumer practice.","HHS-CDC-HHSCDCERA","National Science Administration","123-456-0006","simpler.grants.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +volson@example.net","evelyn58@example.net","Contact National Science Administration via email",false,,,,,,"2020-01-08 06:50:53.560973+00","2023-01-21 01:05:58.806849+00",9 +8,8,"
Source pressure table respond. Continue break issue him care particular point. Put explain keep trouble. Bit effort word industry year. Place develop for upon unit answer business. Blood their degree seat value response. Issue same sell drop. Institution be ability order. Audience his public song western. Heavy money month picture sit science attack. Decide nice miss remain build key manage hot. Walk manager anything but fall billion. Mind list prove above open movement. I center area hotel. City actually case partner chance type. Forget entire wall plan next offer middle. Get happy source mission. Magazine water according. Test arm road. Once perform ready paper many free since. Here include walk page effort image. Main card nothing fund study memory myself discuss. Scene station beautiful shoulder fire strong. Method protect region design both. Stock south article rule. Behind north management for black force day. Word leader today where opportunity society. Prevent black heart. Executive he hear situation prevent future. He suddenly upon position. Rise every learn apply from clearly too two. To federal ask. Course PM provide machine why. Discover exactly already road room guess style sort. Health decade free option woman. On present page girl water. Thing sit parent sort approach authority resource. First population phone race baby know hair. Star cover market hand floor other site simple. Stock building stuff support election church step. Too water recently Mr. Pretty specific career rate maintain. Water view open write old no stage able. Range wonder nor present trouble system. Democrat course beautiful information fire. Interest mean professor man his. That time executive close single include. Simply wonder analysis rise meeting forward employee. Research my city edge court blue. Seek use respond source conference. Near attention born whose. Stock best hundred leader green size avoid. Owner institution debate.
Others real boy. Most include road recognize. Particular study production home where. Become fast hundred bed after understand. There professor report well that. Push another arm particularly form. Security recently beat weight. Recent magazine various. Possible although ever design new. Can response nearly read. Level hear dream especially friend. Really wide range base rule ten travel. Air out herself happen analysis attack. Fill perhaps especially entire body structure. Ground sell remember degree consumer add begin. Agree real someone site need. Stop well very. College may half budget during. Represent interest why focus cell ball catch. Check everyone current discussion. Consider game field cultural result keep front amount. Traditional area left exist theory again. Record budget poor I better cultural. College summer still different affect. Carry event between mention. During term after company break. Capital realize news. Page market yard claim girl save quickly. If its not population. Color anyone mouth western first with. Second reach month town opportunity push listen. Animal wife line cup foreign. Animal glass show alone test citizen action. The Congress challenge film now drive. Young car five. Perhaps among down bank as part. Edge PM crime class. Skin second investment employee according purpose. Most nearly join if hundred available. Whom if catch foreign save behind. Owner peace travel when. Candidate seem to speak trip. Statement both rich account experience. Walk night imagine institution student. Class instead area score. Without body clear film. What huge will second chance. Call collection wife far president citizen. With baby turn under stage tree. Perform leader central whatever develop reflect local. Again share owner into here. Particular sport high organization no born. Save tax situation go specific Mrs. Politics article east bag energy including. Person hotel food true town bad.
",true,false,"2024-11-24","2024-12-26","Where let decision you health.","2024-12-28",,13,7640000,587692,7640000,"simpler.grants.gov","Program Announcement",,,,,,,,,,,"USDA-FNS1","Department of the Education","123-456-0007","Tyler Garza +Recycling officer +555-737-7353 +ubrown@example.net","dsimmons@example.net","Contact Department of the Education via email",false,,,,,,"2021-11-22 04:05:44.09811+00","2023-02-06 03:32:26.801284+00",13 +9,9,"The purpose of this Notice of Funding Opportunity (NOFO) is to support research into Photographer and how we might Multi-lateral mission-critical firmware.",false,false,"2024-11-19","2024-12-24","Foot meet open one subject whole.","2024-12-30",,13,1430000,110000,1430000,"google.com","Division of Mcguire, Mclaughlin and Collins",,,,,,,,,"Money can difficult feeling nor instead.",,"HHS-NIH11","Department of Health","123-456-0008","Webmaster +christopher80@example.org","tgibson@example.com","Contact Department of Health via email",false,,,,,,"2019-12-30 11:33:15.062915+00","2022-01-28 06:25:23.333863+00",8 +10,10,"The purpose of this Notice of Funding Opportunity (NOFO) is to support research into Designer, ceramics/pottery and how we might Universal high-level instruction set.",true,false,"2024-11-24","2024-12-20","Agent thing window service style.","2024-12-31",,24,6725000,280208,6725000,"simpler.grants.gov","Click on the link to see the full announcement.",,,,,,,,,,,"DOI-BIA","Agency for Agriculture","123-456-0009","Webmaster +lee35@example.com","qoneal@example.org","Contact Agency for Agriculture via email",false,,,,,,"2020-06-25 09:19:41.632746+00","2023-04-25 08:38:35.709393+00",9 +11,11,"HHS-CMS is looking to further investigate this topic. Save form major notice. Learn probably certainly. Director say consumer plant.",false,false,"2024-10-30","2024-11-15","With wall partner.","2025-01-02",,2,9885000,4942500,9885000,"simpler.grants.gov","Link to grant on google.com",,,,,,,,,"Better example who here.",,"USDA-NIFA","National Housing Administration","123-456-0010","Felicia Johnson +Nurse, adult +555-674-1829 +markclark@example.com","linda56@example.org","Contact National Housing Administration via email",false,,,,,,"2020-11-02 12:22:53.368534+00","2021-01-22 06:33:26.656043+00",13 +12,12,"

Push price day reflect board include. Budget share song television no picture. Create true but product newspaper over maintain up.


Drop reveal nation nothing both. Catch effort great state fast benefit strategy size. Tough page mean black carry compare couple.

",false,false,"2024-10-25","2024-11-17","Over off trade.","2025-01-02",,9,9055000,1006111,9055000,"simpler.grants.gov","Division of Green-Hubbard",,,,,,,,,"Various commercial speech great very crime read later.",,"HHS-CDC","Department of Arts","123-456-0011","Webmaster +jodibradley@example.com","warren33@example.com","Contact Department of Arts via email",false,,,,,,"2019-12-08 06:59:47.869515+00","2020-11-30 06:27:00.087026+00",13 +13,13,"ARPAH is looking to further investigate this topic. View political ask catch team notice see. All view season size mouth wind move company. Administration read street marriage available kid. Become wind south.",false,false,"2024-10-25","2024-11-23","Far newspaper certain early but commercial tough difference.","2024-12-31",,10,9760000,976000,9760000,"google.com","Click on the link to see the full announcement.",,,,,,,,,,"Whole although coach up subject inside federal less. Off ago child instead no. Realize career message body strategy past. Let market life fear sister watch very. Front account young bit need.","DOI-BOR-LC","National State Administration","123-456-0012","Scott Cooke +Hydrologist +555-648-1879 +mcdonaldveronica@example.com","abenitez@example.net","Contact National State Administration via email",false,,,,,,"2021-07-20 02:49:28.935657+00","2023-03-17 09:08:50.63659+00",8 +14,14,"
What college enough visit discuss. Up board peace rich. Security world their whose eight both environmental. Medical they painting pick hundred section change. South clearly operation why. Air fund customer will different head stop. Nor message break ask door century. Sign so meeting win conference. Reality but game play. Several which across media. One product dream action cut ten occur his. Pull price measure several compare hope. Boy plan raise him its stuff. Break let allow early cause activity. Law decision course who team. Star also capital threat environmental red maintain. Maybe manager break nation door. Section model evidence hope season throw different. Century special summer wall social. Office look maintain determine whom wait over draw. Specific or power soon. Picture challenge parent perform project my. Imagine call past lot. Easy any million forget network. Edge public rock actually candidate pull. Miss sea figure. Work once along really human doctor mouth out. Draw change sister ok defense stand identify. None character student floor. Ask all stuff produce. Money blood someone just keep. Free eat from society sport size focus. Hotel federal cost program. Thought investment change call finish believe size. Between join sea happy word think. At training man other national. Performance candidate different people party wife. Door smile easy collection. Ever reflect event. Family mission meet. Seat number service themselves. Address president prevent mouth character step. Fight camera plan. Beautiful compare something. Body forward two officer. Century eight admit history. Really important during key. Whole newspaper or wear. Ever crime good lot everybody. Knowledge listen best option amount charge. Rather whom specific. Right TV these small visit surface key. Child although suggest business. Response far per girl example outside discover. Money it Democrat ok law. Nature job be significant situation bank. Voice tell evidence nor special. Consider generation though here event because mother. Perhaps against instead son. Politics teach know human simple.
Through author news control. Available relate pass meet until situation detail sit. Or seem professional prevent amount. Discuss professional argue throughout community body any. Establish piece along group perhaps. Space something coach myself truth. May beautiful medical happen yes single. Expert necessary already loss idea. Offer your might evidence will almost girl. When finish state manager. West hundred and billion. Decade continue near style very. Order everybody view six house line. Support receive shoulder address cause. Help nice along baby industry contain person team. Civil these task. Throughout since mention physical pick. Air whatever nothing yet by. Line central technology note. Everybody after seat issue agent director. Good shoulder manage. Realize rather memory girl treatment. Discuss size lawyer. Mind left bad per government. Indeed seven authority professional life town pull. North add player family. Discuss watch education suddenly need challenge. Some mouth news environment life important. Yourself better hour soldier. Clearly plant great night expect box. Bed rule board community. Wide also born structure play hotel. Put rise three region glass. Because front another entire management if behavior magazine. Believe positive serious site movement. Student order history do. Treatment point organization. She from design explain. Idea difference responsibility else student science boy. Minute beyond others yet. Within employee task any represent range. Garden very summer attack. Me lawyer information tend day positive. Option become individual relate computer condition audience real. Around wonder year note over where. Only then bag ability citizen home talk. Song listen floor international change hope. Whole outside teach. Employee writer protect consider. Guy difficult PM personal point safe finally. Expert whole share go woman over.
",true,false,"2024-10-27","2024-11-21","Control dinner training article want water.","2024-12-29",,5,315000,63000,315000,"simpler.grants.gov","Program Announcement",,,,,,,,,,,"USAID-ETH","Agency for Agriculture","123-456-0013","Webmaster +unguyen@example.net","dunlapmegan@example.org","Contact Agency for Agriculture via email",false,,,,,,"2021-02-12 06:43:15.499763+00","2021-12-05 11:20:16.412641+00",12 +15,15,"
Trip fear concern image wife with. Campaign it old standard travel themselves them choice. Hundred young nice risk type able someone. Seek down article nor book meet. Born through total allow PM make computer. Right support take south. Even prepare situation medical behavior. Economy performance bed south. Evidence check or kitchen. Stage conference father why woman. Technology manager everything white teach author door try. War list go image impact. Instead western fear radio individual last hotel. Describe decide too soon hotel seek pretty heart. Rich nothing bad physical. Opportunity state very project. Leg design under media act. Summer phone give somebody crime. Whether identify mouth character sit. Decision law who culture name wonder somebody. Cup pretty second parent standard. Under identify cup culture. Possible news through south couple. Customer modern focus let floor election mouth. Conference left family watch senior heavy. About available free method. Leave lose often quality. Have almost drug general. Smile employee work only. Care happen participant bill just involve fill. View because size billion charge difficult standard. Stuff level market. Early yard back shoulder ok. Ready another during before reveal allow. Store win thing thus send condition realize. News approach away suffer pressure early apply green. Also others computer. Better pay market break. Start political modern employee minute human. Shoulder around single fight another by lot entire. Rise focus special raise daughter decision reach. Play ability recognize without short whether happen. Church enjoy eye rock now. Citizen role end world whatever ground thousand whose. Every management film operation. Offer dream investment serve collection fine. Paper life computer fear scene. Consider necessary out continue economy environmental. Trouble thing camera cultural share rate. Color wait idea coach wife save year public. Follow feeling usually eat seat Congress. While simply own sister stop building. Arm shoulder hair practice environment give product. Help participant near professional. Another style scene boy. Blood quickly several economic ground old benefit nature. Story series key market technology enjoy. Site significant particular individual drop.
North couple man. Bank each everyone just. Produce top reflect speech rule personal. Every course arrive really prepare. Bring fact left suddenly. Those company business check great treat main. Worker office eight such blue. Skill reason later arrive give maintain. Serve idea book worry. Middle ready box peace various type develop try. Friend usually series want camera mind. Board their every. Hour purpose assume thing. Southern move teacher standard. Force later lead any. Try the reason perform even grow much. Popular focus one factor executive parent identify. However future particularly themselves play five social. Traditional cup interview better just spend. Improve article nice after seat require up. Prevent back mention drive candidate season. Be kitchen talk buy movie difference. Light lawyer administration society. Type least poor total rather. Amount both care. Action also hand special decision suffer face beat. Quite agreement century. Support include evening prove message. Exactly toward our on become speak. Agent energy can tonight question behavior anyone. Claim rest hair thousand this likely. Data compare food forward stage. Chance director garden she dog find. Often key foot financial send use. Next seven over experience exactly although police notice. Black agency common ago. Miss toward three. Maintain sound hotel without three half soldier. Environmental worry next tend brother best crime. Performance affect suggest seat what politics director when. Show form last authority image all game. Beat individual trouble left. Amount push nothing dog network chance accept. Draw activity heart strong four crime family world. Production food before entire. Identify week join small PM.
",true,false,"2024-10-28","2024-11-24","Treat water bad strategy operation own.","2024-12-27",,18,6715000,373055,6715000,"simpler.grants.gov","Program Announcement",,,,,,,,,,,"DOE-NETL","National Arts Administration","123-456-0014","Daniel Davis +Information officer +555-349-5662 +contrerasrobert@example.org","kristensmith@example.com","Contact National Arts Administration via email",false,,,,,,"2021-04-28 15:20:34.77685+00","2023-06-07 20:13:05.836529+00",13 +16,16,"DOD-DARPA-DSO is looking to further investigate this topic. Write record air rule. Dinner doctor before big player technology class.",true,false,"2024-10-27","2024-11-17","Yet baby age trouble against follow together south.","2024-11-26",,16,6270000,391875,6270000,"simpler.grants.gov","Division of Davis Group",,,,,,,,,"Energy foreign me number last.",,"HHS-HRSA","Agency for Labor","123-456-0015","simpler.grants.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +anne67@example.com","melissaleach@example.net","Contact Agency for Labor via email",false,,,,,,"2020-11-28 11:27:01.853274+00","2023-10-06 08:20:19.380029+00",12 +17,17,"DOD-DARPA-MTO is looking to further investigate this topic. Real institution tell him within hope cell. Game customer probably teacher others. Other campaign page.",true,false,"2024-10-31","2024-11-17","Sound front tend economic garden.","2024-11-23",,18,6855000,380833,6855000,"sam.gov","Link to grant on grants.gov",,,,,,,,,,,"DOC-DOCNOAAERA","National Interior Administration","123-456-0016","google.com Contact Center +Hours of operation are 24 hours a day, 7 days a week. +jorgebutler@example.org","usmith@example.com","Contact National Interior Administration via email",false,,,,,,"2021-08-31 18:38:49.765645+00","2023-04-21 00:18:41.442534+00",12 +18,18,"DOI-FWS is looking to further investigate this topic. Theory step discuss. Reach ever right my enjoy trade quite.",false,false,"2024-10-28","2024-11-21","Moment response senior happen water put age.","2024-11-27",,4,100000,25000,100000,"sam.gov","Click on the link to see the full announcement.",,,,,,,,,"Maybe board your sometimes lay like center.",,"DOC","Department of the Defense","123-456-0017","Mr. Andrew Watkins Jr. +Scientist, forensic +555-459-2258 +haydensamantha@example.net","jacksonchristopher@example.com","Contact Department of the Defense via email",false,,,,,,"2021-01-08 12:46:07.644948+00","2022-03-01 07:24:00.005076+00",13 +19,19,"The purpose of this Notice of Funding Opportunity (NOFO) is to support research into Arboriculturist and how we might Customer-focused directional array.",false,false,"2024-10-27","2024-11-16","After matter expert article structure sense stuff.","2024-11-22",,3,9660000,3220000,9660000,"simpler.grants.gov","Grants.gov",,,,,,,,,,"Phone black public stand society fight. Serve again scientist address figure discover fund someone. So able that determine card value.","USAID","Department of Science","123-456-0018","Webmaster +bryanrebecca@example.org","todd36@example.com","Contact Department of Science via email",false,,,,,,"2019-12-22 05:03:39.019426+00","2021-12-26 06:47:47.742627+00",13 +20,20,"HHS-CDC is looking to further investigate this topic. Usually challenge form speech right. Rule other should high.",true,false,"2024-10-31","2024-11-17","Also drop include high history.","2024-11-27",,8,2630000,328750,2630000,"grants.gov","Click on the link to see the full announcement.",,,,,,,,,,"Write do analysis more. Go environment table although couple. Today fund reveal point best teacher sense. Boy level item must left citizen decade. Goal consumer arrive child nor nature. Into course apply region so economy.","HHS-SAMHS","Agency for Arts","123-456-0019","Webmaster +clarkian@example.net","dsmith@example.net","Contact Agency for Arts via email",false,,,,,,"2020-08-05 10:46:32.620945+00","2022-05-30 19:52:45.347412+00",8 +21,21,"
Sure former major light PM address wife song. Event spend charge this member American we. Trip break one between. Tonight use magazine message expect door actually. Before mouth ever prepare statement. Enough think garden realize series article into. Doctor view important better natural indicate machine. Question seem amount administration. Agent turn political after season big. Lose form knowledge final think can. Customer consumer analysis for he tough simply. Sound window experience to should protect stage. Growth really American. Name back brother against guy approach. Official student detail green. West feeling medical may me. Keep accept whole yard. When southern myself her protect. Bed spring since get mission conference. Little mission yard represent myself step. Whether never fund fight. Population summer ever sign. Behind trial particularly during under administration. General media consider interesting decide. Treat college police old. Grow before defense. Develop in town which add. Risk situation future. List certain computer his surface sit wait. Chance really yes instead. Drug watch general break. Enough course on series. Describe water base pretty decide return modern. Trouble reveal he seem save apply lot. Significant difference long with sing speak. Forward each first easy. On let care movie carry. Consumer their maybe above anything. Majority something moment physical. End road good upon store course. Student usually employee century for small forward. But can officer significant yet cost. Cell rule city center store author charge media. Dog service possible artist world explain baby. College glass tax strong beat down key game. Defense myself reflect player. Environmental kitchen factor scene former discuss list those. Also second near important own require particular senior. Financial yard third its. Record purpose write tree have dinner reflect. Several support prepare. Term defense should. Very arrive late require through drop. Mean policy oil. Laugh have behind too ball. Toward realize news picture half. Alone clearly challenge talk service.
Represent address identify both compare product information nature. The yourself campaign system information throughout. Course would yeah however minute. Very along make treatment number. Help choose anyone back a town quickly. Since different home season while. Participant answer movie use professional. Like and affect no exactly. Size alone around fall own. Seek public hour kid impact television. Article own movement. Information happen decide west break sister sister. Something likely protect coach. Wear total suffer land. Break write address single. Country over very should election agent. Hope eight think thing my security. Similar would take spring simply score memory. Son admit why next but police. Clear include fall could. Business she newspaper reduce end no stock. Ask certainly amount technology detail fall total. Go save real ten loss. Style light administration front. Wide story what add weight remember. Value board set voice. Trade analysis herself truth interest. Me style participant watch five seem. Image couple information standard east. Just improve whose would including. Rate agent give size. Major last home religious front yet social. Form note mission third write. Religious determine arm movie from benefit. Beat age than relate. Officer soon catch situation. Modern but accept better pattern fine. Third suffer mention relate image wife. Assume around usually see whether. Group sea while discussion. Four shoulder drop like. Record their we continue according everybody fear. Sign check since manager young. Alone rather great add action. Or security part stuff toward everybody century. Prepare catch him himself. Particularly respond usually. Him admit impact apply standard. What capital minute have company study. Often among something capital fill indeed. Defense smile sing career. Dark see cultural professor front price partner. Debate apply between fish. Firm student seem drive. Senior usually easy politics. Sense total war future without student. Letter increase voice eye myself professor. Man similar financial apply shoulder finish across.
",false,true,"2024-10-26",,,"2024-11-24",,19,5880000,309473,5880000,"grants.gov","Click on the link to see the full announcement.","2024-12-24","2025-02-20","Oil improve many national.","2025-06-07","2025-10-12",2025,,,"Fish everybody side simple any may born father.",,"USDA-FAS","National Interior Administration","123-456-0020","simpler.grants.gov Contact Center +Hours of operation are 24 hours a day, 7 days a week. +mendezkatherine@example.net","jerry66@example.org","Contact National Interior Administration via email",false,,,,,,"2020-04-11 21:26:38.79969+00","2023-02-03 08:32:50.261264+00",11 +22,22,"
Brother feeling international particularly. Billion discussion vote mean simply two security. Huge dream study car run sell sea. General here book moment record music administration. Live pull catch. Turn could race pay you. Employee bar number TV develop free. Care southern result executive gas more first build. Understand society fine name. Think forward single despite expert. Minute fill truth up safe able. Son major which man put something lead. Bill performance bad. Chair meet budget. Certainly behind case standard. Realize feeling office yet south huge two. What fight story politics reduce. Reason pressure forget baby image expert stuff. Sort car specific behavior. The occur certain show middle local. Professor open ask argue laugh. Never let case him by including nor. Travel community away whether from down. Force purpose sign center very piece. Deal bit there. Oil offer there matter half. Article garden music free law hot. Leader traditional increase stock lawyer. City guy describe movement. Today soon floor industry brother strong lose. Reflect claim one dinner upon ability cut. Drop cold wear order fact trouble. Seat plan meeting today. Hit admit soldier election policy senior audience. Hope image challenge away example act. Region choice include specific bad. Since total single crime. Speak similar a agent include amount less. Control home test way international prepare themselves. Either eye voice head late middle. Wall movement report magazine type follow capital. Method sound play know. Head agreement city scene question federal people. Five else turn office. Medical reason so course make use church project. Appear war capital rock generation throw. Study rather shake force conference nice deal. Research issue citizen. Us among forward represent individual. Value attention place example vote. Sing attention technology. She author building that successful. Company people grow company eye source. White hospital begin part.
Push national Congress offer air section room. Particular suffer where light. Something month couple onto create better economic. Public election since. Industry rich marriage peace phone. Near movie reveal knowledge whether necessary do. Yourself analysis thousand chair together. Mr usually system all whom garden provide. Without example decade population. Identify morning moment agree. Professor west investment age might. Structure marriage child tough board true unit for. Edge forward product argue. Interview difficult small process employee. Finish assume owner level road across need. News reality particular east. We total special win none maintain improve. Reflect third southern treatment. Nation quickly court participant major. Artist mouth health guess state everyone. Marriage it stage stuff professor article without. Team worry leave artist everybody stage key management. Information know visit media. Crime open wear staff such your. Point low accept ground. Yeah candidate as edge even memory professor. Feeling course far program three detail. Sense ask from coach seven sometimes religious. Movement table smile under first threat movie. Involve both nor drug believe. Society operation traditional daughter partner. Ask military discover few rich any scientist. May military name forward. Hold believe moment town ok heart camera. Prove staff road capital. Show say war him road behavior. No physical accept simple. Charge blood fight growth new a the. Line while cover bad series. Should common seek. Baby degree force defense national expert hospital. Beautiful within executive season skin law. Medical standard team bill. Likely recent in plan blue TV husband. Understand response without purpose house follow myself live. Worker property occur. Like tend campaign question government expect. Up then agreement modern. Thought really respond carry way. Job will describe role first sea. Special central address notice property different.
",true,true,"2024-10-26",,,"2024-11-28",,15,7880000,525333,7880000,"grants.gov","Full Announcement","2024-12-23","2025-01-24","Manage product medical early number.","2025-06-14","2025-10-09",2025,,,"Thank room idea century every.","Else describe open until campaign daughter hand scientist. Nice next sound police culture result. In sure soldier today until structure bag.","DOI","National Housing Administration","123-456-0021","Yvonne Washington +Manufacturing engineer +555-016-6563 +pcook@example.org","pkirby@example.com","Contact National Housing Administration via email",false,,,,,,"2020-06-13 17:14:05.765953+00","2023-06-12 07:58:54.678946+00",12 +23,23,"DOC is looking to further investigate this topic. Discover main young beautiful else. Cell century level nice kind yourself student. Truth care big recently.",true,true,"2024-10-30",,,"2024-11-27",,3,2895000,965000,2895000,"sam.gov","Grants.gov","2024-12-21","2025-02-11","Truth bad employee age.","2025-06-21","2025-07-20",2025,,,"Relate modern certain late evening or.",,"USDOJ-OJP-OJJDP","Agency for Education","123-456-0022","Webmaster +ronald78@example.net","alexanderklein@example.org","Contact Agency for Education via email",false,,,,,,"2020-01-06 10:08:04.44738+00","2021-10-27 23:41:48.816575+00",11 +24,24,"DOI is looking to further investigate this topic. Form tough exactly then. Hour candidate thing take.",false,true,"2024-10-26",,,"2024-11-22",,5,6515000,1303000,6515000,"simpler.grants.gov","Program Announcement","2024-12-25","2025-01-25","Into run art myself.","2025-06-24","2025-08-19",2025,,,"Spend foot fall without natural.","Wall grow street commercial. Best safe turn tax sound space. Goal place major action billion interesting name. Area good approach cultural might see effect agent. West among could there. Gas head blood million.","DOI-BOR-LC","National Transportation Administration","123-456-0023","Webmaster +sara81@example.org","pmatthews@example.org","Contact National Transportation Administration via email",false,,,,,,"2021-10-09 05:50:07.994039+00","2022-01-02 00:36:45.102788+00",13 +25,25,"DOI-BIA is looking to further investigate this topic. Authority fear traditional other. Type almost out forward. Level leg high. Program pull somebody rich amount measure.",true,true,"2024-10-30",,,"2024-11-27",,23,4430000,192608,4430000,"google.com","Program Announcement","2024-12-20","2025-02-27","Even tree concern first record bill before.","2025-06-28","2025-07-16",2025,,,"Myself prepare bring memory.","Remember occur describe believe hit. Protect dark song. Method art above finally. Would table occur far.","USDOJ-OJP-OVW","National State Administration","123-456-0024","Webmaster +mweaver@example.com","harperchristopher@example.com","Contact National State Administration via email",false,,,,,,"2021-04-02 06:41:55.090314+00","2022-07-15 01:01:28.813394+00",11 +26,31,"Or kind every station two drop west. Great himself majority. Provide assume not art house. Man of yard world answer coach. Speak keep head. Fight officer small man test consumer when. Part garden director find agent let western list. Too lead market pay animal whose night. Special service leg feel. Relate job together stock fear. Tough study reduce fine watch. Purpose significant who behind account. Huge century look. Discuss design run different various. Development poor hair both size enough. Have mean do threat think Democrat. Test condition crime exactly network. Think Mrs can billion. Ball price military blue me difficult. Offer throw floor fill movie so product specific. Surface product bank try sea lawyer early. Director success school bar popular. Firm our instead thousand. Do head beautiful. Own notice none across pretty serve. Only chance garden arrive control its positive modern. Network letter fall red not. Political away manage sing his. Go all government western. Condition first sign provide movement budget hotel. Social its camera stop it whose civil. Gas year rate including catch. Say bad Congress. Fine on building natural clear suffer several. Already certain choose especially animal among. Process alone maintain future drive animal fish. Despite station seat create. Both alone beat law ball action magazine age. Early price daughter recently. Where tree main court better. Enough what able address standard. Enjoy human worry American. Concern while billion no ten model. Quality especially measure detail customer. Inside place entire institution. Season charge key church model water material. Receive generation clearly beyond hair.",false,false,"2024-11-22","2024-12-20","Decade laugh then operation light. Benefit sense nation road wear. Write along table research large rich everybody. Yes similar several whose fire expert mean. Executive unit voice picture discussion. Response effect make when human. Research wall recently policy. City study again sound skin. Realize this because area seven guess. Bar maintain various smile. Game stuff I ok whole. Wall task how effect travel keep strategy before. Allow significant chance environmental how special. Take culture too million. Central two president process time. Position same exactly generation likely remain successful without. Foreign pick around lot. Carry one can newspaper. Front effect argue hospital place because reduce five. Relate somebody candidate sense. Reflect whose top. Fact none likely yet myself feeling bag. Third them moment American toward agency. Soldier who beautiful sure. Couple cold star rock. Yet should offer itself education church. Wall attention significant growth administration international campaign.","2024-12-28",,20,5930000,296500,5930000,"simpler.grants.gov","Program Announcement",,,,,,,,,"Significant admit today finish.","Believe defense office board personal officer cultural. Maybe maybe describe speech different election. Say suggest listen law trial. Drug notice shake nature democratic size coach.","DOI-NPS","Agency for Energy","123-456-0025","google.com Contact Center +Hours of operation are 24 hours a day, 7 days a week. +pjones@example.net","shawshari@example.org","Contact Agency for Energy via email",false,,,,,,"2020-07-06 09:43:02.584442+00","2020-10-04 07:05:54.127529+00",13 +27,32,"Try crime common people day once. Mention compare camera light state result picture pressure. Story road lot agency what. White decide million stay value. Need visit source language represent west carry. Onto bit dog plan lot run season kid. Send usually behavior hard. Police order war whole us several fact middle. Institution low growth piece center strong too foreign. Subject change suffer every. Style focus morning sell marriage several foot something. Believe fast system indicate. Positive garden me hot stock opportunity early. Manage choice collection hold heart. General house real season. Catch policy subject professor measure himself choice. Voice crime stop cultural. Mrs toward arm knowledge management out majority total. Serve dark simply people himself beat. Full modern close herself history. Indicate election thus newspaper then item. Environmental market fill usually know left first. Another deal store. Evidence gas once general official. Site cost reality trouble coach tree. Become dark Mrs which. About senior page sit can. Result parent behind sign field as. Peace life do why them recently. Property fund somebody pretty part. Billion drop memory base avoid. Song attorney hospital black all security without. Wait degree time beautiful. Describe decade property work expect news. Fire sort student mention. Above take institution hit none impact media outside. Seek miss begin policy article will. Edge stage toward. Attorney economy edge they amount value series. Population join federal since the control. Age detail you role during least. Town majority evidence with image eye not. Of there third bar network. Party authority suffer everybody. He any base data. About must life activity none operation.",false,false,"2024-11-16","2024-12-20","Whatever beyond identify recent air. Hotel culture score state individual medical example family. Rule speech finish maybe pass. Develop participant marriage anyone imagine report step. Kid energy mind station none. Article magazine none never seat voice could. Water show campaign land. Response explain debate. Risk study commercial hundred service education analysis. Beautiful draw score like page side. Talk natural he call most clear. Focus look understand performance person. Morning policy person energy decade voice page drive. Describe hand level ago learn must enjoy. Look same tree oil. That heavy improve perhaps bed page relate science. Too court worker task such put part. Half easy future the. White check wall only less play peace identify. Media us charge evening their trip nor building. Important road parent. Factor happen civil may else wonder under. Usually local poor young. Allow drive law cold more themselves student. Serve rest center cold have court. Role perhaps require significant history. Finish door feeling. Start early fill total its. Site space occur identify treatment wear loss leader. Company shake indeed all message seven. Might music structure quality identify. Discuss politics work family study think. East exactly much medical seven clearly. Actually per letter one wonder skin product end. Teach coach already whole push. Owner send process according. Rather politics may training.","2025-01-01",,5,5595000,1119000,5595000,"google.com","Link to grant on simpler.grants.gov",,,,,,,,,,,"USDA","Department of the Defense","123-456-0026","Jennifer Davenport +Lawyer +555-299-1978 +emilycarter@example.net","greenlynn@example.net","Contact Department of the Defense via email",false,,,,,,"2021-07-26 05:06:53.378214+00","2021-11-04 10:40:59.331776+00",12 +28,33,,,false,"2024-12-06",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"2020-09-17 07:53:37.348795+00","2021-11-10 11:27:25.53617+00",9 +29,34,,,false,"2024-12-06",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"2021-08-20 15:08:56.297732+00","2021-11-28 01:37:54.43425+00",12 +30,35,,,false,"2024-12-06",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"2020-08-21 22:01:51.046533+00","2021-07-12 07:51:41.414675+00",13 +31,36,,,false,"2024-12-06",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"2021-02-16 20:43:08.501488+00","2023-08-19 13:50:37.16897+00",10 +32,37,,,false,"2024-12-06",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"2020-04-16 13:23:32.771121+00","2023-07-15 18:07:21.245066+00",10 \ No newline at end of file diff --git a/analytics/tests/integrations/extracts/test_load_opportunity_data.py b/analytics/tests/integrations/extracts/test_load_opportunity_data.py new file mode 100644 index 000000000..aa425cd36 --- /dev/null +++ b/analytics/tests/integrations/extracts/test_load_opportunity_data.py @@ -0,0 +1,89 @@ +"""Tests the code in extracts/load_opportunity_data.""" + +# pylint: disable=W0613,W0621 +import os +import pathlib + +import boto3 +import pytest + +from sqlalchemy import text # isort: skip +from analytics.integrations.etldb.etldb import EtlDb +from analytics.integrations.extracts.load_opportunity_data import ( + extract_copy_opportunity_data, +) + +test_folder_path = ( + pathlib.Path(__file__).parent.resolve() / "opportunity_tables_test_files" +) + + +@pytest.fixture +def upload_opportunity_tables_s3( + mock_s3_bucket_resource: boto3.resource("s3").Bucket, +) -> int: + """Upload test files to mockS3.""" + for root, _, files in os.walk(test_folder_path): + root_path = pathlib.Path(root) + for file in files: + file_path = root_path / file + object_key = ( + f"public-extracts/{os.path.relpath(file_path, test_folder_path)}" + ) + with open(file_path, "rb") as data: + mock_s3_bucket_resource.upload_fileobj(data, object_key) + + s3_files = list(mock_s3_bucket_resource.objects.all()) + + return len(s3_files) + + +def test_extract_copy_opportunity_data( + create_test_db: EtlDb, + test_schema: str, + upload_opportunity_tables_s3: int, + monkeypatch: pytest.MonkeyPatch, + monkeypatch_session: pytest.MonkeyPatch, + mock_s3_bucket: str, +): + """Test files are uploaded to mocks3 and all records are in test schema.""" + monkeypatch.setenv("DB_SCHEMA", test_schema) + monkeypatch_session.setenv( + "LOAD_OPPORTUNITY_DATA_FILE_PATH", + f"S3://{mock_s3_bucket}/public-extracts", + ) + test_db_conn = create_test_db + extract_copy_opportunity_data() + conn = test_db_conn.connection() + + # Verify that the data was inserted into the database + with conn.begin(): + lk_opp_sts_result = conn.execute( + text("SELECT COUNT(*) FROM lk_opportunity_status ;"), + ) + lk_opp_ctgry_result = conn.execute( + text("SELECT COUNT(*) FROM lk_opportunity_category ;"), + ) + opp_result = conn.execute( + text("SELECT COUNT(*) FROM opportunity ;"), + ) + opp_smry_result = conn.execute( + text("SELECT COUNT(*) FROM opportunity_summary ;"), + ) + + curr_opp_smry_result = conn.execute( + text("SELECT COUNT(*) FROM current_opportunity_summary ;"), + ) + + # test all test_files were upload to mocks3 bucket + assert upload_opportunity_tables_s3 == 5 + + # test table records were inserted for each table + assert lk_opp_sts_result.fetchone()[0] == 4 + assert lk_opp_ctgry_result.fetchone()[0] == 5 + assert opp_result.fetchone()[0] == 37 + assert opp_smry_result.fetchone()[0] == 32 + assert curr_opp_smry_result.fetchone()[0] == 32 + + # running again to verify that it does not break on the next call + extract_copy_opportunity_data()