diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b7d430c57..766856111 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,5 +5,6 @@ Please explain the changes you made here. - [ ] Format code according to the existing code style (run `npm run lint:check -- CHANGED_FILES` and fix problems in changed code) - [ ] Create tests which fail without the change (if possible) - [ ] Make all tests (unit and integration) pass (`npm run test:unit` and `npm run test:integration`) +- [ ] Extend the types in index.d.ts file (if necessary) - [ ] Extend the README / documentation and ensure is properly displayed (if necessary) - [ ] Provide JIRA issue id (if possible) or GitHub issue id in commit message diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 7aa959680..fd0698c9b 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -17,9 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Set up Node.js - uses: actions/setup-node@v1 + uses: actions/setup-node@v4 with: node-version: '18.x' - name: Install dependencies diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 339a96dd6..67b31a63c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -36,7 +36,7 @@ jobs: WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }} run: ./ci/build.sh - name: Upload Build Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: artifacts path: artifacts @@ -49,17 +49,17 @@ jobs: fail-fast: false matrix: cloud: [ 'AWS', 'AZURE', 'GCP' ] - nodeVersion: [ '14.x', '16.x', '18.x', '20.x', '22.x'] + nodeVersion: ['18.x', '20.x', '22.x'] steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.nodeVersion }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.7' - name: Download Build Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: artifacts path: artifacts @@ -73,7 +73,7 @@ jobs: CLOUD_PROVIDER: ${{ matrix.cloud }} run: /usr/local/bin/bash ./ci/test_mac.sh - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: # without the token code cov may fail because of Github limits https://github.com/codecov/codecov-action/issues/557 token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }} @@ -87,18 +87,18 @@ jobs: fail-fast: false matrix: cloud: [ 'AWS', 'AZURE', 'GCP' ] - nodeVersion: [ '14.x', '16.x', '18.x', '20.x', '22.x'] + nodeVersion: ['18.x', '20.x', '22.x'] steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.nodeVersion }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.7' architecture: 'x64' - name: Download Build Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: artifacts path: artifacts @@ -121,7 +121,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Download Build Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: artifacts path: artifacts @@ -145,12 +145,12 @@ jobs: steps: - uses: actions/checkout@v4 - name: Download Build Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: artifacts path: artifacts - name: Use Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - name: Tests @@ -160,7 +160,7 @@ jobs: CLOUD_PROVIDER: ${{ matrix.cloud }} run: ./ci/test_ubuntu.sh - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: # without the token code cov may fail because of Github limits https://github.com/codecov/codecov-action/issues/557 token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }} diff --git a/.github/workflows/jira_close.yml b/.github/workflows/jira_close.yml index dfcb8bc73..0dacf7fab 100644 --- a/.github/workflows/jira_close.yml +++ b/.github/workflows/jira_close.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: snowflakedb/gh-actions ref: jira_v1 diff --git a/.github/workflows/jira_issue.yml b/.github/workflows/jira_issue.yml index dd4b717bc..94a0a0890 100644 --- a/.github/workflows/jira_issue.yml +++ b/.github/workflows/jira_issue.yml @@ -14,7 +14,7 @@ jobs: if: ((github.event_name == 'issue_comment' && github.event.comment.body == 'recreate jira' && github.event.comment.user.login == 'sfc-gh-mkeller') || (github.event_name == 'issues' && github.event.pull_request.user.login != 'whitesource-for-github-com[bot]')) steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: snowflakedb/gh-actions ref: jira_v1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 87a48ab4b..8708c72a8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,9 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Set up Node.js - uses: actions/setup-node@v1 + uses: actions/setup-node@v4 with: node-version: '18.x' - name: Install dependencies diff --git a/.github/workflows/parameters_aws_auth_tests.json.gpg b/.github/workflows/parameters_aws_auth_tests.json.gpg new file mode 100644 index 000000000..bea557c13 Binary files /dev/null and b/.github/workflows/parameters_aws_auth_tests.json.gpg differ diff --git a/.github/workflows/rsa_keys/rsa_encrypted_key.p8.gpg b/.github/workflows/rsa_keys/rsa_encrypted_key.p8.gpg new file mode 100644 index 000000000..3643cdb1f Binary files /dev/null and b/.github/workflows/rsa_keys/rsa_encrypted_key.p8.gpg differ diff --git a/.github/workflows/rsa_keys/rsa_key.p8.gpg b/.github/workflows/rsa_keys/rsa_key.p8.gpg new file mode 100644 index 000000000..e90253cd3 Binary files /dev/null and b/.github/workflows/rsa_keys/rsa_key.p8.gpg differ diff --git a/.github/workflows/rsa_keys/rsa_key_invalid.p8.gpg b/.github/workflows/rsa_keys/rsa_key_invalid.p8.gpg new file mode 100644 index 000000000..3d2442a7c Binary files /dev/null and b/.github/workflows/rsa_keys/rsa_key_invalid.p8.gpg differ diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml index ec96a2e4e..abcb2f980 100644 --- a/.github/workflows/snyk-issue.yml +++ b/.github/workflows/snyk-issue.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout action - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: snowflakedb/whitesource-actions token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index e21815ecf..51f952539 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -16,13 +16,13 @@ jobs: if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 - name: checkout action - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: snowflakedb/whitesource-actions token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} diff --git a/.gitignore b/.gitignore index 77d6b857b..5c1e6c21c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.swp .idea .git -parameters.json +parameters*.json snowflake-sdk-*.tgz dist junit*.xml @@ -14,3 +14,4 @@ wss-*-agent.config wss-unified-agent.jar whitesource/ .nyc_output +rsa_*.p8 diff --git a/Jenkinsfile b/Jenkinsfile index 5fec5151f..fa1342f02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,8 +13,8 @@ timestamps { stage('Build') { withCredentials([ usernamePassword(credentialsId: '063fc85b-62a6-4181-9d72-873b43488411', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY'), - string(credentialsId: 'a791118f-a1ea-46cd-b876-56da1b9bc71c',variable: 'NEXUS_PASSWORD') - ]) { + string(credentialsId: 'a791118f-a1ea-46cd-b876-56da1b9bc71c', variable: 'NEXUS_PASSWORD') + ]) { sh '''\ |#!/bin/bash -e |export GIT_BRANCH=${GIT_BRANCH} @@ -23,18 +23,36 @@ timestamps { '''.stripMargin() } } - params = [ - string(name: 'svn_revision', value: 'main'), - string(name: 'branch', value: 'main'), - string(name: 'client_git_commit', value: scmInfo.GIT_COMMIT), - string(name: 'client_git_branch', value: scmInfo.GIT_BRANCH), - string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'nodejs-chainguard-node18'), - string(name: 'parent_job', value: env.JOB_NAME), - string(name: 'parent_build_number', value: env.BUILD_NUMBER) - ] - stage('Test') { - build job: 'RT-LanguageNodeJS-PC',parameters: params - } + + parallel( + 'Test': { + stage('Test') { + def params = [ + string(name: 'svn_revision', value: 'bptp-built'), + string(name: 'branch', value: 'main'), + string(name: 'client_git_commit', value: scmInfo.GIT_COMMIT), + string(name: 'client_git_branch', value: scmInfo.GIT_BRANCH), + string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'nodejs-chainguard-node18'), + string(name: 'parent_job', value: env.JOB_NAME), + string(name: 'parent_build_number', value: env.BUILD_NUMBER) + ] + build job: 'RT-LanguageNodeJS-PC', parameters: params + } + }, + 'Test Authentication': { + stage('Test Authentication') { + withCredentials([ + string(credentialsId: 'a791118f-a1ea-46cd-b876-56da1b9bc71c', variable: 'NEXUS_PASSWORD'), + string(credentialsId: 'sfctest0-parameters-secret', variable: 'PARAMETERS_SECRET') + ]) { + sh '''\ + |#!/bin/bash -e + |$WORKSPACE/ci/test_authentication.sh + '''.stripMargin() + } + } + } + ) } } @@ -61,7 +79,7 @@ pipeline { } def wgetUpdateGithub(String state, String folder, String targetUrl, String seconds) { - def ghURL = "https://api.github.com/repos/snowflakedb/snowflake-connector-nodejs/statuses/$COMMIT_SHA_LONG" - def data = JsonOutput.toJson([state: "${state}", context: "jenkins/${folder}",target_url: "${targetUrl}"]) - sh "wget ${ghURL} --spider -q --header='Authorization: token $GIT_PASSWORD' --post-data='${data}'" + def ghURL = "https://api.github.com/repos/snowflakedb/snowflake-connector-nodejs/statuses/$COMMIT_SHA_LONG" + def data = JsonOutput.toJson([state: "${state}", context: "jenkins/${folder}", target_url: "${targetUrl}"]) + sh "wget ${ghURL} --spider -q --header='Authorization: token $GIT_PASSWORD' --post-data='${data}'" } diff --git a/README.md b/README.md index 931d1f352..8816af920 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ at No Note ---------------------------------------------------------------------- -This driver currently does not support GCP regional endpoints. Please ensure that any workloads using through this driver do not require support for regional endpoints on GCP. If you have questions about this, please contact Snowflake Support. - +This driver starts supporting the GCS regional endpoint starting from version 2.0.0. Please ensure that any workloads using through this driver +below the version 2.0.0 do not require support for regional endpoints on GCP. If you have questions about this, please contact Snowflake Support. Test ====================================================================== @@ -66,7 +66,7 @@ or npm run test:unit ``` -To run single test file use `test:single` script, e.g. run tests in `test/unit/snowflake_test.js` only: +To run a single test file use `test:single` script, e.g. run tests in `test/unit/snowflake_test.js` only: ``` npm run test:single -- test/unit/snowflake_test.js diff --git a/ci/container/test_authentication.sh b/ci/container/test_authentication.sh new file mode 100755 index 000000000..31e18dde5 --- /dev/null +++ b/ci/container/test_authentication.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +set -o pipefail + +AUTH_PARAMETER_FILE=./.github/workflows/parameters_aws_auth_tests.json +eval $(jq -r '.authtestparams | to_entries | map("export \(.key)=\(.value|tostring)")|.[]' $AUTH_PARAMETER_FILE) + +export SNOWFLAKE_AUTH_TEST_PRIVATE_KEY_PATH=./.github/workflows/rsa_keys/rsa_key.p8 +export SNOWFLAKE_AUTH_TEST_ENCRYPTED_PRIVATE_KEY_PATH=./.github/workflows/rsa_keys/rsa_encrypted_key.p8 +export SNOWFLAKE_AUTH_TEST_INVALID_PRIVATE_KEY_PATH=./.github/workflows/rsa_keys/rsa_key_invalid.p8 + +npm run test:authentication diff --git a/ci/container/test_component.sh b/ci/container/test_component.sh index d52a56b79..2de6082d1 100755 --- a/ci/container/test_component.sh +++ b/ci/container/test_component.sh @@ -30,13 +30,6 @@ npm install PACKAGE_NAME=$(cd $WORKSPACE && ls snowflake-sdk*.tgz) npm install $WORKSPACE/${PACKAGE_NAME} -# Since @azure lib has lost compatibility with node14 -#some dependencies have to be replaced by an older version -nodeVersion=$(node -v) -if [[ "$nodeVersion" == 'v14.'* ]]; then - npm install @azure/core-lro@2.6.0 -fi - echo "[INFO] Setting test parameters" if [[ "$LOCAL_USER_NAME" == "jenkins" ]]; then echo "[INFO] Use the default test parameters.json" @@ -110,7 +103,7 @@ if [[ -z "$GITHUB_ACTIONS" ]]; then fi echo "[INFO] Running Tests: Test result: $WORKSPACE/junit.xml" -if ! ${MOCHA_CMD[@]} "$SOURCE_ROOT/test/**/*.js"; then +if ! ${MOCHA_CMD[@]} 'test/{unit,integration}/**/*.js'; then echo "[ERROR] Test failed" [[ -f "$WORKSPACE/junit.xml" ]] && cat $WORKSPACE/junit.xml exit 1 diff --git a/ci/test_authentication.sh b/ci/test_authentication.sh new file mode 100755 index 000000000..49ab3ae26 --- /dev/null +++ b/ci/test_authentication.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +set -o pipefail +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export WORKSPACE=${WORKSPACE:-/tmp} + +gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output $THIS_DIR/../.github/workflows/parameters_aws_auth_tests.json "$THIS_DIR/../.github/workflows/parameters_aws_auth_tests.json.gpg" +gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output $THIS_DIR/../.github/workflows/rsa_keys/rsa_encrypted_key.p8 "$THIS_DIR/../.github/workflows/rsa_keys/rsa_encrypted_key.p8.gpg" +gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output $THIS_DIR/../.github/workflows/rsa_keys/rsa_key.p8 "$THIS_DIR/../.github/workflows/rsa_keys/rsa_key.p8.gpg" +gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output $THIS_DIR/../.github/workflows/rsa_keys/rsa_key_invalid.p8 "$THIS_DIR/../.github/workflows/rsa_keys/rsa_key_invalid.p8.gpg" + +docker run \ + -v $(cd $THIS_DIR/.. && pwd):/mnt/host \ + -v $WORKSPACE:/mnt/workspace \ + --rm \ + nexus.int.snowflakecomputing.com:8086/docker/snowdrivers-test-external-browser:3 \ + "/mnt/host/ci/container/test_authentication.sh" diff --git a/ci/test_windows.bat b/ci/test_windows.bat index c6764a8ca..bf68e9e87 100644 --- a/ci/test_windows.bat +++ b/ci/test_windows.bat @@ -55,14 +55,6 @@ echo [INFO] Installing Snowflake NodeJS Driver copy %GITHUB_WORKSPACE%\artifacts\* . for %%f in (snowflake-sdk*.tgz) do cmd /c npm install %%f -REM Since @azure lib has lost compatibility with node14 -REM some dependencies have to be replaced by an older version -FOR /F "tokens=* USEBACKQ" %%F IN (`node -v`) DO ( -SET nodeVersion=%%F -) -ECHO %nodeVersion% -if not x%nodeVersion:v14.=%==x%str1% cmd /c npm install @azure/core-lro@2.6.0 - if %ERRORLEVEL% NEQ 0 ( echo [ERROR] failed to install the Snowflake NodeJS Driver exit /b 1 @@ -73,7 +65,7 @@ start /b python hang_webserver.py 12345 > hang_webserver.out 2>&1 popd echo [INFO] Testing -cmd /c node_modules\.bin\mocha --timeout %TIMEOUT% --recursive --full-trace --color --reporter spec test/**/*.js +cmd /c node_modules\.bin\mocha --timeout %TIMEOUT% --recursive --full-trace --color --reporter spec \"test/{unit,integration}/**/*.js\" if %ERRORLEVEL% NEQ 0 ( echo [ERROR] failed to run mocha exit /b 1 diff --git a/index.d.ts b/index.d.ts index e2c56ecb4..f9ee49d5c 100644 --- a/index.d.ts +++ b/index.d.ts @@ -25,7 +25,7 @@ declare module 'snowflake-sdk' { // 403001 ERR_GLOBAL_CONFIGURE_INVALID_LOG_LEVEL = 403001, - ERR_GLOBAL_CONFIGURE_INVALID_INSECURE_CONNECT = 403002, + ERR_GLOBAL_CONFIGURE_INVALID_DISABLE_OCSP_CHECKS = 403002, ERR_GLOBAL_CONFIGURE_INVALID_OCSP_MODE = 403003, ERR_GLOBAL_CONFIGURE_INVALID_JSON_PARSER = 403004, ERR_GLOBAL_CONFIGURE_INVALID_XML_PARSER = 403005, @@ -131,6 +131,7 @@ declare module 'snowflake-sdk' { ERR_CONN_EXEC_STMT_INVALID_FETCH_AS_STRING_VALUES = 409012, ERR_CONN_EXEC_STMT_INVALID_REQUEST_ID = 409013, ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC = 409014, + ERR_CONN_EXEC_STMT_INVALID_DESCRIBE_ONLY = 409015, // 410001 ERR_CONN_FETCH_RESULT_MISSING_OPTIONS = 410001, @@ -218,9 +219,9 @@ declare module 'snowflake-sdk' { additionalLogToConsole?: boolean | null; /** - * Check the ocsp checking is off. + * The option to turn off the OCSP check. */ - insecureConnect?: boolean; + disableOCSPChecks?: boolean; /** * The default value is true. @@ -614,6 +615,14 @@ declare module 'snowflake-sdk' { */ requestId?: string; + /** + * The request GUID is a unique identifier of an HTTP request issued to Snowflake. + * Unlike the requestId, it is regenerated even when the request is resend with the retry mechanism. + * If not specified, request GUIDs are attached to all requests to Snowflake for better traceability. + * In the majority of cases it should not be set or filled with false value. + */ + excludeGuid?: string; + /** * Use different rest endpoints based on whether the query id is available. */ @@ -652,6 +661,11 @@ declare module 'snowflake-sdk' { * that is different from the connector directory. */ cwd?: string; + + /** + * `true` to enable a describe only query. + */ + describeOnly?: boolean; } export interface RowStatement { diff --git a/lib/agent/check.js b/lib/agent/check.js index 0f1bdef83..bbc966e3f 100644 --- a/lib/agent/check.js +++ b/lib/agent/check.js @@ -115,7 +115,8 @@ function getResponse(uri, req, cb) { module.exports = function check(options, cb, mock) { let sync = true; - const maxNumRetries = GlobalConfig.getOcspMode() === GlobalConfig.ocspModes.FAIL_CLOSED ? 5 : 1; + const isFailClosed = GlobalConfig.getOcspMode() === GlobalConfig.ocspModes.FAIL_CLOSED; + const maxNumRetries = isFailClosed ? 2 : 1; function done(err, data) { if (sync) { @@ -190,6 +191,15 @@ module.exports = function check(options, cb, mock) { function ocspRequestCallback(err, uri) { if (err) { + //This error message is from @techteamer/ocsp (ocsp.utils.getAuthorityInfo) + if (err.message === 'AuthorityInfoAccess not found in extensions') { + if (!isFailClosed) { + Logger.getInstance().debug('OCSP Responder URL is missing from the certificate.'); + return done(null); + } else { + Logger.getInstance().error('OCSP Responder URL is missing from the certificate, so cannot verify with OCSP. Aborting connection attempt due to OCSP being set to FAIL_CLOSE https://docs.snowflake.com/en/user-guide/ocsp#fail-close'); + } + } return done(err); } diff --git a/lib/agent/socket_util.js b/lib/agent/socket_util.js index 215f42ff6..9f99ec182 100644 --- a/lib/agent/socket_util.js +++ b/lib/agent/socket_util.js @@ -13,11 +13,6 @@ const ErrorCodes = Errors.codes; const REGEX_SNOWFLAKE_ENDPOINT = /.snowflakecomputing./; -const ocspFailOpenWarning = - 'WARNING!!! using fail-open to connect. Driver is connecting to an HTTPS endpoint ' + - 'without OCSP based Certificated Revocation checking as it could not obtain a valid OCSP Response to use from ' + - 'the CA OCSP responder. Details: '; - const socketSecuredEvent = 'secureConnect'; const rawOcspFlag = @@ -120,7 +115,7 @@ exports.secureSocket = function (socket, host, agent, mock) { function isOcspValidationDisabled(host) { // ocsp is disabled if insecure-connect is enabled, or if we've disabled ocsp // for non-snowflake endpoints and the host is a non-snowflake endpoint - return GlobalConfig.isInsecureConnect() || + return GlobalConfig.isOCSPChecksDisabled() || (Parameters.getValue(Parameters.names.JS_DRIVER_DISABLE_OCSP_FOR_NON_SF_ENDPOINTS) && !REGEX_SNOWFLAKE_ENDPOINT.test(host)); } @@ -158,7 +153,7 @@ function canEarlyExitForOCSP(errors) { const err = errors[errorIndex]; if (err && !isValidOCSPError(err)) { // any of the errors is NOT good/revoked/unknown - Logger.getInstance().warn(ocspFailOpenWarning + err); + Logger.getInstance().debug(`OCSP responder didn't respond correctly. Assuming certificate is not revoked. Details: ${err}`); return null; } else if (err && err.code === ErrorCodes.ERR_OCSP_REVOKED) { anyRevoked = err; diff --git a/lib/configuration/client_configuration.js b/lib/configuration/client_configuration.js index 370232008..49694f7d5 100644 --- a/lib/configuration/client_configuration.js +++ b/lib/configuration/client_configuration.js @@ -259,7 +259,7 @@ function ConfigurationUtil(fsPromisesModule, processModule) { } async function searchForConfigInDefaultDirectories() { - Logger.getInstance().debug(`Searching for config in default directories: ${defaultDirectories}`); + Logger.getInstance().debug(`Searching for config in default directories: ${JSON.stringify(defaultDirectories)}`); for (const directory of defaultDirectories) { const configPath = await searchForConfigInDictionary(directory.dir, directory.dirDescription); if (exists(configPath)) { diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 88f4d61dd..56c11af1b 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -7,6 +7,7 @@ const Url = require('url'); const QueryString = require('querystring'); const QueryStatus = require('../constants/query_status'); +const LoggingUtil = require('../logger/logging_util'); const Util = require('../util'); const Errors = require('../errors'); const ErrorCodes = Errors.codes; @@ -20,6 +21,7 @@ const { isOktaAuth } = require('../authentication/authentication'); const { init: initEasyLogging } = require('../logger/easy_logging_starter'); const GlobalConfig = require('../global_config'); const JsonCredentialManager = require('../authentication/secure_storage/json_credential_manager'); +const ExecutionTimer = require('../logger/execution_timer'); /** * Creates a new Connection instance. @@ -30,6 +32,7 @@ const JsonCredentialManager = require('../authentication/secure_storage/json_cre */ function Connection(context) { // validate input + Logger.getInstance().trace('Connection object is being constructed'); Errors.assertInternal(Util.isObject(context)); const services = context.getServices(); @@ -37,6 +40,37 @@ function Connection(context) { // generate an id for the connection const id = uuidv4(); + Logger.getInstance().trace('Generated connection id: %s', id); + + Logger.getInstance().info( + 'Creating Connection[id: %s] with %s, password is %s, region: %s, ' + + 'authenticator: %s, ocsp mode: %s, os: %s, os version: %s', + id, + connectionConfig.describeIdentityAttributes(), + LoggingUtil.describePresence(connectionConfig.password), + connectionConfig.region, connectionConfig.getAuthenticator(), + connectionConfig.getClientEnvironment().OCSP_MODE, + connectionConfig.getClientEnvironment().OS, + connectionConfig.getClientEnvironment().OS_VERSION); + + // Log was split due to possibility of exceeding the max message length of the logger + Logger.getInstance().info( + 'Connection[id: %s] additional details: ' + + 'passcode in password is %s, passcode is %s, private key is %s, ' + + 'application: %s, client name: %s, client version: %s, retry timeout: %s, ' + + 'private key path: %s, private key pass is %s, ' + + 'client store temporary credential: %s, browser response timeout: %s', + id, + LoggingUtil.describePresence(connectionConfig.getPasscodeInPassword()), + LoggingUtil.describePresence(connectionConfig.getPasscode()), + LoggingUtil.describePresence(connectionConfig.getPrivateKey()), + connectionConfig.getClientApplication(), connectionConfig.getClientName(), + connectionConfig.getClientVersion(), connectionConfig.getRetryTimeout(), + connectionConfig.getPrivateKeyPath(), + LoggingUtil.describePresence(connectionConfig.getPrivateKeyPass()), + connectionConfig.getClientStoreTemporaryCredential(), + connectionConfig.getBrowserActionTimeout()); + // async max retry and retry pattern from python connector const asyncNoDataMaxRetry = 24; @@ -60,7 +94,9 @@ function Connection(context) { * @returns {boolean} */ this.isUp = function () { - return services.sf.isConnected(); + const isUp = services.sf.isConnected(); + Logger.getInstance().trace('Connection[id: %s] - isUp called. Returning: %s', this.getId(), isUp); + return isUp; }; /** @@ -69,15 +105,21 @@ function Connection(context) { * @returns {boolean} */ this.isTokenValid = function () { + Logger.getInstance().trace('Connection[id: %s] - isTokenValid called', this.getId()); const tokenInfo = services.sf.getConfig().tokenInfo; const sessionTokenExpirationTime = tokenInfo.sessionTokenExpirationTime; const isSessionValid = sessionTokenExpirationTime > Date.now(); + Logger.getInstance().trace('Connection[id: %s] - isSessionTokenValid: %s', this.getId(), isSessionValid); + const masterTokenExpirationTime = tokenInfo.masterTokenExpirationTime; const isMasterValid = masterTokenExpirationTime > Date.now(); + Logger.getInstance().trace('Connection[id: %s] - isMasterTokenValid: %s', this.getId(), isMasterValid); - return (isSessionValid && isMasterValid); + const areTokensValid = (isSessionValid && isMasterValid); + Logger.getInstance().trace('Connection[id: %s] - isTokenValid returned: %s', this.getId(), areTokensValid); + return areTokensValid; }; this.getServiceName = function () { @@ -106,8 +148,8 @@ function Connection(context) { }; this.heartbeat = callback => { - Logger.getInstance().debug('Issuing heartbeat call'); - const requestID = uuidv4(); + Logger.getInstance().trace('Issuing heartbeat call'); + const requestId = uuidv4(); services.sf.request( { @@ -117,14 +159,14 @@ function Connection(context) { pathname: '/session/heartbeat', search: QueryString.stringify( { - requestId: requestID + requestId: requestId }) }), callback: Util.isFunction(callback) ? callback : function (err, body) { if (err) { Logger.getInstance().error('Error issuing heartbeat call: %s', err.message); } else { - Logger.getInstance().debug('Heartbeat response %s', JSON.stringify(body)); + Logger.getInstance().trace('Heartbeat response %s', JSON.stringify(body)); } } } @@ -134,6 +176,7 @@ function Connection(context) { this.heartbeatAsync = () => { return new Promise((resolve, reject) => { // previous version of driver called `select 1;` which result in `[ { '1': 1 } ]` + Logger.getInstance().trace('Issuing async heartbeat call'); this.heartbeat((err) => err ? reject(err) : resolve([{ '1': 1 }])); }); }; @@ -142,6 +185,7 @@ function Connection(context) { * @return {Promise} */ this.isValidAsync = async () => { + Logger.getInstance().trace('Connection[id: %s] - isValidAsync called', this.getId()); if (!this.isUp()) { return false; } @@ -149,7 +193,7 @@ function Connection(context) { await this.heartbeatAsync(); return true; } catch (e) { - Logger.getInstance().trace('Connection heartbeat failed: %s', JSON.stringify(e, Util.getCircularReplacer())); + Logger.getInstance().debug('Connection[id: %s] - heartbeat failed: %s', this.getId(), JSON.stringify(e, Object.getOwnPropertyNames(e))); return false; } }; @@ -179,6 +223,7 @@ function Connection(context) { const SECONDS_TO_MILLISECONDS_MULTIPLIER = 1000; const KEEP_ALIVE_HEARTBEAT_FREQUENCY_IN_MS = Parameters.getValue(Parameters.names.CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY) * SECONDS_TO_MILLISECONDS_MULTIPLIER; self.keepalive = setInterval(self.heartbeat, KEEP_ALIVE_HEARTBEAT_FREQUENCY_IN_MS, self); + Logger.getInstance().trace('Connection[id: %s] - keepAlive internal created', id); } if (Util.isFunction(callback)) { callback(Errors.externalize(err), self); @@ -186,7 +231,7 @@ function Connection(context) { }; } - this.determineConnectionDomain = () => connectionConfig.accessUrl && connectionConfig.accessUrl.includes('snowflakecomputing.cn') ? 'CHINA' : 'GLOBAL'; + this.determineConnectionDomain = () => connectionConfig.accessUrl && connectionConfig.accessUrl.includes('snowflakecomputing.cn') ? 'CHINA' : 'GLOBAL'; /** * Establishes a connection if we aren't in a fatal state. @@ -196,14 +241,16 @@ function Connection(context) { * @returns {Object} the connection object. */ this.connect = function (callback) { + const timer = new ExecutionTimer().start(); const connectionDomain = this.determineConnectionDomain(); - Logger.getInstance().info(`Connecting to ${connectionDomain} Snowflake domain`); + Logger.getInstance().info('Connection[id: %s] - connecting. Associated Snowflake domain: %s', this.getId(), connectionDomain); // invalid callback Errors.checkArgumentValid( !Util.exists(callback) || Util.isFunction(callback), ErrorCodes.ERR_CONN_CONNECT_INVALID_CALLBACK); if (Util.exists(connectionConfig.host) && Util.isPrivateLink(connectionConfig.host)) { + Logger.getInstance().info('Connection[id: %s] - setting up private link', this.getId()); this.setupOcspPrivateLink(connectionConfig.host); } @@ -213,22 +260,30 @@ function Connection(context) { const self = this; const authenticationType = connectionConfig.getAuthenticator(); + Logger.getInstance().debug('Connection[id: %s] - using authentication type: %s', this.getId(), authenticationType); // check if authentication type is compatible with connect() // external browser and okta are not compatible with connect() due to their usage of async functions if (authenticationType === AuthenticationTypes.EXTERNAL_BROWSER_AUTHENTICATOR || isOktaAuth(authenticationType)) { + const connectingDuration = timer.getDuration(); + Logger.getInstance().error('Connection[id: %s] - connecting failed after %s milliseconds.' + + 'Error: External browser and Okta are not compatible with connection process', this.getId(), connectingDuration + ); throw Errors.createClientError( ErrorCodes.ERR_CONN_CREATE_INVALID_AUTH_CONNECT); } // Get authenticator to use + Logger.getInstance().debug('Connection[id: %s] - retrieving authenticator', this.getId()); const auth = services.sf.getAuthenticator(); + Logger.getInstance().debug('Connection[id: %s] - trying to authenticate', this.getId()); auth.authenticate(connectionConfig.getAuthenticator(), connectionConfig.getServiceName(), connectionConfig.account, connectionConfig.username).then(() => { + Logger.getInstance().info('Connection[id: %s] - authentication successful using: %s', this.getId(), connectionConfig.getAuthenticator()); // JSON for connection const body = Authenticator.formAuthJSON(connectionConfig.getAuthenticator(), connectionConfig.account, @@ -240,23 +295,43 @@ function Connection(context) { // Update JSON body with the authentication values auth.updateBody(body); + Logger.getInstance().debug('Connection[id: %s] - initializing easyLogging', this.getId()); initEasyLogging(connectionConfig.clientConfigFile) .then(() => { + Logger.getInstance().debug('Connection[id: %s] - easyLogging initialized', this.getId()); try { + Logger.getInstance().debug('Connection[id: %s] - connecting through service', this.getId()); services.sf.connect({ callback: connectCallback(self, callback), json: body }); + const connectingDuration = timer.getDuration(); + Logger.getInstance().info( + 'Connection[id: %s] - connected successfully after %s milliseconds', + this.getId(), connectingDuration + ); return this; } catch (e) { // we don't expect an error here since callback method should be called - Logger.getInstance().error('Unexpected error from calling callback function', e); + const connectingDuration = timer.getDuration(); + Logger.getInstance().info('Connection[id: %s] - failed to connect after %s milliseconds. ' + + 'Error: Unexpected error from calling connectCallback function in snowflake service - %s', this.getId(), connectingDuration, e); } }, - () => callback(Errors.createClientError(ErrorCodes.ERR_CONN_CONNECT_INVALID_CLIENT_CONFIG, true))); + () => { + const connectingDuration = timer.getDuration(); + Logger.getInstance().error('Connection[id: %s] - failed to initialize easyLogging. ' + + 'Connecting failed after %s milliseconds', this.getId(), connectingDuration); + callback(Errors.createClientError(ErrorCodes.ERR_CONN_CONNECT_INVALID_CLIENT_CONFIG, true), self); + }); }, - (err) => callback(err)); + (err) => { + const connectingDuration = timer.getDuration(); + Logger.getInstance().error('Connection[id: %s] - authentication failed. Error: %s. ' + + 'Connecting failed after %s milliseconds', this.getId(), err, connectingDuration); + callback(err, self); + }); return this; }; @@ -270,8 +345,9 @@ function Connection(context) { * @returns {Object} the connection object. */ this.connectAsync = async function (callback) { + const timer = new ExecutionTimer().start(); const connectingDomain = this.determineConnectionDomain(); - Logger.getInstance().info(`Connecting to ${connectingDomain} Snowflake domain`); + Logger.getInstance().info('Connection[id: %s] - async connecting. Associated Snowflake domain: %s', this.getId(), connectingDomain); // invalid callback Errors.checkArgumentValid( @@ -280,6 +356,7 @@ function Connection(context) { if (Util.isPrivateLink(connectionConfig.host)) { this.setupOcspPrivateLink(connectionConfig.host); + Logger.getInstance().info('Connection[id: %s] - setting up private link', this.getId()); } // connect to the snowflake service and provide our own callback so that @@ -287,43 +364,60 @@ function Connection(context) { // callback const self = this; - + if (connectionConfig.getClientStoreTemporaryCredential()) { - const key = Util.buildCredentialCacheKey(connectionConfig.host, + Logger.getInstance().debug('Connection[id: %s] - storing temporary credential of client', this.getId()); + const key = Util.buildCredentialCacheKey(connectionConfig.host, connectionConfig.username, AuthenticationTypes.ID_TOKEN_AUTHENTICATOR); if (GlobalConfig.getCredentialManager() === null) { + Logger.getInstance().debug('Connection[id: %s] - using default json credential manager', this.getId()); GlobalConfig.setCustomCredentialManager(new JsonCredentialManager(connectionConfig.getCredentialCacheDir())); } + Logger.getInstance().debug('Connection[id: %s] - reading idToken using credential manager', this.getId()); connectionConfig.idToken = await GlobalConfig.getCredentialManager().read(key); } if (connectionConfig.getClientRequestMFAToken()) { + Logger.getInstance().debug('Connection[id: %s] - extracting mfaToken of client', this.getId()); const key = Util.buildCredentialCacheKey(connectionConfig.host, connectionConfig.username, AuthenticationTypes.USER_PWD_MFA_AUTHENTICATOR); if (GlobalConfig.getCredentialManager() === null) { + Logger.getInstance().debug('Connection[id: %s] - using default json credential manager', this.getId()); GlobalConfig.setCustomCredentialManager(new JsonCredentialManager(connectionConfig.getCredentialCacheDir())); } + Logger.getInstance().debug('Connection[id: %s] - reading mfaToken using credential manager', this.getId()); connectionConfig.mfaToken = await GlobalConfig.getCredentialManager().read(key); } // Get authenticator to use + Logger.getInstance().debug('Connection[id: %s] - retrieving authenticator', this.getId()); const auth = Authenticator.getAuthenticator(connectionConfig, context.getHttpClient()); services.sf.authenticator = auth; try { + Logger.getInstance().debug('Connection[id: %s] - initializing easyLogging', this.getId()); await initEasyLogging(connectionConfig.clientConfigFile); } catch (err) { + const connectingDuration = timer.getDuration(); + Logger.getInstance().error('Connection[id: %s] - failed to initialize easyLogging. ' + + 'Connecting failed after %s milliseconds', this.getId(), connectingDuration); throw Errors.createClientError(ErrorCodes.ERR_CONN_CONNECT_INVALID_CLIENT_CONFIG, true); } + let body = null; try { + Logger.getInstance().debug('Connection[id: %s] - using authentication type: %s', this.getId(), connectionConfig.getAuthenticator()); + + Logger.getInstance().debug('Connection[id: %s] - trying to authenticate', this.getId()); await auth.authenticate(connectionConfig.getAuthenticator(), connectionConfig.getServiceName(), connectionConfig.account, connectionConfig.username); - + + Logger.getInstance().info('Connection[id: %s] - authentication successful using: %s', this.getId(), connectionConfig.getAuthenticator()); + // JSON for connection - const body = Authenticator.formAuthJSON(connectionConfig.getAuthenticator(), + body = Authenticator.formAuthJSON(connectionConfig.getAuthenticator(), connectionConfig.account, connectionConfig.username, connectionConfig.getClientType(), @@ -332,17 +426,35 @@ function Connection(context) { // Update JSON body with the authentication values auth.updateBody(body); + } catch (authErr) { + const connectingDuration = timer.getDuration(); + Logger.getInstance().info('Connection[id: %s] - failed to connect async after %s milliseconds.' + + 'Failed during authentication. Error: %s', this.getId(), connectingDuration, authErr); + Logger.getInstance().error('Connection[id: %s] - failed during authentication. Error: %s', this.getId(), authErr); + callback(authErr); + return this; + } + + try { // Request connection + Logger.getInstance().debug('Connection[id: %s] - connecting through service', this.getId()); services.sf.connect({ callback: connectCallback(self, callback), json: body, }); - } catch (authErr) { - callback(authErr); + // return the connection to facilitate chaining + const connectingDuration = timer.getDuration(); + Logger.getInstance().info('Connection[id: %s] - connected successfully after %s milliseconds', this.getId(), connectingDuration); + + } catch (callbackErr) { + const connectingDuration = timer.getDuration(); + Logger.getInstance().info('Connection[id: %s] - failed to connect async after %s milliseconds.' + + 'Error: Unexpected error from calling connectCallback function in snowflake service - %s', this.getId(), connectingDuration, callbackErr); + callback(callbackErr); + return this; } - - // return the connection to facilitate chaining + return this; }; @@ -354,6 +466,7 @@ function Connection(context) { * @returns {Object} */ this.execute = function (options) { + Logger.getInstance().trace('Connection[id: %s] - execute called with options.', this.getId()); return Statement.createStatementPreExec( options, services, connectionConfig); }; @@ -366,6 +479,7 @@ function Connection(context) { * @returns {Object} */ this.fetchResult = function (options) { + Logger.getInstance().trace('Connection[id: %s] - fetchResult called with options', this.getId()); return Statement.createStatementPostExec( options, services, connectionConfig); }; @@ -380,6 +494,7 @@ function Connection(context) { */ this.destroy = function (callback) { // invalid callback + Logger.getInstance().trace('Connection[id: %s] - destroy called', this.getId()); Errors.checkArgumentValid( !Util.exists(callback) || Util.isFunction(callback), ErrorCodes.ERR_CONN_DESTROY_INVALID_CALLBACK); @@ -387,12 +502,14 @@ function Connection(context) { // log out of the snowflake service and provide our own callback so that // the connection can be passed in when invoking the connection.destroy() // callback + Logger.getInstance().trace('Connection[id: %s] - destroying through service', this.getId()); const self = this; services.sf.destroy( { callback: function (err) { if (Util.exists(self.keepalive)) { clearInterval(self.keepalive); + Logger.getInstance().trace('Connection[id: %s] - keepAlive interval cleared', self.getId()); } if (Util.isFunction(callback)) { @@ -401,6 +518,7 @@ function Connection(context) { } }); + Logger.getInstance().trace('Connection[id: %s] - connection destroyed successfully', this.getId()); // return the connection to facilitate chaining return this; }; @@ -413,11 +531,13 @@ function Connection(context) { * @returns {Object} the query response */ async function getQueryResponse(queryId) { + Logger.getInstance().trace('Connection[id: %s] - requested query response for Query[id: %s]', id, queryId); // Check if queryId exists and is valid uuid Errors.checkArgumentExists(Util.exists(queryId), ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID); Errors.checkArgumentValid(queryIdRegex.test(queryId), ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID, queryId); + Logger.getInstance().debug('Connection[id: %s] - Query[id: %s] is valid', id, queryId); // Form the request options const options = @@ -429,8 +549,12 @@ function Connection(context) { }), }; + Logger.getInstance().debug('Connection[id: %s] - fetching query response for Query[id: %s]', id, queryId); + const timer = new ExecutionTimer().start(); // Get the response containing the query status const response = await services.sf.requestAsync(options); + const fetchingDuration = timer.getDuration(); + Logger.getInstance().debug('Connection[id: %s] - query response for Query[id: %s] fetched successfully after: %s milliseconds', id, queryId, fetchingDuration); return response['data']; } @@ -449,6 +573,7 @@ function Connection(context) { status = queries[0]['status']; } + Logger.getInstance().trace('Connection[id: %s] - Extracted query status: %s', status); return status; } @@ -460,6 +585,7 @@ function Connection(context) { * @returns {String} the query status. */ this.getQueryStatus = async function (queryId) { + Logger.getInstance().trace('Connection[id: %s] - getQueryStatus called for Query[id: %s]', this.getId(), queryId); return extractQueryStatus(await getQueryResponse(queryId)); }; @@ -471,6 +597,7 @@ function Connection(context) { * @returns {String} the query status. */ this.getQueryStatusThrowIfError = async function (queryId) { + Logger.getInstance().trace('Connection[id: %s] - getQueryStatusThrowIfError called for Query[id: %s]', this.getId(), queryId); const response = await getQueryResponse(queryId); const status = extractQueryStatus(response); let sqlState = null; @@ -484,6 +611,7 @@ function Connection(context) { sqlState = response['data']['sqlState']; } + Logger.getInstance().debug('Connection[id: %s] - query error for Query[id: %s]. Error: %s. SQLState: %s', this.getId(), queryId, message, sqlState); throw Errors.createOperationFailedError( code, response, message, sqlState); } @@ -500,16 +628,22 @@ function Connection(context) { */ this.getResultsFromQueryId = async function (options) { const queryId = options.queryId; + Logger.getInstance().trace('Connection[id: %s] - getResultsFromQueryId called for Query[id: %s].', this.getId(), queryId); + let status, noDataCounter = 0, retryPatternPos = 0; // Wait until query has finished executing let queryStillExecuting = true; while (queryStillExecuting) { + Logger.getInstance().trace('Connection[id: %s] - checking if Query[id: %s] is still executing. Retries with no data count: %d', this.getId(), queryId, noDataCounter); // Check if query is still running. // Trigger exception if it failed or there is no query data in the server. status = await this.getQueryStatusThrowIfError(queryId); queryStillExecuting = this.isStillRunning(status); if (!queryStillExecuting || status === QueryStatus.code.NO_QUERY_DATA) { + Logger.getInstance().trace('Connection[id: %s] - end of waiting for Query[id: %s] to finish executing. ' + + queryStillExecuting ? 'Query is no longer executing. ' : '' + + 'Query status: %s.', this.getId(), queryId, status); break; } @@ -521,8 +655,10 @@ function Connection(context) { // If no data, increment the no data counter if (QueryStatus.code[status] === QueryStatus.code.NO_DATA) { noDataCounter++; + Logger.getInstance().trace('Connection[id: %s] - no data returned for Query[id: %s]. Retries with no data count: %d', this.getId(), queryId, noDataCounter); // Check if retry for no data is exceeded if (noDataCounter > asyncNoDataMaxRetry) { + Logger.getInstance().error('Connection[id: %s] - no data returned for Query[id: %s]. Retry limit: %s reached.', this.getId(), queryId, asyncNoDataMaxRetry); throw Errors.createClientError( ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NO_DATA, true, queryId); } @@ -534,15 +670,18 @@ function Connection(context) { } if (QueryStatus.code[status] === QueryStatus.code.NO_QUERY_DATA) { + Logger.getInstance().error('Connection[id: %s] - Query[id: %s] did not succeed. Final status: %s', this.getId(), queryId, status); throw Errors.createClientError( ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NO_DATA, true, queryId, status); } if (QueryStatus.code[status] !== QueryStatus.code.SUCCESS) { + Logger.getInstance().error('Connection[id: %s] - Query[id: %s] did not succeed. Final status: %s', this.getId(), queryId, status); throw Errors.createClientError( ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NOT_SUCCESS_STATUS, true, queryId, status); } + Logger.getInstance().debug('Connection[id: %s] - Query[id: %s] succeeded. Fetching the result.', this.getId(), queryId); return this.fetchResult(options); }; @@ -554,6 +693,7 @@ function Connection(context) { * @returns {Boolean} */ this.isStillRunning = function (status) { + Logger.getInstance().trace('Connection[id: %s] - checking if status %s is still running', this.getId(), status); return QueryStatus.runningStatuses.includes(QueryStatus.code[status]); }; @@ -574,6 +714,7 @@ function Connection(context) { * @returns {String} */ this.serialize = function () { + Logger.getInstance().trace('Connection[id: %s] - serialize called', this.getId()); return JSON.stringify(context.getConfig()); }; diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 2bbec6519..55e8ef426 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -5,6 +5,7 @@ const os = require('os'); const url = require('url'); const Util = require('../util'); +const ProxyUtil = require('../proxy_util'); const Errors = require('../errors'); const ConnectionConstants = require('../constants/connection_constants'); const path = require('path'); @@ -16,6 +17,7 @@ const levenshtein = require('fastest-levenshtein'); const RowMode = require('./../constants/row_mode'); const DataTypes = require('./result/data_types'); const Logger = require('../logger'); +const LoggingUtil = require('../logger/logging_util'); const WAIT_FOR_BROWSER_ACTION_TIMEOUT = 120000; const DEFAULT_PARAMS = [ @@ -222,7 +224,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) { protocol: proxyProtocol, noProxy: noProxy }; - Util.validateProxy(proxy); + ProxyUtil.validateProxy(proxy); } const serviceName = options.serviceName; @@ -835,6 +837,33 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) { return passcode; }; + /** + * Returns attributes of Connection Config object that can be used to identify + * the connection, when ID is not available in the scope. This is not sufficient set, + * since multiple connections can be instantiated for the same config, but can be treated as a hint. + * + * @returns {string} + */ + this.describeIdentityAttributes = function () { + return `host: ${this.host}, account: ${this.account}, accessUrl: ${this.accessUrl}, ` + + `user: ${this.username}, role: ${this.getRole()}, database: ${this.getDatabase()}, ` + + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + this.describeProxy(); + }; + + /** + * @returns {string} + */ + this.describeProxy = function () { + const proxy = this.getProxy(); + if (Util.exists(proxy)) { + return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, proxyUser: ${proxy.user}, ` + + `proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, ` + + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`; + } else { + return 'proxy was not configured'; + } + }; + // save config options this.username = options.username; this.password = options.password; diff --git a/lib/connection/connection_context.js b/lib/connection/connection_context.js index c064aa48e..3c583929c 100644 --- a/lib/connection/connection_context.js +++ b/lib/connection/connection_context.js @@ -6,6 +6,7 @@ const Util = require('../util'); const Errors = require('../errors'); const SfService = require('../services/sf'); const LargeResultSetService = require('../services/large_result_set'); +const Logger = require('../logger'); /** * Creates a new ConnectionContext. @@ -18,6 +19,7 @@ const LargeResultSetService = require('../services/large_result_set'); */ function ConnectionContext(connectionConfig, httpClient, config) { // validate input + Logger.getInstance().trace('Creating ConnectionContext object.'); Errors.assertInternal(Util.isObject(connectionConfig)); Errors.assertInternal(Util.isObject(httpClient)); @@ -25,12 +27,15 @@ function ConnectionContext(connectionConfig, httpClient, config) { // that it has all the information we need let sfServiceConfig; if (Util.exists(config)) { + Logger.getInstance().trace('ConnectionContext - validating received config.'); + Errors.assertInternal(Util.isObject(config)); Errors.assertInternal(Util.isObject(config.services)); Errors.assertInternal(Util.isObject(config.services.sf)); sfServiceConfig = config.services.sf; } + Logger.getInstance().debug('ConnectionContext - received data was validated.'); // create a map that contains all the services we'll be using const services = @@ -38,6 +43,7 @@ function ConnectionContext(connectionConfig, httpClient, config) { sf: new SfService(connectionConfig, httpClient, sfServiceConfig), largeResultSet: new LargeResultSetService(connectionConfig, httpClient) }; + Logger.getInstance().debug('ConnectionContext - services were instantiated.'); /** * Returns the ConnectionConfig for use by the connection. diff --git a/lib/connection/statement.js b/lib/connection/statement.js index fef7b2cbb..b67f81357 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -353,6 +353,12 @@ function createContextPreExec( ErrorCodes.ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC); } + // if a describeOnly flag is specified, make sure it's boolean + if (Util.exists(statementOptions.describeOnly)) { + Errors.checkArgumentValid(Util.isBoolean(statementOptions.describeOnly), + ErrorCodes.ERR_CONN_EXEC_STMT_INVALID_DESCRIBE_ONLY); + } + // create a statement context const statementContext = createStatementContext(); @@ -384,6 +390,11 @@ function createContextPreExec( statementContext.cwd = statementOptions.cwd; } + // if the describeOnly flag is specified, add it to the statement context + if (Util.exists(statementOptions.describeOnly)) { + statementContext.describeOnly = statementOptions.describeOnly; + } + // validate non-user-specified arguments Errors.assertInternal(Util.isObject(services)); Errors.assertInternal(Util.isObject(connectionConfig)); @@ -1244,6 +1255,11 @@ function sendRequestPreExec(statementContext, onResultAvailable) { json.asyncExec = statementContext.asyncExec; } + // include describeOnly flag if a value was specified + if (Util.exists(statementContext.describeOnly)) { + json.describeOnly = statementContext.describeOnly; + } + // use the snowflake service to issue the request sendSfRequest(statementContext, { diff --git a/lib/constants/error_messages.js b/lib/constants/error_messages.js index 4a1c34145..b1bfff5e0 100644 --- a/lib/constants/error_messages.js +++ b/lib/constants/error_messages.js @@ -18,7 +18,7 @@ exports[402002] = 'Request to S3/Blob failed.'; // 403001 exports[403001] = 'Invalid logLevel. The specified value must be one of these five levels: error, warn, debug, info and trace.'; -exports[403002] = 'Invalid insecureConnect option. The specified value must be a boolean.'; +exports[403002] = 'Invalid disableOCSPChecks option. The specified value must be a boolean.'; exports[403003] = 'Invalid OCSP mode. The specified value must be FAIL_CLOSED, FAIL_OPEN, or INSECURE_MODE.'; exports[403004] = 'Invalid custom JSON parser. The specified value must be a function.'; exports[403005] = 'Invalid custom XML parser. The specified value must be a function.'; @@ -126,6 +126,7 @@ exports[409011] = 'Invalid fetchAsString value. The specified value must be an A exports[409012] = 'Invalid fetchAsString type: %s. The supported types are: String, Boolean, Number, Date, Buffer, and JSON.'; exports[409013] = 'Invalid requestId. The specified value must be a string.'; exports[409014] = 'Invalid asyncExec. The specified value must be a boolean.'; +exports[409015] = 'Invalid describeOnly. The specified value must be a boolean.'; // 410001 exports[410001] = 'Fetch-result options must be specified.'; diff --git a/lib/constants/sf_params.js b/lib/constants/sf_params.js new file mode 100644 index 000000000..3f2b9747f --- /dev/null +++ b/lib/constants/sf_params.js @@ -0,0 +1,8 @@ +exports.paramsNames = Object.freeze({ + SF_REQUEST_GUID: 'request_guid', + SF_REQUEST_ID: 'requestId', + SF_TOKEN: 'token', + SF_WAREHOUSE_NAME: 'warehouse', + SF_DB_NAME: 'databaseName', + SF_SCHEMA_NAME: 'schemaName', +}); diff --git a/lib/core.js b/lib/core.js index f8e398899..5f8b02a6f 100644 --- a/lib/core.js +++ b/lib/core.js @@ -32,15 +32,18 @@ function Core(options) { // set the logger instance Logger.setInstance(new (options.loggerClass)()); + Logger.getInstance().trace('Logger was initialized.'); // if a connection class is specified, it must be an object or function let connectionClass = options.connectionClass; if (Util.exists(connectionClass)) { Errors.assertInternal( Util.isObject(connectionClass) || Util.isFunction(connectionClass)); + Logger.getInstance().debug('Connection class provided in driver core options will be used.'); } else { // fall back to Connection connectionClass = Connection; + Logger.getInstance().debug('Connection class was not overridden. Default connection class will be used.'); } const qaMode = options.qaMode; @@ -64,27 +67,38 @@ function Core(options) { // Alternatively, if the connectionOptions includes token information then we will use that // instead of the username/password + Logger.getInstance().info('Creating new connection object'); + if (connectionOptions == null) { + Logger.getInstance().info('Connection options were not specified. Loading connection configuration.'); try { connectionOptions = loadConnectionConfiguration(); } catch ( error ) { - Logger.getInstance().debug(`Problem during reading connection configuration from file: ${error.message}`); + Logger.getInstance().error('Unable to load the connection configuration. Error: %s', error.message); Errors.checkArgumentExists(Util.exists(connectionOptions), ErrorCodes.ERR_CONN_CREATE_MISSING_OPTIONS); } } const validateCredentials = !config && (connectionOptions && !connectionOptions.sessionToken); + const connectionConfig = new ConnectionConfig(connectionOptions, validateCredentials, qaMode, clientInfo); + Logger.getInstance().debug('Connection configuration object created'); // if an http client was specified in the options passed to the module, use // it, otherwise create a new HttpClient const httpClient = options.httpClient || new options.httpClientClass(connectionConfig); + Logger.getInstance().debug('HttpClient setup finished'); + - return new connectionClass( - new ConnectionContext(connectionConfig, httpClient, config)); + const connection = new connectionClass( + new ConnectionContext(connectionConfig, httpClient, config) + ); + + Logger.getInstance().info('Connection[id: %s] - connection object created successfully.', connection.getId()); + return connection; }; const instance = @@ -124,6 +138,8 @@ function Core(options) { */ deserializeConnection: function (options, serializedConnection) { // check for missing serializedConfig + Logger.getInstance().trace('Deserializing connection'); + Errors.checkArgumentExists(Util.exists(serializedConnection), ErrorCodes.ERR_CONN_DESERIALIZE_MISSING_CONFIG); @@ -131,6 +147,8 @@ function Core(options) { Errors.checkArgumentValid(Util.isString(serializedConnection), ErrorCodes.ERR_CONN_DESERIALIZE_INVALID_CONFIG_TYPE); + Logger.getInstance().debug('Deserializing connection from string object'); + // try to json-parse serializedConfig let config; try { @@ -140,6 +158,7 @@ function Core(options) { Errors.checkArgumentValid(Util.isObject(config), ErrorCodes.ERR_CONN_DESERIALIZE_INVALID_CONFIG_FORM); } + Logger.getInstance().debug('Connection deserialized successfully'); return createConnection(options, config); }, @@ -152,6 +171,7 @@ function Core(options) { * @returns {String} a serialized version of the connection. */ serializeConnection: function (connection) { + Logger.getInstance().trace('Connection[id: %s] - serializing connection.', connection.getId()); return connection ? connection.serialize() : connection; }, @@ -161,26 +181,29 @@ function Core(options) { * @param {Object} options */ configure: function (options) { + Logger.getInstance().debug('Configuring Snowflake core module.'); const logLevel = extractLogLevel(options); const logFilePath = options.logFilePath; const additionalLogToConsole = options.additionalLogToConsole; + if (logLevel != null || logFilePath) { - Logger.getInstance().debug(`Configuring logger with level: ${logLevel}, filePath: ${logFilePath}, additionalLogToConsole: ${additionalLogToConsole}`); Logger.getInstance().configure( { level: logLevel, filePath: logFilePath, additionalLogToConsole: additionalLogToConsole }); + Logger.getInstance().info('Configuring logger with level: %s, filePath: %s, additionalLogToConsole: %s', logLevel, logFilePath, additionalLogToConsole); } - const insecureConnect = options.insecureConnect; - if (Util.exists(insecureConnect)) { + const disableOCSPChecks = options.disableOCSPChecks; + if (Util.exists(disableOCSPChecks)) { // check that the specified value is a boolean - Errors.checkArgumentValid(Util.isBoolean(insecureConnect), - ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_INSECURE_CONNECT); + Errors.checkArgumentValid(Util.isBoolean(disableOCSPChecks), + ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_DISABLE_OCSP_CHECKS); - GlobalConfig.setInsecureConnect(insecureConnect); + GlobalConfig.setDisableOCSPChecks(disableOCSPChecks); + Logger.getInstance().debug('Setting disableOCSPChecks to value from core options: %s', disableOCSPChecks); } const ocspFailOpen = options.ocspFailOpen; @@ -189,6 +212,7 @@ function Core(options) { ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_OCSP_MODE); GlobalConfig.setOcspFailOpen(ocspFailOpen); + Logger.getInstance().debug('Setting ocspFailOpen to value from core options: %s ', ocspFailOpen); } const jsonColumnVariantParser = options.jsonColumnVariantParser; @@ -197,6 +221,7 @@ function Core(options) { ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_JSON_PARSER); GlobalConfig.setJsonColumnVariantParser(jsonColumnVariantParser); + Logger.getInstance().debug('Setting JSON Column Variant Parser to value from core options'); } const xmlColumnVariantParser = options.xmlColumnVariantParser; @@ -206,8 +231,10 @@ function Core(options) { ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_XML_PARSER); GlobalConfig.setXmlColumnVariantParser(xmlColumnVariantParser); + Logger.getInstance().debug('Setting XML Column Variant Parser to value from core options'); } else if (Util.exists(xmlParserConfig)) { GlobalConfig.createXmlColumnVariantParserWithParameters(xmlParserConfig); + Logger.getInstance().debug('Creating XML Column Variant Parser with parameters from core options'); } const keepAlive = options.keepAlive; @@ -216,6 +243,7 @@ function Core(options) { ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_KEEP_ALIVE); GlobalConfig.setKeepAlive(keepAlive); + Logger.getInstance().debug('Setting keepAlive to value from core options: %s', keepAlive); } const useEnvProxy = options.useEnvProxy; @@ -232,7 +260,8 @@ function Core(options) { ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_CUSTOM_CREDENTIAL_MANAGER); GlobalConfig.setCustomCredentialManager(customCredentialManager); - } + Logger.getInstance().debug('Setting customCredentialManager to value from core options %s', customCredentialManager); + } } }; @@ -275,15 +304,17 @@ function Core(options) { * @returns {Object} */ this.create = function () { + Logger.getInstance().debug('Creating new connection from factory.'); const connection = new createConnection(connectionOptions); return new Promise((resolve, reject) => { connection.connect( function (err, conn) { if (err) { - Logger.getInstance().error('Unable to connect: ' + err.message); + Logger.getInstance().error('Connection[id: %s] - Unable to connect. Error: %s', conn.getId(), err.message); reject(new Error(err.message)); } else { + Logger.getInstance().debug('Connection[id: %s] - connected successfully. Callback called.', conn.getId()); resolve(conn); } } @@ -299,10 +330,13 @@ function Core(options) { * @returns {Object} */ this.destroy = function (connection) { + Logger.getInstance().debug('Destroying connection instance.'); return new Promise((resolve) => { - connection.destroy(function (err) { + connection.destroy(function (err, conn) { if (err) { - Logger.getInstance().error('Unable to disconnect: ' + err.message); + Logger.getInstance().error('Connection[id: %s] - disconnecting failed with error: %s', conn.getId(), err.message); + } else { + Logger.getInstance().debug('Connection[id: %s] - connection disconnected successfully. Callback called.', conn.getId()); } resolve(); }); @@ -317,6 +351,7 @@ function Core(options) { * @returns {Boolean} */ this.validate = async function (connection) { + Logger.getInstance().debug('Connection[id: %s] - validating connection instance', connection.getId()); return await connection.isValidAsync(); }; } @@ -330,19 +365,25 @@ function Core(options) { * @returns {Object} */ const createPool = function createPool(connectionOptions, poolOptions) { + Logger.getInstance().info('Creating connection pool with provided options'); + const connectionPool = GenericPool.createPool( new ConnectionFactory(connectionOptions), poolOptions ); + Logger.getInstance().debug('Base for connection pool created'); // avoid infinite loop if factory creation fails connectionPool.on('factoryCreateError', function (err) { + Logger.getInstance().error('Connection pool factory creation failed: %s', err.message); const clientResourceRequest = connectionPool._waitingClientsQueue.dequeue(); if (clientResourceRequest) { clientResourceRequest.reject(err); } }); + Logger.getInstance().info('Connection pool object created successfully'); + return connectionPool; }; diff --git a/lib/errors.js b/lib/errors.js index b57e4dc39..9a31d81fd 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -23,7 +23,7 @@ codes.ERR_LARGE_RESULT_SET_RESPONSE_FAILURE = 402002; // 403001 codes.ERR_GLOBAL_CONFIGURE_INVALID_LOG_LEVEL = 403001; -codes.ERR_GLOBAL_CONFIGURE_INVALID_INSECURE_CONNECT = 403002; +codes.ERR_GLOBAL_CONFIGURE_INVALID_DISABLE_OCSP_CHECKS = 403002; codes.ERR_GLOBAL_CONFIGURE_INVALID_OCSP_MODE = 403003; codes.ERR_GLOBAL_CONFIGURE_INVALID_JSON_PARSER = 403004; codes.ERR_GLOBAL_CONFIGURE_INVALID_XML_PARSER = 403005; @@ -130,6 +130,7 @@ codes.ERR_CONN_EXEC_STMT_INVALID_FETCH_AS_STRING = 409011; codes.ERR_CONN_EXEC_STMT_INVALID_FETCH_AS_STRING_VALUES = 409012; codes.ERR_CONN_EXEC_STMT_INVALID_REQUEST_ID = 409013; codes.ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC = 409014; +codes.ERR_CONN_EXEC_STMT_INVALID_DESCRIBE_ONLY = 409015; // 410001 codes.ERR_CONN_FETCH_RESULT_MISSING_OPTIONS = 410001; @@ -341,7 +342,7 @@ exports.createOperationFailedError = function ( return createError(errorTypes.OperationFailedError, { code: errorCode, - data: { ...data, queryId: null }, + data: data, message: message, sqlState: sqlState }); diff --git a/lib/file_transfer_agent/azure_util.js b/lib/file_transfer_agent/azure_util.js index 496b0aaee..005c403bd 100644 --- a/lib/file_transfer_agent/azure_util.js +++ b/lib/file_transfer_agent/azure_util.js @@ -6,6 +6,9 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata; const FileHeader = require('./file_util').FileHeader; const expandTilde = require('expand-tilde'); const resultStatus = require('./file_util').resultStatus; +const ProxyUtil = require('../proxy_util'); +const { isBypassProxy } = require('../http/node'); +const Logger = require('../logger'); const EXPIRED_TOKEN = 'ExpiredToken'; @@ -26,7 +29,7 @@ function AzureLocation(containerName, path) { * @returns {Object} * @constructor */ -function AzureUtil(azure, filestream) { +function AzureUtil(connectionConfig, azure, filestream) { const AZURE = typeof azure !== 'undefined' ? azure : require('@azure/storage-blob'); const fs = typeof filestream !== 'undefined' ? filestream : require('fs'); @@ -42,11 +45,23 @@ function AzureUtil(azure, filestream) { const sasToken = stageCredentials['AZURE_SAS_TOKEN']; const account = stageInfo['storageAccount']; - + const connectionString = `https://${account}.blob.core.windows.net${sasToken}`; + let proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'Azure Util'); + if (proxy && !isBypassProxy(proxy, connectionString)) { + Logger.getInstance().debug(`The destination host is: ${ProxyUtil.getHostFromURL(connectionString)} and the proxy host is: ${proxy.host}`); + Logger.getInstance().trace(`Initializing the proxy information for the Azure Client: ${ProxyUtil.describeProxy(proxy)}`); + + proxy = ProxyUtil.getAzureProxy(proxy); + Logger.getInstance().trace(connectionConfig.describe); + } + ProxyUtil.hideEnvironmentProxy(); const blobServiceClient = new AZURE.BlobServiceClient( - `https://${account}.blob.core.windows.net${sasToken}` + connectionString, null, + { + proxyOptions: proxy, + } ); - + ProxyUtil.restoreEnvironmentProxy(); return blobServiceClient; }; @@ -203,7 +218,7 @@ function AzureUtil(azure, filestream) { blobContentEncoding: 'UTF-8', blobContentType: 'application/octet-stream' } - }); + }); } catch (err) { if (err['statusCode'] === 403 && detectAzureTokenExpireError(err)) { meta['lastError'] = err; @@ -215,7 +230,6 @@ function AzureUtil(azure, filestream) { } return; } - meta['dstFileSize'] = meta['uploadSize']; meta['resultStatus'] = resultStatus.UPLOADED; }; @@ -262,7 +276,6 @@ function AzureUtil(azure, filestream) { } return; } - meta['resultStatus'] = resultStatus.DOWNLOADED; }; @@ -282,5 +295,4 @@ function AzureUtil(azure, filestream) { errstr.includes('Server failed to authenticate the request.'); } } - module.exports = AzureUtil; diff --git a/lib/file_transfer_agent/encrypt_util.js b/lib/file_transfer_agent/encrypt_util.js index 3f48b6f17..835634b43 100644 --- a/lib/file_transfer_agent/encrypt_util.js +++ b/lib/file_transfer_agent/encrypt_util.js @@ -12,6 +12,28 @@ const blockSize = parseInt(AES_BLOCK_SIZE / 8); // in bytes const QUERY_STAGE_MASTER_KEY = 'queryStageMasterKey'; const BASE64 = 'base64'; +const DEFAULT_AAD = Buffer.from(''); +const AUTH_TAG_LENGTH_IN_BYTES = 16; + +const AES_CBC = { + cipherName: function (keySizeInBytes) { + return `aes-${keySizeInBytes * 8}-cbc`; + }, + ivSize: 16 +}; + +const AES_ECB = { + cipherName: function (keySizeInBytes) { + return `aes-${keySizeInBytes * 8}-ecb`; + } +}; + +const AES_GCM = { + cipherName: function (keySizeInBytes) { + return `aes-${keySizeInBytes * 8}-gcm`; + }, + ivSize: 12 +}; // Material Descriptor function MaterialDescriptor(smkId, queryId, keySize) { @@ -23,22 +45,17 @@ function MaterialDescriptor(smkId, queryId, keySize) { } // Encryption Material -function EncryptionMetadata(key, iv, matDesc) { +function EncryptionMetadata(key, dataIv, matDesc, keyIv, dataAad, keyAad) { return { 'key': key, - 'iv': iv, - 'matDesc': matDesc + 'iv': dataIv, + 'matDesc': matDesc, + 'keyIv': keyIv, + 'dataAad': dataAad, + 'keyAad': keyAad }; } -function aesCbc(keySizeInBytes) { - return `aes-${keySizeInBytes * 8}-cbc`; -} - -function aesEcb(keySizeInBytes) { - return `aes-${keySizeInBytes * 8}-ecb`; -} - exports.EncryptionMetadata = EncryptionMetadata; function TempFileGenerator() { @@ -97,6 +114,7 @@ function TempFileGenerator() { */ function EncryptUtil(encrypt, filestream, temp) { const crypto = typeof encrypt !== 'undefined' ? encrypt : require('crypto'); + // TODO: SNOW-1814883: Replace 'fs' with 'fs/promises' const fs = typeof filestream !== 'undefined' ? filestream : require('fs'); const tmp = typeof temp !== 'undefined' ? temp : new TempFileGenerator(); @@ -125,206 +143,261 @@ function EncryptUtil(encrypt, filestream, temp) { return newMatDesc; } + function createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv, keyIv = null, dataAad = null, keyAad = null) { + const matDesc = new MaterialDescriptor( + encryptionMaterial.smkId, + encryptionMaterial.queryId, + keySize * 8 + ); + + return new EncryptionMetadata( + encryptedKey.toString(BASE64), + dataIv.toString(BASE64), + matDescToUnicode(matDesc), + keyIv ? keyIv.toString(BASE64) : null, + dataAad ? dataAad.toString(BASE64) : null, + keyAad ? keyAad.toString(BASE64) : null + ); + } + /** - * Encrypt file stream using AES algorithm. - * - * @param {Object} encryptionMaterial - * @param {String} fileStream - * @param {String} tmpDir - * @param {Number} chunkSize - * - * @returns {Object} - */ - this.encryptFileStream = async function (encryptionMaterial, fileStream) { - // Get decoded key from base64 encoded value - const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - const keySize = decodedKey.length; + * Encrypt content using AES-CBC algorithm. + */ + this.encryptFileStream = async function (encryptionMaterial, content) { + return this.encryptDataCBC(encryptionMaterial, content); + }; + + this.encryptDataCBC = function (encryptionMaterial, data) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keySize = decodedKek.length; - // Get secure random bytes with block size - const ivData = getSecureRandom(blockSize); + const dataIv = getSecureRandom(AES_CBC.ivSize); const fileKey = getSecureRandom(keySize); - // Create cipher with file key, AES CBC, and iv data - let cipher = crypto.createCipheriv(aesCbc(keySize), fileKey, ivData); - const encrypted = cipher.update(fileStream); - const final = cipher.final(); - const encryptedData = Buffer.concat([encrypted, final]); + const dataCipher = crypto.createCipheriv(AES_CBC.cipherName(keySize), fileKey, dataIv); + const encryptedData = performCrypto(dataCipher, data); - // Create key cipher with decoded key and AES ECB - cipher = crypto.createCipheriv(aesEcb(keySize), decodedKey, null); + const keyCipher = crypto.createCipheriv(AES_ECB.cipherName(keySize), decodedKek, null); + const encryptedKey = performCrypto(keyCipher, fileKey); - // Encrypt with file key - const encKek = Buffer.concat([ - cipher.update(fileKey), - cipher.final() - ]); + return { + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv), + dataStream: encryptedData + }; + }; - const matDesc = MaterialDescriptor( - encryptionMaterial.smkId, - encryptionMaterial.queryId, - keySize * 8 - ); + //TODO: SNOW-940981: Add proper usage when feature is ready + this.encryptDataGCM = function (encryptionMaterial, data) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keySize = decodedKek.length; - const metadata = EncryptionMetadata( - encKek.toString(BASE64), - ivData.toString(BASE64), - matDescToUnicode(matDesc) - ); + const dataIv = getSecureRandom(AES_GCM.ivSize); + const fileKey = getSecureRandom(keySize); + + const encryptedData = this.encryptGCM(data, fileKey, dataIv, DEFAULT_AAD); + + const keyIv = getSecureRandom(AES_GCM.ivSize); + const encryptedKey = this.encryptGCM(fileKey, decodedKek, keyIv, DEFAULT_AAD); return { - encryptionMetadata: metadata, + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv, keyIv, DEFAULT_AAD, DEFAULT_AAD), dataStream: encryptedData }; }; + + this.encryptGCM = function (data, key, iv, aad) { + const cipher = crypto.createCipheriv(AES_GCM.cipherName(key.length), key, iv, { authTagLength: AUTH_TAG_LENGTH_IN_BYTES }); + if (aad) { + cipher.setAAD(aad); + } + const encryptedData = performCrypto(cipher, data); + return Buffer.concat([encryptedData, cipher.getAuthTag()]); + }; + + this.decryptGCM = function (data, key, iv, aad) { + const decipher = crypto.createDecipheriv(AES_GCM.cipherName(key.length), key, iv, { authTagLength: AUTH_TAG_LENGTH_IN_BYTES }); + if (aad) { + decipher.setAAD(aad); + } + // last 16 bytes of data is the authentication tag + const authTag = data.slice(data.length - AUTH_TAG_LENGTH_IN_BYTES, data.length); + const cipherText = data.slice(0, data.length - AUTH_TAG_LENGTH_IN_BYTES); + decipher.setAuthTag(authTag); + return performCrypto(decipher, cipherText); + }; + /** - * Encrypt file using AES algorithm. - * - * @param {Object} encryptionMaterial - * @param {String} inFileName - * @param {String} tmpDir - * @param {Number} chunkSize - * - * @returns {Object} - */ - this.encryptFile = async function (encryptionMaterial, inFileName, + * Encrypt file using AES algorithm. + */ + this.encryptFile = async function (encryptionMaterial, inputFilePath, tmpDir = null, chunkSize = blockSize * 4 * 1024) { - // Get decoded key from base64 encoded value - const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - const keySize = decodedKey.length; + return await this.encryptFileCBC(encryptionMaterial, inputFilePath, tmpDir, chunkSize); + }; - // Get secure random bytes with block size - const ivData = getSecureRandom(blockSize); - const fileKey = getSecureRandom(keySize); + this.encryptFileCBC = async function (encryptionMaterial, inputFilePath, + tmpDir = null, chunkSize = blockSize * 4 * 1024) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keySize = decodedKek.length; - // Create cipher with file key, AES CBC, and iv data - let cipher = crypto.createCipheriv(aesCbc(keySize), fileKey, ivData); + const dataIv = getSecureRandom(AES_CBC.ivSize); + const fileKey = getSecureRandom(keySize); + const dataCipher = crypto.createCipheriv(AES_CBC.cipherName(keySize), fileKey, dataIv); + const encryptedFilePath = await performFileStreamCrypto(dataCipher, tmpDir, inputFilePath, chunkSize); - // Create temp file - const tmpobj = tmp.fileSync({ dir: tmpDir, prefix: path.basename(inFileName) + '#' }); - const tempOutputFileName = tmpobj.name; - const tempFd = tmpobj.fd; + const keyCipher = crypto.createCipheriv(AES_ECB.cipherName(keySize), decodedKek, null); + const encryptedKey = performCrypto(keyCipher, fileKey); - await new Promise(function (resolve) { - const infile = fs.createReadStream(inFileName, { highWaterMark: chunkSize }); - const outfile = fs.createWriteStream(tempOutputFileName); - - infile.on('data', function (chunk) { - // Encrypt chunk using cipher - const encrypted = cipher.update(chunk); - // Write to temp file - outfile.write(encrypted); - }); - infile.on('close', function () { - outfile.write(cipher.final()); - outfile.close(resolve); - }); - }); + return { + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv), + dataFile: encryptedFilePath + }; + }; - // Create key cipher with decoded key and AES ECB - cipher = crypto.createCipheriv(aesEcb(keySize), decodedKey, null); + //TODO: SNOW-940981: Add proper usage when feature is ready + this.encryptFileGCM = async function (encryptionMaterial, inputFilePath, tmpDir = null) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - // Encrypt with file key - const encKek = Buffer.concat([ - cipher.update(fileKey), - cipher.final() - ]); + const dataIv = getSecureRandom(AES_GCM.ivSize); + const fileKey = getSecureRandom(decodedKek.length); - const matDesc = MaterialDescriptor( - encryptionMaterial.smkId, - encryptionMaterial.queryId, - keySize * 8 - ); + const fileContent = await new Promise((resolve, reject) => { + fs.readFile(inputFilePath, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); - const metadata = EncryptionMetadata( - encKek.toString(BASE64), - ivData.toString(BASE64), - matDescToUnicode(matDesc) - ); + const encryptedData = this.encryptGCM(fileContent, fileKey, dataIv, DEFAULT_AAD); + const encryptedFilePath = await writeContentToFile(tmpDir, path.basename(inputFilePath) + '#', encryptedData); - // Close temp file - fs.closeSync(tempFd); + const keyIv = getSecureRandom(AES_GCM.ivSize); + const encryptedKey = this.encryptGCM(fileKey, decodedKek, keyIv, DEFAULT_AAD); return { - encryptionMetadata: metadata, - dataFile: tempOutputFileName + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, fileKey.length, encryptedKey, dataIv, keyIv, DEFAULT_AAD, DEFAULT_AAD), + dataFile: encryptedFilePath }; }; /** - * Decrypt file using AES algorithm. - * - * @param {Object} encryptionMaterial - * @param {String} inFileName - * @param {String} tmpDir - * @param {Number} chunkSize - * - * @returns {String} - */ - this.decryptFile = async function (metadata, encryptionMaterial, inFileName, + * Decrypt file using AES algorithm. + */ + this.decryptFile = async function (metadata, encryptionMaterial, inputFilePath, tmpDir = null, chunkSize = blockSize * 4 * 1024) { - // Get key and iv from metadata - const keyBase64 = metadata.key; - const ivBase64 = metadata.iv; + return await this.decryptFileCBC(metadata, encryptionMaterial, inputFilePath, tmpDir, chunkSize); + }; - // Get decoded key from base64 encoded value - const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - const keySize = decodedKey.length; + this.decryptFileCBC = async function (metadata, encryptionMaterial, inputFilePath, + tmpDir = null, chunkSize = blockSize * 4 * 1024) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keyBytes = new Buffer.from(metadata.key, BASE64); + const ivBytes = new Buffer.from(metadata.iv, BASE64); + const keyDecipher = crypto.createDecipheriv(AES_ECB.cipherName(decodedKek.length), decodedKek, null); + const fileKey = performCrypto(keyDecipher, keyBytes); + + const dataDecipher = crypto.createDecipheriv(AES_CBC.cipherName(fileKey.length), fileKey, ivBytes); + return await performFileStreamCrypto(dataDecipher, tmpDir, inputFilePath, chunkSize); + }; - // Get key bytes and iv bytes from base64 encoded value - const keyBytes = new Buffer.from(keyBase64, BASE64); - const ivBytes = new Buffer.from(ivBase64, BASE64); + //TODO: SNOW-940981: Add proper usage when feature is ready + this.decryptFileGCM = async function (metadata, encryptionMaterial, inputFilePath, tmpDir = null) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keyBytes = new Buffer.from(metadata.key, BASE64); + const keyIvBytes = new Buffer.from(metadata.keyIv, BASE64); + const dataIvBytes = new Buffer.from(metadata.iv, BASE64); + const dataAadBytes = new Buffer.from(metadata.dataAad, BASE64); + const keyAadBytes = new Buffer.from(metadata.keyAad, BASE64); - // Create temp file - let tempOutputFileName; - let tempFd; - await new Promise((resolve, reject) => { - tmp.file({ dir: tmpDir, prefix: path.basename(inFileName) + '#' }, (err, path, fd) => { + const fileKey = this.decryptGCM(keyBytes, decodedKek, keyIvBytes, keyAadBytes); + + const fileContent = await new Promise((resolve, reject) => { + fs.readFile(inputFilePath, (err, data) => { if (err) { reject(err); + } else { + resolve(data); } - tempOutputFileName = path; - tempFd = fd; - resolve(); }); }); - // Create key decipher with decoded key and AES ECB - let decipher = crypto.createDecipheriv(aesEcb(keySize), decodedKey, null); - const fileKey = Buffer.concat([ - decipher.update(keyBytes), - decipher.final() - ]); - - // Create decipher with file key, iv bytes, and AES CBC - decipher = crypto.createDecipheriv(aesCbc(keySize), fileKey, ivBytes); + const decryptedData = this.decryptGCM(fileContent, fileKey, dataIvBytes, dataAadBytes); + return await writeContentToFile(tmpDir, path.basename(inputFilePath) + '#', decryptedData); + }; + + function performCrypto(cipherOrDecipher, data) { + const encryptedOrDecrypted = cipherOrDecipher.update(data); + const final = cipherOrDecipher.final(); + return Buffer.concat([encryptedOrDecrypted, final]); + } + async function performFileStreamCrypto(cipherOrDecipher, tmpDir, inputFilePath, chunkSize) { + const outputFile = await new Promise((resolve, reject) => { + tmp.file({ dir: tmpDir, prefix: path.basename(inputFilePath) + '#' }, (err, path, fd) => { + if (err) { + reject(err); + } else { + resolve({ path, fd }); + } + }); + }); await new Promise(function (resolve) { - const infile = fs.createReadStream(inFileName, { highWaterMark: chunkSize }); - const outfile = fs.createWriteStream(tempOutputFileName); - - infile.on('data', function (chunk) { - // Dncrypt chunk using decipher - const decrypted = decipher.update(chunk); - // Write to temp file - outfile.write(decrypted); + const inputStream = fs.createReadStream(inputFilePath, { highWaterMark: chunkSize }); + const outputStream = fs.createWriteStream(outputFile.path); + + inputStream.on('data', function (chunk) { + const encrypted = cipherOrDecipher.update(chunk); + outputStream.write(encrypted); }); - infile.on('close', function () { - outfile.write(decipher.final()); - outfile.close(resolve); + inputStream.on('close', function () { + outputStream.write(cipherOrDecipher.final()); + outputStream.close(resolve); }); }); - // Close temp file await new Promise((resolve, reject) => { - fs.close(tempFd, (err) => { + fs.close(outputFile.fd, (err) => { if (err) { - reject(err); + reject(err); + } else { + resolve(); } - resolve(); }); }); + return outputFile.path; + } - return tempOutputFileName; - }; + async function writeContentToFile(tmpDir, prefix, content,) { + const outputFile = await new Promise((resolve, reject) => { + tmp.file({ dir: tmpDir, prefix: prefix }, (err, path, fd) => { + if (err) { + reject(err); + } else { + resolve({ path, fd }); + } + }); + }); + await new Promise((resolve, reject) => { + fs.writeFile(outputFile.path, content, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + await new Promise((resolve, reject) => { + fs.close(outputFile.fd, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + return outputFile.path; + } } exports.EncryptUtil = EncryptUtil; diff --git a/lib/file_transfer_agent/gcs_util.js b/lib/file_transfer_agent/gcs_util.js index 2a9cf251d..fc46557ac 100644 --- a/lib/file_transfer_agent/gcs_util.js +++ b/lib/file_transfer_agent/gcs_util.js @@ -69,8 +69,15 @@ function GCSUtil(httpclient, filestream) { } }); - const storage = new Storage({ interceptors_: interceptors }); - + //TODO: SNOW-1789759 hardcoded region will be replaced in the future + const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl; + let endPoint = null; + if (stageInfo['endPoint']) { + endPoint = stageInfo['endPoint']; + } else if (isRegionalUrlEnabled) { + endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`; + } + const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors }); client = { gcsToken: gcsToken, gcsClient: storage }; } else { client = null; diff --git a/lib/file_transfer_agent/remote_storage_util.js b/lib/file_transfer_agent/remote_storage_util.js index 0657788a1..ffc599ec8 100644 --- a/lib/file_transfer_agent/remote_storage_util.js +++ b/lib/file_transfer_agent/remote_storage_util.js @@ -44,7 +44,7 @@ function RemoteStorageUtil(connectionConfig) { if (type === 'S3') { return new SnowflakeS3Util(connectionConfig); } else if (type === 'AZURE') { - return new SnowflakeAzureUtil(); + return new SnowflakeAzureUtil(connectionConfig); } else if (type === 'GCS') { return new SnowflakeGCSUtil(); } else { diff --git a/lib/file_transfer_agent/s3_util.js b/lib/file_transfer_agent/s3_util.js index 9608b4c23..21ee56789 100644 --- a/lib/file_transfer_agent/s3_util.js +++ b/lib/file_transfer_agent/s3_util.js @@ -2,14 +2,12 @@ * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. */ -const { NodeHttpHandler } = require('@aws-sdk/node-http-handler'); +const { NodeHttpHandler } = require('@smithy/node-http-handler'); const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata; const FileHeader = require('./file_util').FileHeader; const expandTilde = require('expand-tilde'); const getProxyAgent = require('../http/node').getProxyAgent; -const Util = require('../util'); -const Logger = require('../logger'); -const GlobalConfig = require('../global_config'); +const ProxyUtil = require('../proxy_util'); const AMZ_IV = 'x-amz-iv'; const AMZ_KEY = 'x-amz-key'; @@ -54,12 +52,19 @@ function S3Util(connectionConfig, s3, filestream) { this.createClient = function (stageInfo, useAccelerateEndpoint) { const stageCredentials = stageInfo['creds']; const securityToken = stageCredentials['AWS_TOKEN']; + const isRegionalUrlEnabled = stageInfo.useRegionalUrl || stageInfo.useS3RegionalUrl; + // if GS sends us an endpoint, it's likely for FIPS. Use it. let endPoint = null; if (stageInfo['endPoint']) { - endPoint = 'https://' + stageInfo['endPoint']; + endPoint = `https://${stageInfo['endPoint']}`; + } else { + if (stageInfo.region && isRegionalUrlEnabled) { + const domainSuffixForRegionalUrl = (stageInfo.region).toLowerCase().startsWith('cn-') ? 'amazonaws.com.cn' : 'amazonaws.com'; + endPoint = `https://s3.${stageInfo.region}.${domainSuffixForRegionalUrl}`; + } } - + const config = { apiVersion: '2006-03-01', region: stageInfo['region'], @@ -72,15 +77,9 @@ function S3Util(connectionConfig, s3, filestream) { useAccelerateEndpoint: useAccelerateEndpoint }; - let proxy = connectionConfig.getProxy(); - if (!proxy && GlobalConfig.isEnvProxyActive()) { - proxy = Util.getProxyFromEnv(); - if (proxy) { - Logger.getInstance().debug(`S3 Util loads the proxy info from the environment variable host: ${proxy.host}`); - } - } + const proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'S3 Util'); if (proxy) { - const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), SNOWFLAKE_S3_DESTINATION); + const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || SNOWFLAKE_S3_DESTINATION); config.requestHandler = new NodeHttpHandler({ httpAgent: proxyAgent, httpsAgent: proxyAgent diff --git a/lib/global_config.js b/lib/global_config.js index 23cdd1c24..e47e30543 100644 --- a/lib/global_config.js +++ b/lib/global_config.js @@ -11,27 +11,27 @@ const Util = require('./util'); const Logger = require('./logger'); const { XMLParser, XMLValidator } = require('fast-xml-parser'); -let insecureConnect = false; +let disableOCSPChecks = false; /** - * Updates the value of the 'insecureConnect' parameter. + * Updates the value of the 'disableOCSPChecks' parameter. * * @param {boolean} value */ -exports.setInsecureConnect = function (value) { +exports.setDisableOCSPChecks = function (value) { // validate input Errors.assertInternal(Util.isBoolean(value)); - insecureConnect = value; + disableOCSPChecks = value; }; /** - * Returns the value of the 'insecureConnect' parameter. + * Returns the value of the 'disableOCSPChecks' parameter. * * @returns {boolean} */ -exports.isInsecureConnect = function () { - return insecureConnect; +exports.isOCSPChecksDisabled = function () { + return disableOCSPChecks; }; let ocspFailOpen = true; @@ -71,7 +71,7 @@ exports.ocspModes = ocspModes; * @returns {string} */ exports.getOcspMode = function () { - if (insecureConnect) { + if (disableOCSPChecks) { return ocspModes.INSECURE; } else if (!ocspFailOpen) { return ocspModes.FAIL_CLOSED; diff --git a/lib/http/base.js b/lib/http/base.js index e2d3b8445..a365f6e25 100644 --- a/lib/http/base.js +++ b/lib/http/base.js @@ -5,8 +5,10 @@ const zlib = require('zlib'); const Util = require('../util'); const Logger = require('../logger'); +const ExecutionTimer = require('../logger/execution_timer'); const axios = require('axios'); const URL = require('node:url').URL; +const requestUtil = require('./request_util'); const DEFAULT_REQUEST_TIMEOUT = 360000; @@ -18,6 +20,8 @@ const DEFAULT_REQUEST_TIMEOUT = 360000; */ function HttpClient(connectionConfig) { // save the connection config + Logger.getInstance().trace('Initializing base HttpClient with Connection Config[%s]', + connectionConfig.describeIdentityAttributes()); this._connectionConfig = connectionConfig; } @@ -29,42 +33,56 @@ function HttpClient(connectionConfig) { * @returns {Object} an object representing the request that was issued. */ HttpClient.prototype.request = function (options) { - let request; + Logger.getInstance().trace('Request%s - preparing for sending.', requestUtil.describeRequestFromOptions(options)); + + let requestPromise; const requestOptions = prepareRequestOptions.call(this, options); let sendRequest = async function sendRequest() { - request = axios.request(requestOptions).then(response => { + Logger.getInstance().trace('Request%s - sending.', requestUtil.describeRequestFromOptions(requestOptions)); + const timer = new ExecutionTimer().start(); + requestPromise = axios.request(requestOptions).then(response => { + const httpResponseTime = timer.getDuration(); + Logger.getInstance().debug('Request%s - response received after %s milliseconds with status %s.', requestUtil.describeRequestFromOptions(requestOptions), httpResponseTime, response.status); sanitizeAxiosResponse(response); if (Util.isFunction(options.callback)) { + Logger.getInstance().trace('Request%s - calling callback function.', requestUtil.describeRequestFromOptions(requestOptions)); return options.callback(null, normalizeResponse(response), response.data); } else { - Logger.getInstance().trace(`Callback function was not provided for the call to ${options.url}`); + Logger.getInstance().trace('Request%s - callback function was not provided.', requestUtil.describeRequestFromOptions(requestOptions)); return null; } }).catch(err => { + const httpResponseTime = timer.getDuration(); + Logger.getInstance().debug('Request%s - failed after %s milliseconds.', requestUtil.describeRequestFromOptions(requestOptions), httpResponseTime); sanitizeAxiosError(err); if (Util.isFunction(options.callback)) { if (err.response) { // axios returns error for not 2xx responses - let's unwrap it + Logger.getInstance().trace('Request%s - calling callback function for error from response. Received code: ', requestUtil.describeRequestFromOptions(requestOptions), err.response.status); options.callback(null, normalizeResponse(err.response), err.response.data); } else { + Logger.getInstance().trace('Request%s - calling callback function for error without response.', requestUtil.describeRequestFromOptions(requestOptions)); options.callback(err, normalizeResponse(null), null); } return null; } else { + Logger.getInstance().warn('Request%s - callback function was not provided. Error will be re-raised.', requestUtil.describeRequestFromOptions(requestOptions)); throw err; } }); }; sendRequest = sendRequest.bind(this); - Logger.getInstance().trace(`CALL ${requestOptions.method} with timeout ${requestOptions.timeout}: ${requestOptions.url}`); + Logger.getInstance().trace('Request%s - issued for the next tick.', requestUtil.describeRequestFromOptions(requestOptions)); process.nextTick(sendRequest); // return an externalized request object that only contains // methods we're comfortable exposing to the outside world return { abort: function () { - if (request) { - request.abort(); + if (requestPromise) { + Logger.getInstance().trace('Request%s - aborting.', requestUtil.describeRequestFromOptions(requestOptions)); + // TODO: This line won't work - promise has no method called abort + requestPromise.abort(); } } }; @@ -78,21 +96,38 @@ HttpClient.prototype.request = function (options) { * @returns {Object} an object representing the request that was issued. */ HttpClient.prototype.requestAsync = async function (options) { + Logger.getInstance().trace('Request%s - preparing for async sending.', requestUtil.describeRequestFromOptions(options)); + const timer = new ExecutionTimer(); try { const requestOptions = prepareRequestOptions.call(this, options); + + timer.start(); const response = await axios.request(requestOptions); - if (Util.isString(response['data']) && - response['headers']['content-type'] === 'application/json') { - response['data'] = JSON.parse(response['data']); - } + const httpResponseTime = timer.getDuration(); + Logger.getInstance().debug('Request%s - response received after %s milliseconds with status %s.', requestUtil.describeRequestFromOptions(requestOptions), httpResponseTime, response.status); + parseResponseData(response); sanitizeAxiosResponse(response); return response; } catch (err) { + const httpResponseTime = timer.getDuration(); + Logger.getInstance().debug('Request%s - failed after %s milliseconds. Error will be re-raised.', requestUtil.describeRequestFromOptions(options), httpResponseTime); sanitizeAxiosError(err); throw err; } }; +function parseResponseData(response) { + Logger.getInstance().trace('Request%s - parsing response data.', requestUtil.describeRequestFromResponse(response)); + parseIfJSONData(response); +} + +function parseIfJSONData(response) { + if (Util.isString(response['data']) && + response['headers']['content-type'] === 'application/json') { + response['data'] = JSON.parse(response['data']); + } +} + /** * Issues an HTTP POST request. * @@ -184,6 +219,7 @@ HttpClient.prototype.getAgent = function () { module.exports = HttpClient; function sanitizeAxiosResponse(response) { + Logger.getInstance().trace('Request%s - sanitizing response data.', requestUtil.describeRequestFromResponse(response)); response.request = undefined; if (response.config) { response.config.data = undefined; @@ -195,11 +231,13 @@ function sanitizeAxiosError(error) { error.request = undefined; error.config = undefined; if (error.response) { + Logger.getInstance().trace('Request%s - sanitizing response error data.', requestUtil.describeRequestFromResponse(error.response)); sanitizeAxiosResponse(error.response); } } function prepareRequestOptions(options) { + Logger.getInstance().trace('Request%s - constructing options.', requestUtil.describeRequestFromOptions(options)); const headers = normalizeHeaders(options.headers) || {}; const timeout = options.timeout || @@ -215,8 +253,11 @@ function prepareRequestOptions(options) { if (!err) { data = bufferCompressed; headers['Content-Encoding'] = 'gzip'; + Logger.getInstance().debug('Request%s - original buffer length: %d bytes. Compressed buffer length: %d bytes.', requestUtil.describeRequestFromOptions(options), bufferUncompressed.buffer.byteLength, bufferCompressed.buffer.byteLength); } else { - Logger.getInstance().warn('Could not compress request data.'); + // Logging 'err' variable value should not be done, since it may contain compressed customer's data. + // It can be added only for debugging purposes. + Logger.getInstance().warn('Request%s - could not compress request data.', requestUtil.describeRequestFromOptions(options)); } }); } @@ -254,6 +295,7 @@ function prepareRequestOptions(options) { requestOptions.httpAgent = agent; } + Logger.getInstance().debug('Request%s - options - timeout: %s, retryDelay: %s, responseType: %s', requestUtil.describeRequestFromOptions(options), requestOptions.timeout, requestOptions.retryDelay, requestOptions.responseType); return requestOptions; } @@ -266,10 +308,9 @@ function prepareRequestOptions(options) { * @returns {Object} */ function normalizeHeaders(headers) { - let ret = headers; - + Logger.getInstance().trace('Normalizing headers'); if (Util.isObject(headers)) { - ret = { + const normalizedHeaders = { 'user-agent': Util.userAgent }; @@ -288,15 +329,19 @@ function normalizeHeaders(headers) { headerNameLowerCase = headerName.toLowerCase(); if ((headerNameLowerCase === 'accept') || (headerNameLowerCase === 'content-type')) { - ret[headerNameLowerCase] = headers[headerName]; + normalizedHeaders[headerNameLowerCase] = headers[headerName]; } else { - ret[headerName] = headers[headerName]; + normalizedHeaders[headerName] = headers[headerName]; } } } + Logger.getInstance().trace('Headers were normalized'); + return normalizedHeaders; + } else { + Logger.getInstance().trace('Headers were not an object. Original value will be returned.'); + return headers; } - return ret; } /** @@ -311,6 +356,7 @@ function normalizeHeaders(headers) { function normalizeResponse(response) { // if the response doesn't already have a getResponseHeader() method, add one if (response && !response.getResponseHeader) { + Logger.getInstance().trace('Request%s - normalizing.', requestUtil.describeRequestFromResponse(response)); response.getResponseHeader = function (header) { return response.headers && response.headers[ Util.isString(header) ? header.toLowerCase() : header]; @@ -323,4 +369,4 @@ function normalizeResponse(response) { } return response; -} \ No newline at end of file +} diff --git a/lib/http/browser.js b/lib/http/browser.js index a13f1828f..ce224d4de 100644 --- a/lib/http/browser.js +++ b/lib/http/browser.js @@ -5,6 +5,7 @@ const Util = require('../util'); const request = require('browser-request'); const Base = require('./base'); +const Logger = require('../logger'); /** * Creates a client that can be used to make requests in the browser. @@ -13,6 +14,8 @@ const Base = require('./base'); * @constructor */ function BrowserHttpClient(connectionConfig) { + Logger.getInstance().trace('Initializing BrowserHttpClient with Connection Config[%s]', + connectionConfig.describeIdentityAttributes()); Base.apply(this, [connectionConfig]); } diff --git a/lib/http/node.js b/lib/http/node.js index 5af2a989b..f911e3cf8 100644 --- a/lib/http/node.js +++ b/lib/http/node.js @@ -3,29 +3,33 @@ */ const Util = require('../util'); +const ProxyUtil = require('../proxy_util'); const Base = require('./base'); const HttpsAgent = require('../agent/https_ocsp_agent'); const HttpsProxyAgent = require('../agent/https_proxy_agent'); const HttpAgent = require('http').Agent; const GlobalConfig = require('../../lib/global_config'); const Logger = require('../logger'); +const RequestUtil = require('../http/request_util'); /** * Returns the delay time calculated by exponential backoff with - * decorrelated jitter. - * for more details, check out: + * decorrelated jitter. For more details, check out: * http://www.awsarchitectureblog.com/2015/03/backoff.html - * @param base minimum seconds - * @param cap maximum seconds - * @param previousSleep previous sleep time * @return {Number} number of milliseconds to wait before retrying again the request. */ NodeHttpClient.prototype.constructExponentialBackoffStrategy = function () { - let sleep = this._connectionConfig.getRetrySfStartingSleepTime(); + Logger.getInstance().trace('Calculating exponential backoff strategy'); + + const previousSleepTime = this._connectionConfig.getRetrySfStartingSleepTime(); + // maximum seconds const cap = this._connectionConfig.getRetrySfMaxSleepTime(); + // minimum seconds const base = 1; - sleep = Util.nextSleepTime(base, cap, sleep); - return sleep * 1000; + const nextSleepTime = Util.nextSleepTime(base, cap, previousSleepTime); + const nextSleepTimeInMilliseconds = nextSleepTime * 1000; + Logger.getInstance().trace('Calculated exponential backoff strategy sleep time: %d', nextSleepTimeInMilliseconds); + return nextSleepTimeInMilliseconds; }; /** @@ -35,6 +39,8 @@ NodeHttpClient.prototype.constructExponentialBackoffStrategy = function () { * @constructor */ function NodeHttpClient(connectionConfig) { + Logger.getInstance().trace('Initializing NodeHttpClient with Connection Config[%s]', + connectionConfig.describeIdentityAttributes()); Base.apply(this, [connectionConfig]); } @@ -43,26 +49,28 @@ Util.inherits(NodeHttpClient, Base); const httpsAgentCache = new Map(); function getFromCacheOrCreate(agentClass, options, agentId) { + Logger.getInstance().trace('Agent[id: %s] - trying to retrieve from cache or create.', agentId); let agent = {}; function createAgent(agentClass, agentOptions, agentId) { + Logger.getInstance().trace('Agent[id: %s] - creating a new agent instance.', agentId); const agent = agentClass(agentOptions); httpsAgentCache.set(agentId, agent); - Logger.getInstance().trace(`Create and add to cache new agent ${agentId}`); + Logger.getInstance().trace('Agent[id: %s] - new instance stored in cache.', agentId); // detect and log PROXY envvar + agent proxy settings - const compareAndLogEnvAndAgentProxies = Util.getCompareAndLogEnvAndAgentProxies(agentOptions); - Logger.getInstance().debug(`Proxy settings used in requests:${compareAndLogEnvAndAgentProxies.messages}`); + const compareAndLogEnvAndAgentProxies = ProxyUtil.getCompareAndLogEnvAndAgentProxies(agentOptions); + Logger.getInstance().debug('Agent[id: %s] - proxy settings used in requests: %s', agentId, compareAndLogEnvAndAgentProxies.messages); // if there's anything to warn on (e.g. both envvar + agent proxy used, and they are different) // log warnings on them if (compareAndLogEnvAndAgentProxies.warnings) { - Logger.getInstance().warn(`${compareAndLogEnvAndAgentProxies.warnings}`); + Logger.getInstance().warn('Agent[id: %s] - %s', agentId, compareAndLogEnvAndAgentProxies.warnings); } return agent; } if (httpsAgentCache.has(agentId)) { - Logger.getInstance().trace(`Get agent with id: ${agentId} from cache`); + Logger.getInstance().trace('Agent[id: %s] - retrieving an agent instance from cache.', agentId); agent = httpsAgentCache.get(agentId); } else { agent = createAgent(agentClass, options, agentId); @@ -80,7 +88,7 @@ function enrichAgentOptionsWithProxyConfig(agentOptions, proxy) { } } -function isBypassProxy(proxy, destination) { +function isBypassProxy(proxy, destination, agentId) { if (proxy && proxy.noProxy) { const bypassList = proxy.noProxy.split('|'); for (let i = 0; i < bypassList.length; i++) { @@ -88,7 +96,7 @@ function isBypassProxy(proxy, destination) { host = host.replace('*', '.*?'); const matches = destination.match(host); if (matches) { - Logger.getInstance().debug('bypassing proxy for %s', destination); + Logger.getInstance().debug('Agent[id: %s] - bypassing proxy allowed for destination: %s', agentId, destination); return true; } } @@ -100,17 +108,19 @@ function isBypassProxy(proxy, destination) { * @inheritDoc */ NodeHttpClient.prototype.getAgent = function (parsedUrl, proxy, mock) { + Logger.getInstance().trace('Agent[url: %s] - getting an agent instance.', RequestUtil.describeURL(parsedUrl.href)); if (!proxy && GlobalConfig.isEnvProxyActive()) { const isHttps = parsedUrl.protocol === 'https:'; - proxy = Util.getProxyFromEnv(isHttps); + proxy = ProxyUtil.getProxyFromEnv(isHttps); if (proxy) { - Logger.getInstance().debug(`Load the proxy info from the environment variable host: ${proxy.host} in getAgent`); + Logger.getInstance().debug('Agent[url: %s] - proxy info loaded from the environment variable. Proxy host: %s', RequestUtil.describeURL(parsedUrl.href), proxy.host); } } return getProxyAgent(proxy, parsedUrl, parsedUrl.href, mock); }; function getProxyAgent(proxyOptions, parsedUrl, destination, mock) { + Logger.getInstance().trace('Agent[url: %s] - getting a proxy agent instance.', RequestUtil.describeURL(parsedUrl.href)); const agentOptions = { protocol: parsedUrl.protocol, hostname: parsedUrl.hostname, @@ -120,33 +130,46 @@ function getProxyAgent(proxyOptions, parsedUrl, destination, mock) { if (mock) { const mockAgent = mock.agentClass(agentOptions); if (mockAgent.protocol === parsedUrl.protocol) { + Logger.getInstance().debug('Agent[url: %s] - the mock agent will be used.', RequestUtil.describeURL(parsedUrl.href)); return mockAgent; } } - const agentId = createAgentId(agentOptions.protocol, agentOptions.hostname, agentOptions.keepAlive); - const bypassProxy = isBypassProxy(proxyOptions, destination); + const destHost = ProxyUtil.getHostFromURL(destination); + const agentId = createAgentId(agentOptions.protocol, agentOptions.hostname, destHost, agentOptions.keepAlive); + Logger.getInstance().debug('Agent[id: %s] - the destination host is: %s.', agentId, destHost); + + const bypassProxy = isBypassProxy(proxyOptions, destination, agentId); let agent; const isHttps = agentOptions.protocol === 'https:'; if (isHttps) { if (proxyOptions && !bypassProxy) { + Logger.getInstance().trace('Agent[id: %s] - using HTTPS agent enriched with proxy options.', agentId); enrichAgentOptionsWithProxyConfig(agentOptions, proxyOptions); agent = getFromCacheOrCreate(HttpsProxyAgent, agentOptions, agentId); } else { + Logger.getInstance().trace('Agent[id: %s] - using HTTPS agent without proxy.', agentId); agent = getFromCacheOrCreate(HttpsAgent, agentOptions, agentId); } } else if (proxyOptions && !bypassProxy) { + Logger.getInstance().trace('Agent[id: %s] - using HTTP agent enriched with proxy options.', agentId); enrichAgentOptionsWithProxyConfig(agentOptions, proxyOptions); agent = getFromCacheOrCreate(HttpAgent, agentOptions, agentId); } else { + Logger.getInstance().trace('Agent[id: %s] - using HTTP agent without proxy.', agentId); agent = getFromCacheOrCreate(HttpAgent, agentOptions, agentId); } return agent; } -function createAgentId(protocol, hostname, keepAlive) { - return `${protocol}//${hostname}-${keepAlive ? 'keepAlive' : 'noKeepAlive'}`; +function createAgentId(protocol, hostname, destination, keepAlive) { + return `${protocol}//${hostname}-${destination}-${keepAlive ? 'keepAlive' : 'noKeepAlive'}`; +} + +//This is for the testing purpose. +function getAgentCacheSize() { + return httpsAgentCache.size; } -module.exports = { NodeHttpClient, getProxyAgent }; \ No newline at end of file +module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize, isBypassProxy }; \ No newline at end of file diff --git a/lib/http/request_util.js b/lib/http/request_util.js new file mode 100644 index 000000000..38f69222e --- /dev/null +++ b/lib/http/request_util.js @@ -0,0 +1,202 @@ +const LoggingUtil = require('../logger/logging_util'); +const sfParams = require('../constants/sf_params'); + +// Initial whitelist for attributes - they will be described with values +const DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITH_VALUES = [ + 'baseUrl', + 'path', + 'method', + sfParams.paramsNames.SF_REQUEST_ID, + sfParams.paramsNames.SF_REQUEST_GUID, + sfParams.paramsNames.SF_WAREHOUSE_NAME, + sfParams.paramsNames.SF_DB_NAME, + sfParams.paramsNames.SF_SCHEMA_NAME, +]; + +// Initial blacklist for attributes - described as present/not present only +const DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITHOUT_VALUES = [ + sfParams.paramsNames.SF_TOKEN +]; + +// Helper function to resolve attributes arrays given defaults and overrides. +function resolveAttributeList(defaultAttrs, overrideAttrs) { + return overrideAttrs || defaultAttrs; +} + +/** + * Describes a request based on its options. + * Should work with not-yet-parsed options as well (before calling prepareRequestOptions method). + * + * @param {Object} requestOptions - Object representing the request data with top-level keys. + * @param {Object} [options] - Options for describing attributes. + * @param {Array} [options.overrideAttributesDescribedWithValues] + * @param {Array} [options.overrideAttributesDescribedWithoutValues] + * @returns {string} A string representation of the request data. + */ +function describeRequestFromOptions( + requestOptions, + { + overrideAttributesDescribedWithValues, + overrideAttributesDescribedWithoutValues + } = {} +) { + const describingAttributesWithValues = resolveAttributeList( + DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITH_VALUES, + overrideAttributesDescribedWithValues + ); + + const describingAttributesWithoutValues = resolveAttributeList( + DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITHOUT_VALUES, + overrideAttributesDescribedWithoutValues + ); + + const { method, url, params } = requestOptions || {}; + + return describeRequestData( + { method, url, params }, + describingAttributesWithValues, + describingAttributesWithoutValues + ); +} + +/** + * Creates a string that represents request data from a response. + * Helps to identify the request that was the source of the response. + * + * @param {Object} response - Axios response object. + * @param {Object} [options] - Options for describing attributes. + * @param {Array} [options.overrideAttributesDescribedWithValues] + * @param {Array} [options.overrideAttributesDescribedWithoutValues] + * @returns {string} A string representation of the request data. + */ +function describeRequestFromResponse( + response, + { + overrideAttributesDescribedWithValues, + overrideAttributesDescribedWithoutValues + } = {} +) { + let method; + let url; + let params; + const responseConfig = response?.config; + + const describingAttributesWithValues = resolveAttributeList( + DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITH_VALUES, + overrideAttributesDescribedWithValues + ); + + const describingAttributesWithoutValues = resolveAttributeList( + DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITHOUT_VALUES, + overrideAttributesDescribedWithoutValues + ); + + if (responseConfig) { + method = responseConfig.method; + url = responseConfig.url; + params = responseConfig.params; + } + + return describeRequestData( + { method, url, params }, + describingAttributesWithValues, + describingAttributesWithoutValues + ); +} + +/** + * Constructs a string representation of request data. + * + * @param {Object} requestData - Object containing the method, url, and parameters. + * @param {string} requestData.method - HTTP method. + * @param {string} requestData.url - Request URL. + * @param {Object} [requestData.params] - Additional query parameters. + * @param {Array} attributesWithValues - Attributes to describe with values. + * @param {Array} attributesWithoutValues - Attributes to describe without values. + * @returns {string} A string describing the request data. + */ +function describeRequestData( + { method, url, params } = {}, + attributesWithValues, + attributesWithoutValues +) { + const requestObject = { + // Ensure consistent casing for methods to match request-response pairs in logs. + method: method?.toUpperCase(), + ...constructURLData(url, params), + }; + + return LoggingUtil.describeAttributes( + requestObject, + attributesWithValues, + attributesWithoutValues + ); +} + +/** + * Constructs an object representing URL data including the base URL, path, and query parameters. + * + * @param {string} url - The full URL. + * @param {Object} [params] - Additional query parameters. + * @returns {Object} Contains baseUrl, path, and merged query parameters. + */ +function constructURLData(url, params = {}) { + if (!url) { + return { baseUrl: undefined, path: undefined, queryParams: {} }; + } + + const urlObj = new URL(url); + const queryParams = { ...params }; + + urlObj.searchParams.forEach((value, key) => { + queryParams[key] = value; + }); + + const baseUrl = `${urlObj.protocol}//${urlObj.hostname}${urlObj.port ? `:${urlObj.port}` : ''}`; + + return { + baseUrl: baseUrl, + path: urlObj.pathname, + ...queryParams, + }; +} + +/** + * @param {string} url - The URL to describe. + * @param {Object} [options] - Options for describing attributes. + * @param {Array} [options.overrideAttributesDescribedWithValues] + * @param {Array} [options.overrideAttributesDescribedWithoutValues] + * @returns {string} A string describing the URL. + */ +function describeURL( + url, + { + overrideAttributesDescribedWithValues, + overrideAttributesDescribedWithoutValues + } = {} +) { + const describingAttributesWithValues = resolveAttributeList( + DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITH_VALUES, + overrideAttributesDescribedWithValues + ); + + const describingAttributesWithoutValues = resolveAttributeList( + DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITHOUT_VALUES, + overrideAttributesDescribedWithoutValues + ); + + const urlData = constructURLData(url); + + return LoggingUtil.describeAttributes( + urlData, + describingAttributesWithValues, + describingAttributesWithoutValues + ); +} + +exports.DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITH_VALUES = DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITH_VALUES; +exports.DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITHOUT_VALUES = DEFAULT_ATTRIBUTES_DESCRIBING_REQUEST_WITHOUT_VALUES; + +exports.describeRequestFromOptions = describeRequestFromOptions; +exports.describeRequestFromResponse = describeRequestFromResponse; +exports.describeURL = describeURL; diff --git a/lib/logger/execution_timer.js b/lib/logger/execution_timer.js new file mode 100644 index 000000000..22ea12f6d --- /dev/null +++ b/lib/logger/execution_timer.js @@ -0,0 +1,45 @@ +const { performance } = require('perf_hooks'); +const Logger = require('../logger'); +const Util = require('../util'); + +function ExecutionTimer() { + let startTime = null; + let endTime = null; + + // Private function to log and check if the timer was started + function wasStarted() { + return Util.exists(startTime); + + } + + this.start = function () { + startTime = performance.now(); + endTime = null; // Reset endTime if the timer is reused + return this; + }; + + + this.stop = function () { + if (!wasStarted()) { + // Returning this to allow chaining even after invalid call. + // startTime can be used to check, if any start point was ever recorded. + Logger.getInstance().debug('Tried to stop timer, that was not started'); + return this; + } + endTime = performance.now(); + return this; + }; + + // Get the duration in milliseconds + this.getDuration = function () { + if (!wasStarted()) { + return; + } + if (endTime === null) { + endTime = performance.now(); + } + return endTime - startTime; + }; +} + +module.exports = ExecutionTimer; diff --git a/lib/logger/logging_util.js b/lib/logger/logging_util.js new file mode 100644 index 000000000..b6fcbc3a3 --- /dev/null +++ b/lib/logger/logging_util.js @@ -0,0 +1,61 @@ +const Util = require('../util'); + +const PROVIDED_TEXT = 'provided'; +const NOT_PROVIDED_TEXT = 'not provided'; + +/** + * Describes the presence of a given value. If the value is not empty (as a string), + * returns the corresponding text (by default: 'provided' or 'not provided'). + * + * @param {*} valueToDescribe - The value to check for presence. + * @param {Object} [options] - Optional overrides for the "provided" and "not provided" text. + * @param {string} [options.overrideProvidedText] + * @param {string} [options.overrideNotProvidedText] + * @returns {string} A string indicating the presence of `valueToDescribe`. + */ +exports.describePresence = function (valueToDescribe, { overrideProvidedText, overrideNotProvidedText } = {}) { + const providedText = overrideProvidedText || PROVIDED_TEXT; + const notProvidedText = overrideNotProvidedText || NOT_PROVIDED_TEXT; + return Util.isNotEmptyAsString(valueToDescribe) ? providedText : notProvidedText; +}; + +/** + * @param {Object} sourceObject - The object holding attribute values. + * @param {Array} attributesWithValues - Attributes to show with their values. + * @param {Array} attributesWithoutValues - Attributes to show as present/not present. + * @returns {string} Comma-separated string describing the attributes. + */ +exports.attributesToString = function ( + sourceObject = {}, + attributesWithValues = [], + attributesWithoutValues = [] +) { + const withValues = attributesWithValues + .filter(attr => sourceObject[attr] !== undefined) + .map(attr => `${attr}=${String(sourceObject[attr])}`); + + const withoutValues = attributesWithoutValues + .filter(attr => sourceObject[attr] !== undefined) + .map(attr => `${attr} is ${exports.describePresence(sourceObject[attr])}`); + + return [...withValues, ...withoutValues].join(', '); +}; + +/** + * @param {Object} sourceObject - The object holding attribute values. + * @param {Array} attributesWithValues - Attributes to show with their values. + * @param {Array} attributesWithoutValues - Attributes to show as present/not present. + * @returns {string} A bracketed string of described attributes. + */ +exports.describeAttributes = function ( + sourceObject, + attributesWithValues, + attributesWithoutValues +) { + const attributesDescription = exports.attributesToString( + sourceObject, + attributesWithValues, + attributesWithoutValues + ); + return `[${attributesDescription}]`; +}; diff --git a/lib/proxy_util.js b/lib/proxy_util.js new file mode 100644 index 000000000..01f02d34f --- /dev/null +++ b/lib/proxy_util.js @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + +const Logger = require('./logger'); +const Errors = require('./errors'); +const Util = require('./util'); +const GlobalConfig = require('./global_config'); +const LoggingUtil = require('./logger/logging_util'); +const ErrorCodes = Errors.codes; +/** +* remove http:// or https:// from the input, e.g. used with proxy URL +* @param input +* @returns {string} +*/ +exports.removeScheme = function (input) { + return input.toString().replace(/(^\w+:|^)\/\//, ''); +}; + +/** + * Try to get the PROXY environmental variables + * On Windows, envvar name is case-insensitive, but on *nix, it's case-sensitive + * + * Compare them with the proxy specified on the Connection, if any + * Return with the log constructed from the components detection and comparison + * If there's something to warn the user about, return that too + * + * @param the agentOptions object from agent creation + * @returns {object} + */ +exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { + const envProxy = {}; + const logMessages = { 'messages': '', 'warnings': '' }; + envProxy.httpProxy = process.env.http_proxy || process.env.HTTP_PROXY; + envProxy.httpsProxy = process.env.https_proxy || process.env.HTTPS_PROXY; + envProxy.noProxy = process.env.no_proxy || process.env.NO_PROXY; + + envProxy.logHttpProxy = envProxy.httpProxy ? + 'HTTP_PROXY: ' + envProxy.httpProxy : 'HTTP_PROXY: '; + envProxy.logHttpsProxy = envProxy.httpsProxy ? + 'HTTPS_PROXY: ' + envProxy.httpsProxy : 'HTTPS_PROXY: '; + envProxy.logNoProxy = envProxy.noProxy ? + 'NO_PROXY: ' + envProxy.noProxy : 'NO_PROXY: '; + + // log PROXY envvars + if (envProxy.httpProxy || envProxy.httpsProxy) { + logMessages.messages = logMessages.messages + ' // PROXY environment variables: ' + + `${envProxy.logHttpProxy} ${envProxy.logHttpsProxy} ${envProxy.logNoProxy}.`; + } + + // log proxy config on Connection, if any set + if (agentOptions.host) { + const proxyHostAndPort = agentOptions.host + ':' + agentOptions.port; + const proxyProtocolHostAndPort = agentOptions.protocol ? + ' protocol=' + agentOptions.protocol + ' proxy=' + proxyHostAndPort + : ' proxy=' + proxyHostAndPort; + const proxyUsername = agentOptions.user ? ' user=' + agentOptions.user : ''; + logMessages.messages = logMessages.messages + ` // Proxy configured in Agent:${proxyProtocolHostAndPort}${proxyUsername}`; + + // check if both the PROXY envvars and Connection proxy config is set + // generate warnings if they are, and are also different + if (envProxy.httpProxy && + this.removeScheme(envProxy.httpProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { + logMessages.warnings = logMessages.warnings + ` Using both the HTTP_PROXY (${envProxy.httpProxy})` + + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` + + ' If you experience connectivity issues, try unsetting one of them.'; + } + if (envProxy.httpsProxy && + this.removeScheme(envProxy.httpsProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { + logMessages.warnings = logMessages.warnings + ` Using both the HTTPS_PROXY (${envProxy.httpsProxy})` + + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` + + ' If you experience connectivity issues, try unsetting one of them.'; + } + } + logMessages.messages = logMessages.messages ? logMessages.messages : ' none.'; + + return logMessages; +}; + +exports.validateProxy = function (proxy) { + const { host, port, noProxy, user, password } = proxy; + // check for missing proxyHost + Errors.checkArgumentExists(Util.exists(host), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_HOST); + + // check for invalid proxyHost + Errors.checkArgumentValid(Util.isString(host), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_HOST); + + // check for missing proxyPort + Errors.checkArgumentExists(Util.exists(port), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PORT); + + // check for invalid proxyPort + Errors.checkArgumentValid(Util.isNumber(port), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PORT); + + if (Util.exists(noProxy)) { + // check for invalid noProxy + Errors.checkArgumentValid(Util.isString(noProxy), + ErrorCodes.ERR_CONN_CREATE_INVALID_NO_PROXY); + } + + if (Util.exists(user) || Util.exists(password)) { + // check for missing proxyUser + Errors.checkArgumentExists(Util.exists(user), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_USER); + + // check for invalid proxyUser + Errors.checkArgumentValid(Util.isString(user), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_USER); + + // check for missing proxyPassword + Errors.checkArgumentExists(Util.exists(password), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PASS); + + // check for invalid proxyPassword + Errors.checkArgumentValid(Util.isString(password), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PASS); + + } else { + delete proxy.user; + delete proxy.password; + } +}; + +exports.validateEmptyString = function (value) { + return value !== '' ? value : undefined; +}; + +exports.getProxyFromEnv = function (isHttps = true) { + const protocol = isHttps ? 'https' : 'http'; + let proxyFromEnv = Util.getEnvVar(`${protocol}_proxy`); + if (!proxyFromEnv){ + return null; + } + + Logger.getInstance().debug(`Util.getProxyEnv: Using ${protocol.toUpperCase()}_PROXY from the environment variable`); + if (proxyFromEnv.indexOf('://') === -1) { + Logger.getInstance().info('Util.getProxyEnv: the protocol was missing from the environment proxy. Use the HTTP protocol.'); + proxyFromEnv = 'http' + '://' + proxyFromEnv; + } + proxyFromEnv = new URL(proxyFromEnv); + const proxy = { + host: Util.validateEmptyString(proxyFromEnv.hostname), + port: Number(Util.validateEmptyString(proxyFromEnv.port)), + user: Util.validateEmptyString(proxyFromEnv.username), + password: Util.validateEmptyString(proxyFromEnv.password), + protocol: Util.validateEmptyString(proxyFromEnv.protocol), + noProxy: this.getNoProxyEnv(), + }; + this.validateProxy(proxy); + return proxy; +}; + +exports.getNoProxyEnv = function () { + const noProxy = Util.getEnvVar('no_proxy'); + if (noProxy) { + return noProxy.split(',').join('|'); + } + return undefined; +}; + +exports.getHostFromURL = function (destination) { + if (destination.indexOf('://') === -1) { + destination = 'https' + '://' + destination; + } + + try { + return new URL(destination).hostname; + } catch (err) { + Logger.getInstance().error(`Failed to parse the destination to URL with the error: ${err}. Return destination as the host: ${destination}`); + return destination; + } +}; + +exports.getProxy = function (proxy, fileLocation, isHttps) { + if (!proxy && GlobalConfig.isEnvProxyActive()) { + proxy = this.getProxyFromEnv(isHttps); + if (proxy) { + Logger.getInstance().debug(`${fileLocation} loads the proxy info from the environment variable host: ${proxy.host}`); + } + } + return proxy; +}; + +exports.getAzureProxy = function (proxy) { + const AzureProxy = { + ...proxy, host: `${proxy.protocol}${(proxy.protocol).endsWith(':') ? '' : ':'}//${proxy.host}`, + }; + delete AzureProxy.noProxy; + delete AzureProxy.protocol; + + if (!Util.exists(AzureProxy.user) || !Util.exists(AzureProxy.password)) { + delete AzureProxy.user; + delete AzureProxy.password; + } + return AzureProxy; +}; + +/** + * Currently, there is no way to disable loading the proxy information from the environment path in Azure/blob. + * To control this proxy option on the driver side, A temporary workaround is hide(remove) the environment proxy from the process + * when the client is created (At this time, the client loads the proxy from the environment variables internally). + * After the client is created, restore them with the 'restoreEnvironmentProxy' function. + */ +let envProxyList; +const proxyEnvList = ['http_proxy', 'https_proxy', 'no_proxy']; +exports.hideEnvironmentProxy = function () { + if (GlobalConfig.isEnvProxyActive()) { + return; + } + Logger.getInstance().debug('As the useEnvProxy option is disabled, the proxy environment variables are temporarily hidden during the creation of an Azure client'); + envProxyList = []; + for (const envVar of proxyEnvList) { + saveProxyInfoInList(envVar); + if (!Util.isWindows()) { + saveProxyInfoInList(envVar.toUpperCase()); + } + } +}; + +function saveProxyInfoInList(envVar) { + const proxyEnv = process.env[envVar]; + envProxyList.push(process.env[envVar]); + delete process.env[envVar]; + + if (Util.exists(proxyEnv)) { + Logger.getInstance().debug(`Temporarily exclude ${envVar} from the environment variable value: ${proxyEnv}`); + } else { + Logger.getInstance().debug(`${envVar} was not defined, nothing to do`); + } +} + +exports.restoreEnvironmentProxy = function () { + if (GlobalConfig.isEnvProxyActive()) { + return; + } + + const iterator = envProxyList[Symbol.iterator](); + let nextValue = iterator.next().value; + for (const envVar of proxyEnvList) { + if (Util.exists(nextValue)) { + Logger.getInstance().debug(`The ${envVar} value exists with the value: ${nextValue} Restore back the proxy environment variable values`); + process.env[envVar] = nextValue; + } + nextValue = iterator.next().value; + + if (!Util.isWindows()) { + if (Util.exists(nextValue)) { + Logger.getInstance().debug(`The ${envVar.toUpperCase()} value exists with the value: ${nextValue} Restore back the proxy environment variable values (for Non-Windows machine)`); + process.env[envVar.toUpperCase()] = nextValue; + } + nextValue = iterator.next().value; + } + } + Logger.getInstance().debug('An Azure client has been created. Restore back the proxy environment variable values'); +}; + +exports.describeProxy = function (proxy) { + return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, proxyUser: ${proxy.user}, ` + + `proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, ` + + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`; +}; \ No newline at end of file diff --git a/lib/request_util_test.js b/lib/request_util_test.js new file mode 100644 index 000000000..06f272936 --- /dev/null +++ b/lib/request_util_test.js @@ -0,0 +1,149 @@ +const assert = require('assert'); +const RequestUtil = require('../lib/http/request_util'); +const sfParams = require('../lib/constants/sf_params'); + +describe('RequestUtil', function () { + describe('describeRequestFromOptions', function () { + it('should describe request with default attributes when given basic request options', function () { + const requestOptions = { + method: 'get', + url: 'https://example.com:8080/api/data?foo=bar', + params: { + requestId: '12345', + // eslint-disable-next-line camelcase + request_guid: 'abcde', + warehouse: 'test_warehouse', + databaseName: 'test_db', + schemaName: 'test_schema', + } + }; + + const result = RequestUtil.describeRequestFromOptions(requestOptions); + + // Check if defaults appear correctly + assert.ok(result.includes('method=GET'), 'Should include method=GET'); + assert.ok(result.includes('baseUrl=https://example.com:8080'), 'Should include baseUrl'); + assert.ok(result.includes('path=/api/data'), 'Should include path'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_REQUEST_ID}=12345`), 'Should include requestId'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_REQUEST_GUID}=abcde`), 'Should include request_guid'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_WAREHOUSE_NAME}=test_warehouse`), 'Should include warehouse'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_DB_NAME}=test_db`), 'Should include databaseName'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_SCHEMA_NAME}=test_schema`), 'Should include schemaName'); + + // token not present, should not appear + assert.ok(!result.includes('token='), 'Should not include token='); + assert.ok(!result.includes('token is'), 'Should not include token presence'); + }); + + it('should handle when no requestOptions are provided', function () { + const result = RequestUtil.describeRequestFromOptions(undefined); + // Should be empty brackets since nothing can be described + assert.strictEqual(result, '[]', 'Should return empty brackets'); + }); + + it('should use overridden attributes lists if provided', function () { + const requestOptions = { + method: 'post', + url: 'https://example.org/resource?test=1', + params: { + token: 'secret_token_value' + } + }; + + const overrideAttributesDescribedWithValues = ['baseUrl']; + const overrideAttributesDescribedWithoutValues = ['token']; + + const result = RequestUtil.describeRequestFromOptions(requestOptions, { + overrideAttributesDescribedWithValues, + overrideAttributesDescribedWithoutValues + }); + + assert.ok(result.includes('baseUrl=https://example.org'), 'Should include only baseUrl with value'); + assert.ok(result.includes('token is provided'), 'Should indicate token is provided'); + // Should not have path or other defaults + assert.ok(!result.includes('path='), 'Should not include path'); + }); + }); + + describe('describeRequestFromResponse', function () { + it('should describe the request from a response config', function () { + const response = { + config: { + method: 'post', + url: 'https://api.example.com/action?query=value', + params: { + // eslint-disable-next-line camelcase + request_guid: 'xyz123', + warehouse: 'wh1' + } + } + }; + + const result = RequestUtil.describeRequestFromResponse(response); + assert.ok(result.includes('method=POST'), 'Should have POST method'); + assert.ok(result.includes('baseUrl=https://api.example.com'), 'Should have correct baseUrl'); + assert.ok(result.includes('path=/action'), 'Should have correct path'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_REQUEST_GUID}=xyz123`), 'Should have request_guid'); + assert.ok(result.includes(`${sfParams.paramsNames.SF_WAREHOUSE_NAME}=wh1`), 'Should have warehouse'); + }); + + it('should handle a response without config', function () { + const response = {}; + const result = RequestUtil.describeRequestFromResponse(response); + // No config means no attributes described + assert.strictEqual(result, '[]', 'Should return empty brackets'); + }); + + it('should allow overriding described attributes from a response', function () { + const response = { + config: { + method: 'get', + url: 'https://myapp.com/test?user=john', + params: { + token: 'abcd' + } + } + }; + + const result = RequestUtil.describeRequestFromResponse(response, { + overrideAttributesDescribedWithValues: ['baseUrl'], + overrideAttributesDescribedWithoutValues: ['token'] + }); + + assert.ok(result.includes('baseUrl=https://myapp.com'), 'Should include overridden baseUrl'); + assert.ok(result.includes('token is provided'), 'Should indicate token is provided'); + assert.ok(!result.includes('path='), 'Should not include path since overridden'); + }); + }); + + describe('describeURL', function () { + it('should describe a URL using default attributes', function () { + const url = 'https://example.net:3000/endpoint/sub?flag=true'; + const result = RequestUtil.describeURL(url); + + assert.ok(result.includes('baseUrl=https://example.net:3000'), 'Should include baseUrl'); + assert.ok(result.includes('path=/endpoint/sub'), 'Should include path'); + + // Other defaults would not show because not set + assert.ok(!result.includes('requestId='), 'Should not have requestId'); + assert.ok(!result.includes('token='), 'Should not have token'); + }); + + it('should describe a URL with overridden attributes', function () { + const url = 'https://something.com/path'; + const result = RequestUtil.describeURL(url, { + overrideAttributesDescribedWithValues: ['baseUrl', 'path'], + overrideAttributesDescribedWithoutValues: ['token'] + }); + + assert.ok(result.includes('baseUrl=https://something.com'), 'Should have baseUrl'); + assert.ok(result.includes('path=/path'), 'Should have path'); + assert.ok(!result.includes('token is'), 'Should not indicate token presence'); + }); + + it('should handle empty or invalid URLs', function () { + const result = RequestUtil.describeURL(''); + assert.strictEqual(result, '[]', 'Should return empty brackets for invalid URL'); + }); + }); +}); diff --git a/lib/secret_detector.js b/lib/secret_detector.js index 638c696cc..700b576c8 100644 --- a/lib/secret_detector.js +++ b/lib/secret_detector.js @@ -64,13 +64,17 @@ function SecretDetector(customPatterns, mock) { 'gim'); const PRIVATE_KEY_DATA_PATTERN = new RegExp(String.raw`"privateKeyData": "([a-z0-9/+=\\n]{10,})"`, 'gim'); - const CONNECTION_TOKEN_PATTERN = new RegExp(String.raw`(token|assertion content)([\'\"\s:=]+)([a-z0-9=/_\-\+]{8,})`, + // Colon in the group ([a-z0-9=/:_%-+]{8,}) was added to detect tokens that contain additional details before the actual token. + // Such as version or hint (token=ver:1-hint:1233-realToken...). + const CONNECTION_TOKEN_PATTERN = new RegExp(String.raw`(token|assertion content)([\'\"\s:=]+)([a-z0-9=/:_\%\-\+]{8,})`, 'gi'); const PASSWORD_PATTERN = new RegExp( String.raw`(password|pwd)([\'\"\s:=]+)([a-z0-9!\"#\$%&\\\'\(\)\*\+\,-\./:;<=>\?\@\[\]\^_` + '`' + String.raw`\{\|\}~]{8,})`, 'gi'); + const PASSCODE_PATTERN = new RegExp(String.raw`(passcode|otp|pin|otac)\s*([:=])\s*([0-9]{4,6})`, 'gi'); + function maskAwsKeys(text) { return text.replace(AWS_KEY_PATTERN, String.raw`$1$2****`); @@ -100,6 +104,11 @@ function SecretDetector(customPatterns, mock) { return text.replace(PASSWORD_PATTERN, String.raw`$1$2****`); } + function maskPasscode(text) { + return text.replace(PASSCODE_PATTERN, String.raw`$1$2****`); + } + + /** * Masks any secrets. * @@ -128,13 +137,15 @@ function SecretDetector(customPatterns, mock) { } maskedtxt = - maskConnectionToken( - maskPassword( - maskPrivateKeyData( - maskPrivateKey( - maskAwsToken( - maskSasToken( - maskAwsKeys(text) + maskPasscode( + maskConnectionToken( + maskPassword( + maskPrivateKeyData( + maskPrivateKey( + maskAwsToken( + maskSasToken( + maskAwsKeys(text) + ) ) ) ) diff --git a/lib/services/large_result_set.js b/lib/services/large_result_set.js index 826376561..ac1a570d2 100644 --- a/lib/services/large_result_set.js +++ b/lib/services/large_result_set.js @@ -65,8 +65,8 @@ function LargeResultSetService(connectionConfig, httpClient) { // err happens on timeouts and response is passed when server responded if (err || isUnsuccessfulResponse(response)) { // if we're running in DEBUG loglevel, probably we want to see the full error too - const logErr = err ? JSON.stringify(err, Util.getCircularReplacer()) - : `status: ${JSON.stringify(response.status)} ${JSON.stringify(response.statusText)}` + const logErr = err ? JSON.stringify(err, Object.getOwnPropertyNames(err)) + : `status: ${JSON.stringify(response.status)} ${JSON.stringify(response.statusText)}` + ` headers: ${JSON.stringify(response.headers)}`; Logger.getInstance().debug('Encountered an error when getting data from cloud storage: ' + logErr); // if we haven't exceeded the maximum number of retries yet and the diff --git a/lib/services/sf.js b/lib/services/sf.js index f32ecb04f..2e339c204 100644 --- a/lib/services/sf.js +++ b/lib/services/sf.js @@ -60,6 +60,7 @@ const AuthenticationTypes = require('../authentication/authentication_types'); const AuthOkta = require('../authentication/auth_okta'); const AuthKeypair = require('../authentication/auth_keypair'); const Authenticator = require('../authentication/authentication'); +const sfParams = require('../constants/sf_params'); function isRetryableNetworkError(err) { // anything other than REVOKED error can be retryable. @@ -578,10 +579,16 @@ function StateAbstract(options) { * * @param {Object} requestOptions * @param {Object} httpClient - * + * @param {Object} auth * @returns {Object} the http request object. */ function sendHttpRequest(requestOptions, httpClient, auth) { + + const params = requestOptions.params || {}; + if (!requestOptions.excludeGuid) { + addGuidToParams(params); + } + const realRequestOptions = { method: requestOptions.method, @@ -589,12 +596,13 @@ function StateAbstract(options) { url: requestOptions.absoluteUrl, gzip: requestOptions.gzip, json: requestOptions.json, + params: params, callback: async function (err, response, body) { // if we got an error, wrap it into a network error if (err) { // if we're running in DEBUG loglevel, probably we want to see the full error instead Logger.getInstance().debug('Encountered an error when sending the request. Details: ' - + JSON.stringify(err, Util.getCircularReplacer())); + + JSON.stringify(err, Object.getOwnPropertyNames(err))); err = Errors.createNetworkError( ErrorCodes.ERR_SF_NETWORK_COULD_NOT_CONNECT, err); @@ -679,8 +687,8 @@ function StateAbstract(options) { }; if (requestOptions.retry > 2) { - const includeParam = requestOptions.url.includes('?'); - realRequestOptions.url += (includeParam ? '&' : '?'); + const includesParam = requestOptions.url.includes('?'); + realRequestOptions.url += (includesParam ? '&' : '?'); realRequestOptions.url += ('clientStartTime=' + requestOptions.startTime + '&' + 'retryCount=' + (requestOptions.retry - 1)); @@ -703,7 +711,7 @@ function StateAbstract(options) { /////////////////////////////////////////////////////////////////////////// /** - * Creates a new Request. + * Creates a new Request to Snowflake. * * @param {Object} requestOptions * @constructor @@ -721,18 +729,31 @@ function StateAbstract(options) { // pre-process the request options this.preprocessOptions(this.requestOptions); + const params = this.requestOptions.params || {}; + if (!this.requestOptions.excludeGuid) { + addGuidToParams(params); + } const options = { method: this.requestOptions.method, headers: this.requestOptions.headers, url: this.requestOptions.absoluteUrl, - json: this.requestOptions.json + json: this.requestOptions.json, + params: params }; // issue the async http request + //TODO: this should be wrapped with the same operations, as in the synchronous send method's callback. return await httpClient.requestAsync(options); }; + function addGuidToParams(params) { + // In case of repeated requests for the same request ID, + // the Global UID is added for better traceability. + const guid = uuidv4(); + params[sfParams.paramsNames.SF_REQUEST_GUID] = guid; + } + /** * Sends out the request. * @@ -765,6 +786,8 @@ function StateAbstract(options) { // augment the options with the absolute url requestOptions.absoluteUrl = this.buildFullUrl(requestOptions.url); + + requestOptions.excludeGuid = !Util.exists(requestOptions.excludeGuid) ? false : requestOptions.excludeGuid; }; /** @@ -1256,7 +1279,7 @@ function buildLoginUrl(connectionConfig) { const queryStringObject = {}; if (!connectionConfig.isQaMode()) { - // no requestId is attached to login-request in test mode. + // No requestId is attached to login-request in test mode. queryStringObject.requestId = uuidv4(); } for (let index = 0, length = queryParams.length; index < length; index++) { diff --git a/lib/util.js b/lib/util.js index 5af3e2a34..cc22322d3 100644 --- a/lib/util.js +++ b/lib/util.js @@ -8,7 +8,6 @@ const os = require('os'); const Logger = require('./logger'); const fs = require('fs'); const Errors = require('./errors'); -const ErrorCodes = Errors.codes; /** * Note: A simple wrapper around util.inherits() for now, but this might change @@ -618,66 +617,6 @@ exports.isCorrectSubdomain = function (value) { return subdomainRegex.test(value); }; -/** - * Try to get the PROXY environmental variables - * On Windows, envvar name is case-insensitive, but on *nix, it's case-sensitive - * - * Compare them with the proxy specified on the Connection, if any - * Return with the log constructed from the components detection and comparison - * If there's something to warn the user about, return that too - * - * @param the agentOptions object from agent creation - * @returns {object} - */ -exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { - const envProxy = {}; - const logMessages = { 'messages': '', 'warnings': '' }; - envProxy.httpProxy = process.env.HTTP_PROXY || process.env.http_proxy; - envProxy.httpsProxy = process.env.HTTPS_PROXY || process.env.https_proxy; - envProxy.noProxy = process.env.NO_PROXY || process.env.no_proxy; - - envProxy.logHttpProxy = envProxy.httpProxy ? - 'HTTP_PROXY: ' + envProxy.httpProxy : 'HTTP_PROXY: '; - envProxy.logHttpsProxy = envProxy.httpsProxy ? - 'HTTPS_PROXY: ' + envProxy.httpsProxy : 'HTTPS_PROXY: '; - envProxy.logNoProxy = envProxy.noProxy ? - 'NO_PROXY: ' + envProxy.noProxy : 'NO_PROXY: '; - - // log PROXY envvars - if (envProxy.httpProxy || envProxy.httpsProxy) { - logMessages.messages = logMessages.messages + ' // PROXY environment variables: ' - + `${envProxy.logHttpProxy} ${envProxy.logHttpsProxy} ${envProxy.logNoProxy}.`; - } - - // log proxy config on Connection, if any set - if (agentOptions.host) { - const proxyHostAndPort = agentOptions.host + ':' + agentOptions.port; - const proxyProtocolHostAndPort = agentOptions.protocol ? - ' protocol=' + agentOptions.protocol + ' proxy=' + proxyHostAndPort - : ' proxy=' + proxyHostAndPort; - const proxyUsername = agentOptions.user ? ' user=' + agentOptions.user : ''; - logMessages.messages = logMessages.messages + ` // Proxy configured in Agent:${proxyProtocolHostAndPort}${proxyUsername}`; - - // check if both the PROXY envvars and Connection proxy config is set - // generate warnings if they are, and are also different - if (envProxy.httpProxy && - this.removeScheme(envProxy.httpProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { - logMessages.warnings = logMessages.warnings + ` Using both the HTTP_PROXY (${envProxy.httpProxy})` - + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` - + ' If you experience connectivity issues, try unsetting one of them.'; - } - if (envProxy.httpsProxy && - this.removeScheme(envProxy.httpsProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { - logMessages.warnings = logMessages.warnings + ` Using both the HTTPS_PROXY (${envProxy.httpsProxy})` - + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` - + ' If you experience connectivity issues, try unsetting one of them.'; - } - } - logMessages.messages = logMessages.messages ? logMessages.messages : ' none.'; - - return logMessages; -}; - exports.buildCredentialCacheKey = function (host, username, credType) { if (!host || !username || !credType) { Logger.getInstance().debug('Cannot build the credential cache key because one of host, username, and credType is null'); @@ -710,15 +649,6 @@ exports.checkParametersDefined = function (...parameters) { return parameters.every((element) => element !== undefined && element !== null); }; -/** -* remove http:// or https:// from the input, e.g. used with proxy URL -* @param input -* @returns {string} -*/ -exports.removeScheme = function (input) { - return input.toString().replace(/(^\w+:|^)\/\//, ''); -}; - exports.buildCredentialCacheKey = function (host, username, credType) { if (!host || !username || !credType) { Logger.getInstance().debug('Cannot build the credential cache key because one of host, username, and credType is null'); @@ -810,86 +740,17 @@ exports.getEnvVar = function (variable) { return process.env[variable.toLowerCase()] || process.env[variable.toUpperCase()]; }; -exports.validateProxy = function (proxy) { - const { host, port, noProxy, user, password } = proxy; - // check for missing proxyHost - Errors.checkArgumentExists(this.exists(host), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_HOST); - - // check for invalid proxyHost - Errors.checkArgumentValid(this.isString(host), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_HOST); - - // check for missing proxyPort - Errors.checkArgumentExists(this.exists(port), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PORT); - - // check for invalid proxyPort - Errors.checkArgumentValid(this.isNumber(port), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PORT); - - if (this.exists(noProxy)) { - // check for invalid noProxy - Errors.checkArgumentValid(this.isString(noProxy), - ErrorCodes.ERR_CONN_CREATE_INVALID_NO_PROXY); - } - - if (this.exists(user) || this.exists(password)) { - // check for missing proxyUser - Errors.checkArgumentExists(this.exists(user), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_USER); - - // check for invalid proxyUser - Errors.checkArgumentValid(this.isString(user), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_USER); - - // check for missing proxyPassword - Errors.checkArgumentExists(this.exists(password), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PASS); - - // check for invalid proxyPassword - Errors.checkArgumentValid(this.isString(password), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PASS); - - } else { - delete proxy.user; - delete proxy.password; - } -}; - exports.validateEmptyString = function (value) { return value !== '' ? value : undefined; }; -exports.getProxyFromEnv = function (isHttps = true) { - const protocol = isHttps ? 'https' : 'http'; - let proxyFromEnv = this.getEnvVar(`${protocol}_proxy`); - if (!proxyFromEnv){ - return null; +exports.isNotEmptyAsString = function (variable) { + if (typeof variable === 'string') { + return variable; } - - Logger.getInstance().debug(`Util.getProxyEnv: Using ${protocol.toUpperCase()}_PROXY from the environment variable`); - if (proxyFromEnv.indexOf('://') === -1) { - Logger.getInstance().info('Util.getProxyEnv: the protocol was missing from the environment proxy. Use the HTTP protocol.'); - proxyFromEnv = 'http' + '://' + proxyFromEnv; - } - proxyFromEnv = new URL(proxyFromEnv); - const proxy = { - host: this.validateEmptyString(proxyFromEnv.hostname), - port: Number(this.validateEmptyString(proxyFromEnv.port)), - user: this.validateEmptyString(proxyFromEnv.username), - password: this.validateEmptyString(proxyFromEnv.password), - protocol: this.validateEmptyString(proxyFromEnv.protocol), - noProxy: this.getNoProxyEnv(), - }; - this.validateProxy(proxy); - return proxy; + return exports.exists(variable); }; -exports.getNoProxyEnv = function () { - const noProxy = this.getEnvVar('no_proxy'); - if (noProxy) { - return noProxy.split(',').join('|'); - } - return undefined; +exports.isWindows = function () { + return os.platform() === 'win32'; }; \ No newline at end of file diff --git a/package.json b/package.json index 596e34aad..227cff7a0 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,16 @@ { "name": "snowflake-sdk", - "version": "1.14.0", + "version": "2.0.0", "description": "Node.js driver for Snowflake", "dependencies": { "@aws-sdk/client-s3": "^3.388.0", - "@aws-sdk/node-http-handler": "^3.374.0", + "@smithy/node-http-handler": "^3.2.5", "@azure/storage-blob": "12.25.x", "@google-cloud/storage": "^7.7.0", "@techteamer/ocsp": "1.0.1", "asn1.js-rfc2560": "^5.0.0", "asn1.js-rfc5280": "^3.0.0", - "axios": "^1.6.8", + "axios": "^1.7.7", "big-integer": "^1.6.43", "bignumber.js": "^9.1.2", "binascii": "0.0.2", @@ -61,14 +61,15 @@ "lint:check:all:errorsOnly": "npm run lint:check:all -- --quiet", "lint:fix": "eslint --fix", "test": "mocha -timeout 180000 --recursive --full-trace test/unit/**/*.js test/unit/*.js", + "test:authentication": "mocha --exit -timeout 180000 --recursive --full-trace test/authentication/**/*.js test/authentication/*.js", "test:integration": "mocha -timeout 180000 --recursive --full-trace test/integration/**/*.js test/integration/*.js", "test:single": "mocha -timeout 180000 --full-trace", "test:system": "mocha -timeout 180000 --recursive --full-trace system_test/*.js", "test:unit": "mocha -timeout 180000 --recursive --full-trace test/unit/**/*.js test/unit/*.js", "test:unit:coverage": "nyc npm run test:unit", - "test:ci": "mocha -timeout 180000 --recursive --full-trace test/**/*.js", + "test:ci": "mocha -timeout 180000 --recursive --full-trace 'test/{unit,integration}/**/*.js'", "test:ci:coverage": "nyc npm run test:ci", - "test:ci:withSystemTests": "mocha -timeout 180000 --recursive --full-trace test/**/*.js system_test/*.js", + "test:ci:withSystemTests": "mocha -timeout 180000 --recursive --full-trace 'test/{unit,integration}/**/*.js' system_test/*.js", "test:ci:withSystemTests:coverage": "nyc npm run test:ci:withSystemTests", "test:manual": "mocha -timeout 180000 --full-trace --full-trace test/integration/testManualConnection.js" }, diff --git a/test/authentication/authTestsBaseClass.js b/test/authentication/authTestsBaseClass.js new file mode 100644 index 000000000..74308399a --- /dev/null +++ b/test/authentication/authTestsBaseClass.js @@ -0,0 +1,69 @@ +const assert = require('assert'); +const testUtil = require('../integration/testUtil'); +const snowflake = require('../../lib/snowflake'); + +class AuthTest { + constructor() { + this.connection = null; + this.error = null; + this.callbackCompleted = false; + } + + connectAsyncCallback() { + return (err) => { + this.error = err; + this.callbackCompleted = true; + }; + } + + async waitForCallbackCompletion() { + const timeout = Date.now() + 5000; + while (Date.now() < timeout) { + await new Promise(resolve => setTimeout(resolve, 100)); + if (this.callbackCompleted) { + return; + } + } + throw new Error('Connection callback did not complete'); + } + + async createConnection(connectionOption) { + this.connection = snowflake.createConnection(connectionOption); + } + + async connectAsync() { + await this.connection.connectAsync(this.connectAsyncCallback()); + await this.waitForCallbackCompletion(); + } + + async verifyConnectionIsUp() { + assert.ok(await this.connection.isValidAsync(), 'Connection is not valid'); + await testUtil.executeCmdAsync(this.connection, 'Select 1'); + } + + async verifyConnectionIsNotUp(message = 'Unable to perform operation because a connection was never established.') { + assert.ok(!(this.connection.isUp()), 'Connection should not be up'); + try { + await testUtil.executeCmdAsync(this.connection, 'Select 1'); + assert.fail('Expected error was not thrown'); + } catch (error) { + assert.strictEqual(error.message, message); + } + } + + async destroyConnection() { + if (this.connection !== undefined && this.connection !== null && this.connection.isUp()) { + await testUtil.destroyConnectionAsync(this.connection); + } + } + + verifyNoErrorWasThrown() { + assert.equal(this.error, null); + } + + verifyErrorWasThrown(message) { + assert.strictEqual(this.error?.message, message); + } +} + +module.exports = AuthTest; diff --git a/test/authentication/connectionParameters.js b/test/authentication/connectionParameters.js new file mode 100644 index 000000000..7d9d10745 --- /dev/null +++ b/test/authentication/connectionParameters.js @@ -0,0 +1,95 @@ +const snowflakeAuthTestProtocol = process.env.SNOWFLAKE_AUTH_TEST_PROTOCOL; +const snowflakeAuthTestHost = process.env.SNOWFLAKE_AUTH_TEST_HOST; +const snowflakeAuthTestPort = process.env.SNOWFLAKE_AUTH_TEST_PORT; +const snowflakeAuthTestAccount = process.env.SNOWFLAKE_AUTH_TEST_ACCOUNT; +const snowflakeAuthTestRole = process.env.SNOWFLAKE_AUTH_TEST_ROLE; +const snowflakeAuthTestBrowserUser = process.env.SNOWFLAKE_AUTH_TEST_BROWSER_USER; +const snowflakeAuthTestOktaAuth = process.env.SNOWFLAKE_AUTH_TEST_OKTA_AUTH; +const snowflakeAuthTestOktaUser = process.env.SNOWFLAKE_AUTH_TEST_OKTA_USER; +const snowflakeAuthTestOktaPass = process.env.SNOWFLAKE_AUTH_TEST_OKTA_PASS; +const snowflakeAuthTestOauthUrl = process.env.SNOWFLAKE_AUTH_TEST_OAUTH_URL; +const snowflakeAuthTestOauthClientId = process.env.SNOWFLAKE_AUTH_TEST_OAUTH_CLIENT_ID; +const snowflakeAuthTestOauthClientSecret = process.env.SNOWFLAKE_AUTH_TEST_OAUTH_CLIENT_SECRET; +const snowflakeAuthTestDatabase = process.env.SNOWFLAKE_AUTH_TEST_DATABASE; +const snowflakeAuthTestWarehouse = process.env.SNOWFLAKE_AUTH_TEST_WAREHOUSE; +const snowflakeAuthTestSchema = process.env.SNOWFLAKE_AUTH_TEST_SCHEMA; +const snowflakeAuthTestPrivateKeyPath = process.env.SNOWFLAKE_AUTH_TEST_PRIVATE_KEY_PATH; +const snowflakeAuthTestInvalidPrivateKeyPath = process.env.SNOWFLAKE_AUTH_TEST_INVALID_PRIVATE_KEY_PATH; +const snowflakeAuthTestPrivateKeyPassword = process.env.SNOWFLAKE_AUTH_TEST_PRIVATE_KEY_PASSWORD; +const snowflakeAuthTestEncryptedPrivateKeyPath = process.env.SNOWFLAKE_AUTH_TEST_ENCRYPTED_PRIVATE_KEY_PATH; + +const accessUrlAuthTests = snowflakeAuthTestProtocol + '://' + snowflakeAuthTestHost + ':' + + snowflakeAuthTestPort; + +const baseParameters = + { + accessUrl: accessUrlAuthTests, + account: snowflakeAuthTestAccount, + role: snowflakeAuthTestRole, + host: snowflakeAuthTestHost, + warehouse: snowflakeAuthTestWarehouse, + database: snowflakeAuthTestDatabase, + schema: snowflakeAuthTestSchema, + }; + +const externalBrowser = + { + ...baseParameters, + username: snowflakeAuthTestBrowserUser, + authenticator: 'EXTERNALBROWSER' + }; + +const okta = + { + ...baseParameters, + username: snowflakeAuthTestOktaUser, + password: snowflakeAuthTestOktaPass, + authenticator: snowflakeAuthTestOktaAuth + }; + +const oauth = + { + ...baseParameters, + username: snowflakeAuthTestOktaUser, + authenticator: 'OAUTH' + }; + +const keypairPrivateKey = + { + ...baseParameters, + username: snowflakeAuthTestOktaUser, + authenticator: 'SNOWFLAKE_JWT' + }; + +const keypairPrivateKeyPath = + { + ...baseParameters, + username: snowflakeAuthTestOktaUser, + privateKeyPath: snowflakeAuthTestPrivateKeyPath, + authenticator: 'SNOWFLAKE_JWT' + }; + +const keypairEncryptedPrivateKeyPath = + { + ...baseParameters, + username: snowflakeAuthTestOktaUser, + privateKeyPass: snowflakeAuthTestPrivateKeyPassword, + privateKeyPath: snowflakeAuthTestEncryptedPrivateKeyPath, + authenticator: 'SNOWFLAKE_JWT' + }; + +exports.externalBrowser = externalBrowser; +exports.okta = okta; +exports.oauth = oauth; +exports.keypairPrivateKey = keypairPrivateKey; +exports.keypairPrivateKeyPath = keypairPrivateKeyPath; +exports.keypairEncryptedPrivateKeyPath = keypairEncryptedPrivateKeyPath; +exports.snowflakeTestBrowserUser = snowflakeAuthTestBrowserUser; +exports.snowflakeAuthTestOktaUser = snowflakeAuthTestOktaUser; +exports.snowflakeAuthTestOktaPass = snowflakeAuthTestOktaPass; +exports.snowflakeAuthTestRole = snowflakeAuthTestRole; +exports.snowflakeAuthTestOauthClientId = snowflakeAuthTestOauthClientId; +exports.snowflakeAuthTestOauthClientSecret = snowflakeAuthTestOauthClientSecret; +exports.snowflakeAuthTestOauthUrl = snowflakeAuthTestOauthUrl; +exports.snowflakeAuthTestPrivateKeyPath = snowflakeAuthTestPrivateKeyPath; +exports.snowflakeAuthTestInvalidPrivateKeyPath = snowflakeAuthTestInvalidPrivateKeyPath; diff --git a/test/authentication/testExternalBrowser.js b/test/authentication/testExternalBrowser.js new file mode 100644 index 000000000..ade0f03ce --- /dev/null +++ b/test/authentication/testExternalBrowser.js @@ -0,0 +1,157 @@ +const assert = require('assert'); +const connParameters = require('./connectionParameters'); +const { spawn } = require('child_process'); +const Util = require('../../lib/util'); +const JsonCredentialManager = require('../../lib/authentication/secure_storage/json_credential_manager'); +const AuthTest = require('./authTestsBaseClass.js'); + +describe('External browser authentication tests', function () { + const runAuthTestsManually = process.env.RUN_AUTH_TESTS_MANUALLY === 'true'; + const cleanBrowserProcessesPath = '/externalbrowser/cleanBrowserProcesses.js'; + const provideBrowserCredentialsPath = '/externalbrowser/provideBrowserCredentials.js'; + const login = connParameters.snowflakeTestBrowserUser; + const password = connParameters.snowflakeAuthTestOktaPass; + let authTest; + + beforeEach(async () => { + authTest = new AuthTest(); + await cleanBrowserProcesses(); + }); + + afterEach(async () => { + await authTest.destroyConnection(); + }); + + describe('External browser tests', async () => { + it('Successful connection', async () => { + const connectionOption = { ...connParameters.externalBrowser, clientStoreTemporaryCredential: false }; + authTest.createConnection(connectionOption); + const provideCredentialsPromise = execWithTimeout('node', [provideBrowserCredentialsPath, 'success', login, password], 15000); + await connectAndProvideCredentials(provideCredentialsPromise); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('Mismatched Username', async () => { + const connectionOption = { ...connParameters.externalBrowser, username: 'differentUsername', clientStoreTemporaryCredential: false }; + authTest.createConnection(connectionOption); + const provideCredentialsPromise = execWithTimeout('node', [provideBrowserCredentialsPath, 'success', login, password], 15000); + await connectAndProvideCredentials(provideCredentialsPromise); + authTest.verifyErrorWasThrown('The user you were trying to authenticate as differs from the user currently logged in at the IDP.'); + await authTest.verifyConnectionIsNotUp('Unable to perform operation using terminated connection.'); + }); + + it('Wrong credentials', async () => { + const login = 'itsnotanaccount.com'; + const password = 'fakepassword'; + const connectionOption = { ...connParameters.externalBrowser, browserActionTimeout: 10000, clientStoreTemporaryCredential: false }; + authTest.createConnection(connectionOption); + const provideCredentialsPromise = execWithTimeout('node', [provideBrowserCredentialsPath, 'fail', login, password]); + await connectAndProvideCredentials(provideCredentialsPromise); + authTest.verifyErrorWasThrown('Error while getting SAML token: Browser action timed out after 10000 ms.'); + await authTest.verifyConnectionIsNotUp(); + }); + + it('External browser timeout', async () => { + const connectionOption = { ...connParameters.externalBrowser, browserActionTimeout: 100, clientStoreTemporaryCredential: false }; + authTest.createConnection(connectionOption); + const connectToBrowserPromise = execWithTimeout('node', [provideBrowserCredentialsPath, 'timeout']); + await connectAndProvideCredentials(connectToBrowserPromise); + authTest.verifyErrorWasThrown('Error while getting SAML token: Browser action timed out after 100 ms.'); + await authTest.verifyConnectionIsNotUp(); + }); + }); + + describe('ID Token authentication tests', async () => { + const connectionOption = { ...connParameters.externalBrowser, clientStoreTemporaryCredential: true }; + const key = Util.buildCredentialCacheKey(connectionOption.host, connectionOption.username, 'ID_TOKEN'); + const defaultCredentialManager = new JsonCredentialManager(); + let firstIdToken; + + before(async () => { + await defaultCredentialManager.remove(key); + }); + + it('obtains the id token from the server and saves it on the local storage', async function () { + authTest.createConnection(connectionOption); + const provideCredentialsPromise = execWithTimeout('node', [provideBrowserCredentialsPath, 'success', login, password], 15000); + await connectAndProvideCredentials(provideCredentialsPromise); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('the token is saved in the credential manager', async function () { + firstIdToken = await defaultCredentialManager.read(key); + assert.notStrictEqual(firstIdToken, null); + }); + + it('authenticates by token, browser credentials not needed', async function () { + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('opens browser okta authentication again when token is incorrect', async function () { + await defaultCredentialManager.write(key, '1234'); + authTest.createConnection(connectionOption); + const provideCredentialsPromise = execWithTimeout('node', [provideBrowserCredentialsPath, 'success', login, password], 15000); + await connectAndProvideCredentials(provideCredentialsPromise); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('refreshes the token for credential cache key', async function () { + const newToken = await defaultCredentialManager.read(key); + assert.notStrictEqual(firstIdToken, newToken); + }); + }); + + async function cleanBrowserProcesses() { + if (!runAuthTestsManually) { + await execWithTimeout('node', [cleanBrowserProcessesPath], 15000); + } + } + + async function connectAndProvideCredentials(provideCredentialsPromise) { + if (runAuthTestsManually) { + await authTest.connectAsync(); + } else { + await Promise.allSettled([authTest.connectAsync(), provideCredentialsPromise]); + } + } +}); + +function execWithTimeout(command, args, timeout = 5000) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { shell: true }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data; + }); + + child.stderr.on('data', (data) => { + stderr += data; + }); + + child.on('error', (err) => { + reject(err); + }); + + child.on('close', (code) => { + if (code !== 0) { + reject(new Error(`Process exited with code: ${code}, error: ${stderr}`)); + } else { + resolve({ stdout, stderr }); + } + }); + + setTimeout(() => { + child.kill(); + reject(new Error('Process timed out')); + }, timeout); + }); +} diff --git a/test/authentication/testKeyPair.js b/test/authentication/testKeyPair.js new file mode 100644 index 000000000..60b48df28 --- /dev/null +++ b/test/authentication/testKeyPair.js @@ -0,0 +1,94 @@ +const AuthTest = require('./authTestsBaseClass'); +const connParameters = require('./connectionParameters'); +const snowflake = require('../../lib/snowflake'); +const path = require('path'); +const assert = require('node:assert'); +const fs = require('fs').promises; + + +describe('Key-pair authentication', function () { + let authTest; + + beforeEach(async () => { + authTest = new AuthTest(); + }); + + afterEach(async () => { + await authTest.destroyConnection(); + }); + + describe('Private key', function () { + it('Successful connection', async function () { + const privateKey = await getFileContent(connParameters.snowflakeAuthTestPrivateKeyPath); + const connectionOption = { ... connParameters.keypairPrivateKey, privateKey: privateKey }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('Invalid private key format', async function () { + const invalidPrivateKeyFormat = 'invalidKey'; + const connectionOption = { ... connParameters.keypairPrivateKey, privateKey: invalidPrivateKeyFormat }; + try { + snowflake.createConnection(connectionOption); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.message, 'Invalid private key. The specified value must be a string in pem format of type pkcs8'); + } + }); + + it('Invalid private key', async function () { + const privateKey = await getFileContent(connParameters.snowflakeAuthTestInvalidPrivateKeyPath); + const connectionOption = { ... connParameters.keypairPrivateKey, privateKey: privateKey }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + assert.match(authTest.error?.message, /JWT token is invalid./); + await authTest.verifyConnectionIsNotUp('Unable to perform operation using terminated connection.'); + }); + }); + + describe('Private key path', function () { + it('Successful connection', async function () { + const connectionOption = connParameters.keypairPrivateKeyPath; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('Invalid private key', async function () { + const connectionOption = { ...connParameters.keypairPrivateKeyPath, privateKeyPath: connParameters.snowflakeAuthTestInvalidPrivateKeyPath }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + assert.match(authTest.error?.message, /JWT token is invalid./); + await authTest.verifyConnectionIsNotUp('Unable to perform operation using terminated connection.'); + }); + + it('Successful connection using encrypted private key', async function () { + const connectionOption = connParameters.keypairEncryptedPrivateKeyPath; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + //todo SNOW-1844747 improve error message + it('Invalid private key password', async function () { + const connectionOption = { ...connParameters.keypairEncryptedPrivateKeyPath, privateKeyPass: 'invalid' }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + assert.match(authTest.error?.message, /bad decrypt/); + await authTest.verifyConnectionIsNotUp(); + }); + }); +}); + +async function getFileContent(filePath) { + try { + const absolutePath = path.resolve(filePath); + return await fs.readFile(absolutePath, 'utf8'); + } catch (err) { + throw new Error(`Error reading file: ${err.message}`); + } +} diff --git a/test/authentication/testOauth.js b/test/authentication/testOauth.js new file mode 100644 index 000000000..5f1148626 --- /dev/null +++ b/test/authentication/testOauth.js @@ -0,0 +1,67 @@ +const assert = require('assert'); +const connParameters = require('./connectionParameters'); +const axios = require('axios'); +const { snowflakeAuthTestOktaUser, snowflakeAuthTestOktaPass, snowflakeAuthTestRole, snowflakeAuthTestOauthClientId, + snowflakeAuthTestOauthClientSecret, snowflakeAuthTestOauthUrl +} = require('./connectionParameters'); +const AuthTest = require('./authTestsBaseClass'); + + +describe('Oauth authentication', function () { + let authTest; + + beforeEach(async () => { + authTest = new AuthTest(); + }); + + afterEach(async () => { + await authTest.destroyConnection(); + }); + + it('Successful connection', async function () { + const token = await getToken(); + const connectionOption = { ...connParameters.oauth, token: token }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('Invalid token', async function () { + const connectionOption = { ...connParameters.oauth, token: 'invalidToken' }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyErrorWasThrown('Invalid OAuth access token. '); + await authTest.verifyConnectionIsNotUp('Unable to perform operation using terminated connection.'); + }); + + it('Mismatched username', async function () { + const token = await getToken(); + const connectionOption = { ...connParameters.oauth, username: 'itsnotanaccount.com', token: token }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyErrorWasThrown('The user you were trying to authenticate as differs from the user tied to the access token.'); + await authTest.verifyConnectionIsNotUp('Unable to perform operation using terminated connection.'); + }); +}); + +async function getToken() { + const response = await axios.post(snowflakeAuthTestOauthUrl, data, { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + auth: { + username: snowflakeAuthTestOauthClientId, + password: snowflakeAuthTestOauthClientSecret + } + }); + assert.strictEqual(response.status, 200, 'Failed to get access token'); + return response.data.access_token; +} + +const data = [ + `username=${snowflakeAuthTestOktaUser}`, + `password=${snowflakeAuthTestOktaPass}`, + 'grant_type=password', + `scope=session:role:${snowflakeAuthTestRole.toLowerCase()}` +].join('&'); diff --git a/test/authentication/testOkta.js b/test/authentication/testOkta.js new file mode 100644 index 000000000..1230bb0db --- /dev/null +++ b/test/authentication/testOkta.js @@ -0,0 +1,39 @@ +const connParameters = require('./connectionParameters'); +const AuthTest = require('./authTestsBaseClass'); + +describe('Okta authentication', function () { + let authTest; + + beforeEach(async () => { + authTest = new AuthTest(); + }); + + afterEach(async () => { + await authTest.destroyConnection(); + }); + + it('Successful connection', async function () { + const connectionOption = connParameters.okta; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyNoErrorWasThrown(); + await authTest.verifyConnectionIsUp(); + }); + + it('Wrong credentials', async function () { + const connectionOption = { ...connParameters.okta, username: 'itsnotanaccount.com', password: 'fakepassword' }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyErrorWasThrown('Request failed with status code 401'); + await authTest.verifyConnectionIsNotUp(); + }); + + //todo SNOW-1844747 improve error message + it('Wrong okta url', async function () { + const connectionOption = { ...connParameters.okta, authenticator: 'https://testinvalidaccoount.com' }; + authTest.createConnection(connectionOption); + await authTest.connectAsync(); + authTest.verifyErrorWasThrown('Cannot read properties of null (reading \'ssoUrl\')'); + await authTest.verifyConnectionIsNotUp(); + }); +}); diff --git a/test/configureLogger.js b/test/configureLogger.js index c8f9728e7..ce8b7e9cf 100644 --- a/test/configureLogger.js +++ b/test/configureLogger.js @@ -5,7 +5,7 @@ const snowflake = require('../lib/snowflake'); /** * @param logLevel one of OFF | ERROR | WARN | INFO | DEBUG | TRACE */ -exports.configureLogger = (logLevel = 'ERROR') => { +exports.configureLogger = (logLevel = 'INFO') => { Logger.getInstance().closeTransports(); Logger.setInstance(new NodeLogger({ filePath: 'STDOUT' })); snowflake.configure({ logLevel }); diff --git a/test/integration/connectionOptions.js b/test/integration/connectionOptions.js index 38c0ee623..88c56e344 100644 --- a/test/integration/connectionOptions.js +++ b/test/integration/connectionOptions.js @@ -16,17 +16,6 @@ const snowflakeTestRole = process.env.SNOWFLAKE_TEST_ROLE; const snowflakeTestPassword = process.env.SNOWFLAKE_TEST_PASSWORD; const snowflakeTestAdminUser = process.env.SNOWFLAKE_TEST_ADMIN_USER; const snowflakeTestAdminPassword = process.env.SNOWFLAKE_TEST_ADMIN_PASSWORD; -const snowflakeTestBrowserUser = process.env.SNOWFLAKE_TEST_BROWSER_USER; -const snowflakeTestPrivateKeyUser = process.env.SNOWFLAKE_JWT_TEST_USER; -const snowflakeTestPrivateKey = process.env.SNOWFLAKE_TEST_PRIVATE_KEY; -const snowflakeTestPrivateKeyPath = process.env.SNOWFLAKE_TEST_PRIVATE_KEY_PATH; -const snowflakeTestPrivateKeyPass = process.env.SNOWFLAKE_TEST_PRIVATE_KEY_PASS; -const snowflakeTestPrivateKeyPathUnencrypted = process.env.SNOWFLAKE_TEST_PRIVATE_KEY_PATH_UNENCRYPTED; -const snowflakeTestOauthUser = process.env.SNOWFLAKE_TEST_OAUTH_USER; -const snowflakeTestToken = process.env.SNOWFLAKE_TEST_OAUTH_TOKEN; -const snowflakeTestOktaUser = process.env.SNOWFLAKE_TEST_OKTA_USER; -const snowflakeTestOktaPass = process.env.SNOWFLAKE_TEST_OKTA_PASS; -const snowflakeTestOktaAuth = process.env.SNOWFLAKE_TEST_OKTA_AUTH; const snowflakeTestPasscode = process.env.SNOWFLAKE_TEST_PASSCODE; if (snowflakeTestProtocol === undefined) { @@ -89,122 +78,12 @@ const wrongPwd = account: snowflakeTestAccount }; -const externalBrowser = -{ - accessUrl: accessUrl, - username: snowflakeTestBrowserUser, - account: snowflakeTestAccount, - warehouse: snowflakeTestWarehouse, - database: snowflakeTestDatabase, - schema: snowflakeTestSchema, - role: snowflakeTestRole, - host: snowflakeTestHost, - authenticator: 'EXTERNALBROWSER' -}; - -const externalBrowserWithShortTimeout = { - ...externalBrowser, - browserActionTimeout: 100, -}; - -const externalBrowserMismatchUser = -{ - accessUrl: accessUrl, - username: 'node', - account: snowflakeTestAccount, - authenticator: 'EXTERNALBROWSER' -}; - -const keypairPrivateKey = -{ - accessUrl: accessUrl, - username: snowflakeTestPrivateKeyUser, - account: snowflakeTestAccount, - warehouse: snowflakeTestWarehouse, - database: snowflakeTestDatabase, - schema: snowflakeTestSchema, - role: snowflakeTestRole, - privateKey: snowflakeTestPrivateKey, - authenticator: 'SNOWFLAKE_JWT' -}; - -const keypairPathEncrypted = -{ - accessUrl: accessUrl, - username: snowflakeTestPrivateKeyUser, - account: snowflakeTestAccount, - warehouse: snowflakeTestWarehouse, - database: snowflakeTestDatabase, - schema: snowflakeTestSchema, - role: snowflakeTestRole, - privateKeyPath: snowflakeTestPrivateKeyPath, - privateKeyPass: snowflakeTestPrivateKeyPass, - authenticator: 'SNOWFLAKE_JWT' -}; - -const keypairPathUnencrypted = -{ - accessUrl: accessUrl, - username: snowflakeTestPrivateKeyUser, - account: snowflakeTestAccount, - warehouse: snowflakeTestWarehouse, - database: snowflakeTestDatabase, - schema: snowflakeTestSchema, - role: snowflakeTestRole, - privateKeyPath: snowflakeTestPrivateKeyPathUnencrypted, - authenticator: 'SNOWFLAKE_JWT' -}; - -const keypairWrongToken = -{ - accessUrl: accessUrl, - username: 'node', - account: snowflakeTestAccount, - privateKey: snowflakeTestPrivateKey, - authenticator: 'SNOWFLAKE_JWT' -}; - const MFA = { ...valid, authenticator: 'USER_PWD_MFA_AUTHENTICATOR', passcode: snowflakeTestPasscode, }; -const oauth = -{ - accessUrl: accessUrl, - username: snowflakeTestOauthUser, - account: snowflakeTestAccount, - warehouse: snowflakeTestWarehouse, - database: snowflakeTestDatabase, - schema: snowflakeTestSchema, - role: snowflakeTestRole, - token: snowflakeTestToken, - authenticator: 'OAUTH' -}; - -const oauthMismatchUser = -{ - accessUrl: accessUrl, - username: 'node', - account: snowflakeTestAccount, - token: snowflakeTestToken, - authenticator: 'OAUTH' -}; - -const okta = -{ - accessUrl: accessUrl, - username: snowflakeTestOktaUser, - password: snowflakeTestOktaPass, - account: snowflakeTestAccount, - warehouse: snowflakeTestWarehouse, - database: snowflakeTestDatabase, - schema: snowflakeTestSchema, - role: snowflakeTestRole, - authenticator: snowflakeTestOktaAuth -}; - const privatelink = { accessUrl: accessUrl, @@ -233,16 +112,6 @@ exports.wrongUserName = wrongUserName; exports.wrongPwd = wrongPwd; exports.accessUrl = accessUrl; exports.account = snowflakeTestAccount; -exports.externalBrowser = externalBrowser; -exports.externalBrowserWithShortTimeout = externalBrowserWithShortTimeout; -exports.externalBrowserMismatchUser = externalBrowserMismatchUser; -exports.keypairPrivateKey = keypairPrivateKey; -exports.keypairPathEncrypted = keypairPathEncrypted; -exports.keypairPathUnencrypted = keypairPathUnencrypted; -exports.keypairWrongToken = keypairWrongToken; -exports.oauth = oauth; -exports.oauthMismatchUser = oauthMismatchUser; -exports.okta = okta; exports.privatelink = privatelink; exports.connectionWithProxy = connectionWithProxy; exports.MFA = MFA; diff --git a/test/integration/testConnection.js b/test/integration/testConnection.js index dac08cb6b..addfa2a88 100644 --- a/test/integration/testConnection.js +++ b/test/integration/testConnection.js @@ -199,7 +199,17 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assert.deepEqual(output, []); + const expectedMessagesParts = [ + 'Creating new connection object', + 'Creating Connection[id:', + 'Connection[id:', + 'connection object created successfully' + ]; + + // Check if all output messages match the expected patterns + output.forEach((item, index) => { + assert(item.includes(expectedMessagesParts[index]), `Output message at index ${index} does not match expected pattern. \nReceived message: ${item} \nExpected substring: ${expectedMessagesParts[index]}`); + }); }); it('Invalid "warehouse" parameter', function () { @@ -212,7 +222,7 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assertLogMessage('ERROR', '\'waerhouse\' is an unknown connection parameter. Did you mean \'warehouse\'?', output[0]); + assertLogMessage('ERROR', '\'waerhouse\' is an unknown connection parameter. Did you mean \'warehouse\'?', output[1]); }); it('Valid "database" parameter', function () { @@ -225,7 +235,17 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assert.deepEqual(output, []); + const expectedMessagesParts = [ + 'Creating new connection object', + 'Creating Connection[id:', + 'Connection[id:', + 'connection object created successfully' + ]; + + // Check if all output messages match the expected patterns + output.forEach((item, index) => { + assert(item.includes(expectedMessagesParts[index]), `Output message at index ${index} does not match expected pattern. \nReceived message: ${item} \nExpected substring: ${expectedMessagesParts[index]}`); + }); }); it('Invalid "db" parameter', function () { @@ -238,7 +258,7 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assertLogMessage('ERROR', '\'db\' is an unknown connection parameter. Did you mean \'host\'?', output[0]); + assertLogMessage('ERROR', '\'db\' is an unknown connection parameter. Did you mean \'host\'?', output[1]); }); it('Invalid "database" parameter', function () { @@ -251,7 +271,7 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assertLogMessage('ERROR', '\'datbse\' is an unknown connection parameter. Did you mean \'database\'?', output[0]); + assertLogMessage('ERROR', '\'datbse\' is an unknown connection parameter. Did you mean \'database\'?', output[1]); }); it('Valid "schema" parameter', function () { @@ -264,7 +284,17 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assert.deepEqual(output, []); + const expectedMessagesParts = [ + 'Creating new connection object', + 'Creating Connection[id:', + 'Connection[id:', + 'connection object created successfully' + ]; + + // Check if all output messages match the expected patterns + output.forEach((item, index) => { + assert(item.includes(expectedMessagesParts[index]), `Output message at index ${index} does not match expected pattern. \nReceived message: ${item} \nExpected substring: ${expectedMessagesParts[index]}`); + }); }); it('Invalid "schema" parameter', function () { @@ -277,7 +307,7 @@ describe('Connection test - validate default parameters', function () { validateDefaultParameters: true, }); }); - assertLogMessage('ERROR', '\'shcema\' is an unknown connection parameter. Did you mean \'schema\'?', output[0]); + assertLogMessage('ERROR', '\'shcema\' is an unknown connection parameter. Did you mean \'schema\'?', output[1]); }); }); diff --git a/test/integration/testEncrypt.js b/test/integration/testEncrypt.js new file mode 100644 index 000000000..3aef8914b --- /dev/null +++ b/test/integration/testEncrypt.js @@ -0,0 +1,83 @@ +const assert = require('assert'); +const path = require('path'); +const os = require('os'); +const fs = require('fs/promises'); +const SnowflakeEncryptionUtil = require('../../lib/file_transfer_agent/encrypt_util').EncryptUtil; + + +describe('Test Encryption/Decryption', function () { + const BASE64 = 'base64'; + const UTF8 = 'utf-8'; + let encryptUtil; + + before(function () { + encryptUtil = new SnowflakeEncryptionUtil(); + }); + + it('GCM - Encrypt and decrypt raw data', function () { + const data = 'abc'; + const iv = Buffer.from('ab1234567890'); + const key = Buffer.from('1234567890abcdef'); + + const encryptedData = encryptUtil.encryptGCM(data, key, iv, null); + assert.strictEqual(encryptedData.toString(BASE64), 'iG+lT4o27hkzj3kblYRzQikLVQ=='); + + const decryptedData = encryptUtil.decryptGCM(encryptedData, key, iv, null); + assert.strictEqual(decryptedData.toString(UTF8), data); + }); + + it('GCM - Encrypt raw data based on encryption material', function () { + const data = 'abc'; + + const encryptionMaterial = { + 'queryStageMasterKey': 'YWJjZGVmMTIzNDU2Nzg5MA==', + 'queryId': 'unused', + 'smkId': '123' + }; + + const { dataStream, encryptionMetadata } = encryptUtil.encryptDataGCM(encryptionMaterial, data); + assert.ok(dataStream); + assert.ok(encryptionMetadata); + + const decodedKek = Buffer.from(encryptionMaterial['queryStageMasterKey'], BASE64); + const keyBytes = new Buffer.from(encryptionMetadata.key, BASE64); + const keyIvBytes = new Buffer.from(encryptionMetadata.keyIv, BASE64); + const dataIvBytes = new Buffer.from(encryptionMetadata.iv, BASE64); + const dataAadBytes = new Buffer.from(encryptionMetadata.dataAad, BASE64); + const keyAadBytes = new Buffer.from(encryptionMetadata.keyAad, BASE64); + + const fileKey = encryptUtil.decryptGCM(keyBytes, decodedKek, keyIvBytes, keyAadBytes); + + const decryptedData = encryptUtil.decryptGCM(dataStream, fileKey, dataIvBytes, dataAadBytes); + assert.strictEqual(decryptedData.toString(UTF8), data); + }); + + it('GCM - Encrypt and decrypt file', async function () { + await encryptAndDecryptFile('gcm', async function (encryptionMaterial, inputFilePath) { + const output = await encryptUtil.encryptFileGCM(encryptionMaterial, inputFilePath, os.tmpdir()); + return await encryptUtil.decryptFileGCM(output.encryptionMetadata, encryptionMaterial, output.dataFile, os.tmpdir()); + }); + }); + + it('CBC - Encrypt and decrypt file', async function () { + await encryptAndDecryptFile('cbc', async function (encryptionMaterial, inputFilePath) { + const output = await encryptUtil.encryptFileCBC(encryptionMaterial, inputFilePath, os.tmpdir()); + return await encryptUtil.decryptFileCBC(output.encryptionMetadata, encryptionMaterial, output.dataFile, os.tmpdir()); + }); + }); + + async function encryptAndDecryptFile(encryptionTypeName, encryptAndDecrypt) { + const data = 'abc'; + const inputFilePath = path.join(os.tmpdir(), `${encryptionTypeName}_file_encryption_test`); + await fs.writeFile(inputFilePath, data); + + const encryptionMaterial = { + 'queryStageMasterKey': 'YWJjZGVmMTIzNDU2Nzg5MA==', + 'queryId': 'unused', + 'smkId': '123' + }; + const decryptedFilePath = await encryptAndDecrypt(encryptionMaterial, inputFilePath, os.tmpdir()); + const decryptedContent = await fs.readFile(decryptedFilePath); + assert.strictEqual(decryptedContent.toString('utf-8'), data); + } +}); \ No newline at end of file diff --git a/test/integration/testExecute.js b/test/integration/testExecute.js index f2e515976..48f32357c 100644 --- a/test/integration/testExecute.js +++ b/test/integration/testExecute.js @@ -137,6 +137,54 @@ describe('Execute test', function () { done ); }); + + describe('testDescribeOnly', async function () { + const selectWithDescribeOnly = 'SELECT 1.0::NUMBER(30,2) as C1, 2::NUMBER(38,0) AS C2, \'t3\' AS C3, 4.2::DOUBLE AS C4, \'abcd\'::BINARY(8388608) AS C5, true AS C6'; + const expectedRows = [{ 'C1': 1, 'C2': 2, 'C3': 't3', 'C4': 4.2, 'C5': { 'type': 'Buffer', 'data': [171, 205] }, 'C6': true }]; + const testCases = + [ + { + name: 'describeOnly - true', + describeOnly: true, + expectedRows: [] + }, + { + name: 'describeOnly - false', + describeOnly: false, + expectedRows: expectedRows + }, + { + name: 'describeOnly - undefined', + describeOnly: undefined, + expectedRows: expectedRows + }, + ]; + + const executeQueryAndVerifyResultDependOnDescribeOnly = async (describeOnly, expectedReturnedRows) => { + return new Promise((resolve, reject) => { + connection.execute({ + sqlText: selectWithDescribeOnly, + describeOnly: describeOnly, + complete: (err, stmt, rows) => { + if (err) { + return reject(err); + } + assert.strictEqual(stmt.getColumns().length, 6); + assert.strictEqual(rows.length, expectedReturnedRows.length); + if (rows.length > 0) { + const columnsNamesInMetadata = stmt.getColumns().map(cl => cl.getName()); + const columnsNames = Object.keys(rows[0]); + columnsNames.every((element, index) => assert.strictEqual(element, columnsNamesInMetadata[index])); + } + return resolve(rows); + } + }); + } + ); + }; + + testCases.forEach(testCase => it(testCase.name, () => executeQueryAndVerifyResultDependOnDescribeOnly(testCase.describeOnly, testCase.expectedRows))); + }); }); describe('Execute test - variant', function () { @@ -383,5 +431,70 @@ describe('Execute test - variant', function () { it(testCase.name, createItCallback(testCase, rowAsserts)); }); -}); + describe( 'connection.execute() Resubmitting requests using requestId and different connections', function () { + const createTable = 'create or replace table test_request_id(colA string)'; + let firstConnection; + let secondConnection; + before(async () => { + firstConnection = testUtil.createConnection(); + secondConnection = testUtil.createConnection(); + await testUtil.connectAsync(firstConnection); + await testUtil.connectAsync(secondConnection); + await testUtil.executeCmdAsync(firstConnection, createTable); + }); + + beforeEach(async () => { + await testUtil.executeCmdAsync(firstConnection, 'truncate table if exists test_request_id'); + }); + + after(async () => { + await testUtil.executeCmdAsync(firstConnection, 'drop table if exists test_request_id'); + await testUtil.destroyConnectionAsync(firstConnection); + await testUtil.destroyConnectionAsync(secondConnection); + }); + + it('Do not INSERT twice when the same request id and connection', async () => { + let result; + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');'); + const requestId = result.rowStatement.getRequestId(); + + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, + 'INSERT INTO test_request_id VALUES (\'testValue\');', + { requestId: requestId }); + assert.strictEqual(result.rowStatement.getRequestId(), requestId); + + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id ;'); + assert.strictEqual(result.rows.length, 1); + }); + + it('Execute INSERT for the same request id and different connection', async () => { + let result; + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');'); + const requestId = result.rowStatement.getRequestId(); + + result = await testUtil.executeCmdAsyncWithAdditionalParameters(secondConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');', { requestId: requestId }); + assert.strictEqual(result.rowStatement.getRequestId(), requestId); + + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id ;'); + assert.strictEqual(result.rows.length, 2); + }); + + it('Execute SELECT for the same request id and different data', async () => { + await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');'); + let result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id;'); + assert.strictEqual(result.rows.length, 1); + const requestId = result.rowStatement.getRequestId(); + + await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');'); + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id;', { requestId: requestId }); + assert.strictEqual(result.rows.length, 1); + + result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id ;'); + assert.strictEqual(result.rows.length, 2); + }); + }); + + + +}); diff --git a/test/integration/testManualConnection.js b/test/integration/testManualConnection.js index ff375850f..9fe3ad78d 100644 --- a/test/integration/testManualConnection.js +++ b/test/integration/testManualConnection.js @@ -3,7 +3,6 @@ */ const snowflake = require('./../../lib/snowflake'); -const async = require('async'); const assert = require('assert'); const connOption = require('./connectionOptions'); const testUtil = require('./testUtil'); @@ -13,224 +12,6 @@ const JsonCredentialManager = require('../../lib/authentication/secure_storage/j if (process.env.RUN_MANUAL_TESTS_ONLY === 'true') { describe('Run manual tests', function () { - describe('Connection test - external browser', function () { - it('Simple Connect', function (done) { - const connection = snowflake.createConnection( - connOption.externalBrowser - ); - - connection.connectAsync(function (err, connection) { - try { - assert.ok(connection.isUp(), 'not active'); - testUtil.destroyConnection(connection, function () { - try { - assert.ok(!connection.isUp(), 'not active'); - done(); - } catch (err) { - done(err); - } - }); - } catch (err) { - done(err); - } - }); - }); - - it('Connect - external browser timeout', function (done) { - const connection = snowflake.createConnection( - connOption.externalBrowserWithShortTimeout - ); - - connection.connectAsync(function (err) { - try { - const browserActionTimeout = - connOption.externalBrowserWithShortTimeout.browserActionTimeout; - assert.ok( - err, - `Browser action timed out after ${browserActionTimeout} ms.` - ); - done(); - } catch (err) { - done(err); - } - }); - }); - - it('Mismatched Username', function (done) { - const connection = snowflake.createConnection( - connOption.externalBrowserMismatchUser - ); - connection.connectAsync(function (err) { - try { - assert.ok( - err, - 'Logged in with different user than one on connection string' - ); - assert.equal( - 'The user you were trying to authenticate as differs from the user currently logged in at the IDP.', - err['message'] - ); - done(); - } catch (err) { - done(err); - } - }); - }); - }); - - describe('Connection - ID Token authenticator', function () { - const connectionOption = { ...connOption.externalBrowser, clientStoreTemporaryCredential: true }; - const key = Util.buildCredentialCacheKey(connectionOption.host, connectionOption.username, 'ID_TOKEN'); - const defaultCredentialManager = new JsonCredentialManager(); - let oldToken; - before( async () => { - await defaultCredentialManager.remove(key); - }); - - it('test - obtain the id token from the server and save it on the local storage', function (done) { - const connection = snowflake.createConnection(connectionOption); - connection.connectAsync(function (err) { - try { - assert.ok(!err); - done(); - } catch (err){ - done(err); - } - }); - }); - - it('test - the token is saved in the credential manager correctly', function (done) { - defaultCredentialManager.read(key).then((idToken) => { - try { - oldToken = idToken; - assert.notStrictEqual(idToken, null); - done(); - } catch (err){ - done(err); - } - }); - }); - - - // Web Browser should not be open. - it('test - id token authentication', function (done) { - const idTokenConnection = snowflake.createConnection(connectionOption); - try { - idTokenConnection.connectAsync(function (err) { - assert.ok(!err); - done(); - }); - } catch (err) { - done(err); - } - }); - - // Web Browser should be open. - it('test - id token reauthentication', function (done) { - defaultCredentialManager.write(key, '1234').then(() => { - const wrongTokenConnection = snowflake.createConnection(connectionOption); - wrongTokenConnection.connectAsync(function (err) { - assert.ok(!err); - done(); - }); - }); - }); - - //Compare two idToken. Those two should be different. - it('test - the token is refreshed', function (done) { - oldToken = undefined; - defaultCredentialManager.read(key).then((idToken) => { - try { - assert.notStrictEqual(idToken, oldToken); - done(); - } catch (err) { - done(err); - } - }); - }); - }); - - describe('Connection test - oauth', function () { - it('Simple Connect', function (done) { - const connection = snowflake.createConnection(connOption.oauth); - - async.series([ - function (callback) { - connection.connect(function (err) { - done(err); - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(connection.isUp(), 'not active'); - callback(); - }, - function (callback) { - connection.destroy(function (err) { - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(!connection.isUp(), 'still active'); - callback(); - }, - ]); - }); - - it('Mismatched Username', function (done) { - const connection = snowflake.createConnection( - connOption.oauthMismatchUser - ); - connection.connect(function (err) { - try { - assert.ok( - err, - 'Logged in with different user than one on connection string' - ); - assert.equal( - 'The user you were trying to authenticate as differs from the user tied to the access token.', - err['message'] - ); - done(); - } catch (err) { - done(err); - } - }); - }); - }); - - describe('Connection test - okta', function () { - it('Simple Connect', function (done) { - const connection = snowflake.createConnection(connOption.okta); - - async.series([ - function (callback) { - connection.connectAsync(function (err) { - done(err); - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(connection.isUp(), 'not active'); - callback(); - }, - function (callback) { - connection.destroy(function (err) { - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(!connection.isUp(), 'still active'); - callback(); - }, - ]); - }); - }); - describe('Connection - MFA authenticator with DUO', function () { const connectionOption = connOption.MFA; @@ -319,113 +100,6 @@ if (process.env.RUN_MANUAL_TESTS_ONLY === 'true') { }); }); }); - - describe('Connection test - keypair', function () { - it('Simple Connect - specify private key', function (done) { - const connection = snowflake.createConnection( - connOption.keypairPrivateKey - ); - - async.series([ - function (callback) { - connection.connect(function (err) { - done(err); - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(connection.isUp(), 'not active'); - callback(); - }, - function (callback) { - connection.destroy(function (err) { - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(!connection.isUp(), 'still active'); - callback(); - }, - ]); - }); - - it('Simple Connect - specify encrypted private key path and passphrase', function (done) { - const connection = snowflake.createConnection( - connOption.keypairPathEncrypted - ); - - async.series([ - function (callback) { - connection.connect(function (err) { - done(err); - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(connection.isUp(), 'not active'); - callback(); - }, - function (callback) { - connection.destroy(function (err) { - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(!connection.isUp(), 'still active'); - callback(); - }, - ]); - }); - - it('Simple Connect - specify unencrypted private key path without passphrase', function (done) { - const connection = snowflake.createConnection( - connOption.keypairPathEncrypted - ); - - async.series([ - function (callback) { - connection.connect(function (err) { - done(err); - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(connection.isUp(), 'not active'); - callback(); - }, - function (callback) { - connection.destroy(function (err) { - assert.ok(!err, JSON.stringify(err)); - callback(); - }); - }, - function (callback) { - assert.ok(!connection.isUp(), 'still active'); - callback(); - }, - ]); - }); - - it('Wrong JWT token', function (done) { - const connection = snowflake.createConnection( - connOption.keypairWrongToken - ); - connection.connect(function (err) { - try { - assert.ok(err, 'Incorrect JWT token is passed.'); - assert.equal('JWT token is invalid.', err['message']); - done(); - } catch (err) { - done(err); - } - }); - }); - }); }); describe('keepAlive test', function () { diff --git a/test/integration/testRequestParams.js b/test/integration/testRequestParams.js new file mode 100644 index 000000000..f4eb0fd36 --- /dev/null +++ b/test/integration/testRequestParams.js @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + +const testUtil = require('./testUtil'); +const assert = require('assert'); +const httpInterceptorUtils = require('./test_utils/httpInterceptorUtils'); +const Core = require('../../lib/core'); +const Util = require('../../lib/util'); + +describe('SF service tests', function () { + const selectPiTxt = 'select PI();'; + let interceptors; + let coreInstance; + + before(async function () { + interceptors = new httpInterceptorUtils.Interceptors(); + const HttpClientClassWithInterceptors = httpInterceptorUtils.getHttpClientWithInterceptorsClass(interceptors); + coreInstance = Core({ + httpClientClass: HttpClientClassWithInterceptors, + loggerClass: require('./../../lib/logger/node'), + client: { + version: Util.driverVersion, + name: Util.driverName, + environment: process.versions, + }, + }); + }); + + it('GUID called for all', async function () { + let guidAddedWhenExpected = true; + let totalCallsWithGUIDCount = 0; + let expectedCallsWithGUIDCount = 0; + const pathsExpectedToIncludeGuid = [ + 'session?delete=true&requestId', + 'queries/v1/query-request', + 'session/v1/login-request' + ]; + + function countCallsWithGuid(requestOptions) { + pathsExpectedToIncludeGuid.forEach((value) => { + if (requestOptions.url.includes(value)) { + // Counting is done instead of assertions, because we do not want to interrupt + // the flow of operations inside the HttpClient. Retries and other custom exception handling could be triggered. + if (!testUtil.isGuidInRequestOptions(requestOptions)) { + guidAddedWhenExpected = false; + } + expectedCallsWithGUIDCount++; + } + }); + + if (testUtil.isGuidInRequestOptions(requestOptions)) { + totalCallsWithGUIDCount++; + } + } + interceptors.add('request', httpInterceptorUtils.HOOK_TYPE.FOR_ARGS, countCallsWithGuid); + + const connection = testUtil.createConnection({}, coreInstance); + await testUtil.connectAsync(connection); + await testUtil.executeCmdAsync(connection, selectPiTxt); + + const guidCallsOccurred = totalCallsWithGUIDCount > 0; + assert.strictEqual(guidCallsOccurred, true, 'No GUID calls occurred'); + assert.strictEqual(guidAddedWhenExpected, true, `GUID not found in all requests with paths: ${pathsExpectedToIncludeGuid}`); + assert.strictEqual(expectedCallsWithGUIDCount === totalCallsWithGUIDCount, true, `GUID was added to requests not included in the expected paths: ${pathsExpectedToIncludeGuid}.` + + `Total calls with guid: ${totalCallsWithGUIDCount}. Expected calls with guid: ${expectedCallsWithGUIDCount}.` + ); + + await testUtil.destroyConnectionAsync(connection); + interceptors.clear(); + }); +}); diff --git a/test/integration/testStructuredType.js b/test/integration/testStructuredType.js index e26b8d28c..22b776352 100644 --- a/test/integration/testStructuredType.js +++ b/test/integration/testStructuredType.js @@ -33,8 +33,8 @@ describe('Test Structured types', function () { connection = testUtil.createConnection(); async.series([ function (callback) { - // snowflake.configure({ 'insecureConnect': true }); - // GlobalConfig.setInsecureConnect(true); + // snowflake.configure({ 'disableOCSPChecks': true }); + // GlobalConfig.setDisableOCSPChecks(true); testUtil.connect(connection, callback); }, function (callback) { diff --git a/test/integration/testUtil.js b/test/integration/testUtil.js index 9348a3275..1f082d28c 100644 --- a/test/integration/testUtil.js +++ b/test/integration/testUtil.js @@ -12,19 +12,31 @@ const Logger = require('../../lib/logger'); const path = require('path'); const os = require('os'); -module.exports.createConnection = function (validConnectionOptionsOverride = {}) { - return snowflake.createConnection({ +module.exports.createConnection = function (validConnectionOptionsOverride = {}, coreInstance) { + coreInstance = coreInstance || snowflake; + + return coreInstance.createConnection({ ...connOptions.valid, ...validConnectionOptionsOverride, }); }; -module.exports.createProxyConnection = function () { - return snowflake.createConnection(connOptions.connectionWithProxy); +module.exports.createProxyConnection = function (validConnectionOptionsOverride, coreInstance) { + coreInstance = coreInstance || snowflake; + + return coreInstance.createConnection({ + ...connOptions.connectionWithProxy, + ...validConnectionOptionsOverride + }); }; -module.exports.createConnectionPool = function () { - return snowflake.createPool(connOptions.valid, { max: 10, min: 0, testOnBorrow: true }); +module.exports.createConnectionPool = function (validConnectionOptionsOverride, coreInstance) { + coreInstance = coreInstance || snowflake; + + return coreInstance.createPool({ + ...connOptions.valid, + ...validConnectionOptionsOverride + }, { max: 10, min: 0, testOnBorrow: true }); }; module.exports.connect = function (connection, callback) { @@ -96,6 +108,18 @@ const executeCmdAsync = function (connection, sqlText, binds = undefined) { module.exports.executeCmdAsync = executeCmdAsync; +const executeCmdAsyncWithAdditionalParameters = function (connection, sqlText, additionalParameters) { + return new Promise((resolve, reject) => { + const executeParams = { ...{ + sqlText: sqlText, + complete: (err, rowStatement, rows) => + err ? reject(err) : resolve({ rowStatement: rowStatement, rows: rows }) + }, ...additionalParameters }; + connection.execute(executeParams); + }); +}; + +module.exports.executeCmdAsyncWithAdditionalParameters = executeCmdAsyncWithAdditionalParameters; /** * Drop tables one by one if exist - any connection error is ignored * @param connection Connection @@ -343,6 +367,9 @@ module.exports.assertActiveConnectionDestroyedCorrectlyAsync = async function (c module.exports.assertConnectionInactive(connection); }; - module.exports.normalizeRowObject = normalizeRowObject; -module.exports.normalizeValue = normalizeValue; \ No newline at end of file +module.exports.normalizeValue = normalizeValue; + +module.exports.isGuidInRequestOptions = function (requestOptions) { + return requestOptions.url.includes('request_guid') || 'request_guid' in requestOptions.params; +}; \ No newline at end of file diff --git a/test/integration/test_utils/httpInterceptorUtils.js b/test/integration/test_utils/httpInterceptorUtils.js new file mode 100644 index 000000000..408896938 --- /dev/null +++ b/test/integration/test_utils/httpInterceptorUtils.js @@ -0,0 +1,112 @@ +const Logger = require('../../../lib/logger'); +const { NodeHttpClient } = require('../../../lib/http/node'); +const Util = require('../../../lib/util'); + +const HOOK_TYPE = { + FOR_ARGS: 'args', + FOR_RETURNED_VALUE: 'returned', +}; + +module.exports.HOOK_TYPE = HOOK_TYPE; + +class Interceptor { + constructor(methodName, hookType, callback) { + this.methodName = methodName; + this.hookType = hookType || HOOK_TYPE.FOR_ARGS; + this.callback = callback; + } + + execute(...args) { + this.callback(...args); + } +} + +class Interceptors { + constructor(initialInterceptors) { + this.interceptorsMap = this.createInterceptorsMap(initialInterceptors); + } + + add(methodName, hookType, callback, interceptor = undefined) { + if (!interceptor) { + interceptor = new Interceptor(methodName, hookType, callback); + } + this.interceptorsMap[interceptor.methodName][interceptor.hookType] = interceptor; + } + + get(methodName, hookType) { + return this.interceptorsMap[methodName][hookType]; + } + + intercept(methodName, hookType, ...args) { + // When no interceptor registered - ignores and does not raise any error + try { + return this.get(methodName, hookType)?.execute(...args); + } catch (e) { + throw 'Unable to execute interceptor method in tests. Error: ' + e; + } + } + + clear() { + this.interceptorsMap = this.createInterceptorsMap(); + } + + createInterceptorsMap(initialInterceptors = {}) { + if (initialInterceptors instanceof Interceptors) { + return initialInterceptors.interceptorsMap; + } + // Map creating another map for each accessed key not present in the map + // (analogy - DefaultDict from Python). + return new Proxy(initialInterceptors, { + get: (target, prop) => { + if (prop in target) { + return target[prop]; + } else { + // Create an empty object, store it in target, and return it + const newObj = {}; + target[prop] = newObj; + return newObj; + } + } + }); + } +} + +module.exports.Interceptors = Interceptors; + +function HttpClientWithInterceptors(connectionConfig, initialInterceptors) { + Logger.getInstance().trace('Initializing HttpClientWithInterceptors with Connection Config[%s]', + connectionConfig.describeIdentityAttributes()); + this.interceptors = new Interceptors(initialInterceptors); + NodeHttpClient.apply(this, [connectionConfig]); +} + +Util.inherits(HttpClientWithInterceptors, NodeHttpClient); + + +HttpClientWithInterceptors.prototype.requestAsync = async function (url, options) { + this.interceptors.intercept('requestAsync', HOOK_TYPE.FOR_ARGS, url, options); + const response = await NodeHttpClient.prototype.requestAsync.call(this, url, options); + this.interceptors.intercept('requestAsync', HOOK_TYPE.FOR_RETURNED_VALUE, response); + return response; +}; + +HttpClientWithInterceptors.prototype.request = function (url, options) { + this.interceptors.intercept('request', HOOK_TYPE.FOR_ARGS, url, options); + const response = NodeHttpClient.prototype.request.call(this, url, options); + this.interceptors.intercept('request', HOOK_TYPE.FOR_RETURNED_VALUE, response); + return response; +}; + +// Factory method for HttpClientWithInterceptors to be able to partially initialize class +// with interceptors used in fully instantiated object. +function getHttpClientWithInterceptorsClass(interceptors) { + function HttpClientWithInterceptorsWrapper(connectionConfig) { + HttpClientWithInterceptors.apply(this, [connectionConfig, interceptors]); + } + Util.inherits(HttpClientWithInterceptorsWrapper, HttpClientWithInterceptors); + + return HttpClientWithInterceptorsWrapper; +} + + +module.exports.getHttpClientWithInterceptorsClass = getHttpClientWithInterceptorsClass; diff --git a/test/unit/agent_cache_test.js b/test/unit/agent_cache_test.js new file mode 100644 index 000000000..fabe534dc --- /dev/null +++ b/test/unit/agent_cache_test.js @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + +const GlobalConfig = require('./../../lib/global_config'); +const getProxyAgent = require('./../../lib/http/node').getProxyAgent; +const getAgentCacheSize = require('./../../lib/http/node').getAgentCacheSize; +const assert = require('assert'); + +describe('getProxtAgent', function () { + const mockProxy = new URL('https://user:pass@myproxy.server.com:1234'); + const fakeAccessUrl = new URL('http://fakeaccount.snowflakecomputing.com'); + + const testCases = [ + { + destination: 'test.destination.com', + isNewAgent: true, + keepAlive: true + }, + { + destination: '://test.destination.com', + isNewAgent: true, + keepAlive: true + }, + { + destination: 'This is not a URL', + isNewAgent: true, + keepAlive: true + }, + { + destination: 's4.amazonaws.com', + isNewAgent: true, + keepAlive: true + }, + { + destination: 'http://test.destination.com/login/somewhere', + isNewAgent: false, + keepAlive: true + }, + { + destination: 'http://s4.amazonaws.com', + isNewAgent: false, + keepAlive: true + }, + { + destination: 'https://s4.amazonaws.com', + isNewAgent: true, + keepAlive: false + }, + { + destination: 'https://test.destination.com/login/somewhere', + isNewAgent: true, + keepAlive: false + }, + { + destination: 'https://fakeaccounttesting.snowflakecomputing.com/login/sessionId=something', + isNewAgent: true, + keepAlive: true + }, + { + destination: 'https://fakeaccounttesting.snowflakecomputing.com/other/request', + isNewAgent: false, + keepAlive: true + }, + { + destination: 'http://fakeaccounttesting.snowflakecomputing.com/another/request', + isNewAgent: true, + keepAlive: false + }, + ]; + + it('test http(s) agent cache', () => { + let numofAgent = getAgentCacheSize(); + testCases.forEach(({ destination, isNewAgent, keepAlive }) => { + GlobalConfig.setKeepAlive(keepAlive); + getProxyAgent(mockProxy, fakeAccessUrl, destination); + if (isNewAgent) { + numofAgent++; + } + assert.strictEqual(getAgentCacheSize(), numofAgent); + }); + }); +}); + \ No newline at end of file diff --git a/test/unit/authentication/json_credential_manager_test.js b/test/unit/authentication/json_credential_manager_test.js index c0a882e98..0ffd353eb 100644 --- a/test/unit/authentication/json_credential_manager_test.js +++ b/test/unit/authentication/json_credential_manager_test.js @@ -13,37 +13,29 @@ const credType = 'mock_cred'; const key = Util.buildCredentialCacheKey(host, user, credType); const randomPassword = randomUUID(); const os = require('os'); -const currentNodeVersion = parseInt(process.version.slice(1), 10); -if (!(currentNodeVersion <= 14 && (os.platform() === 'win32'))) { - describe('Json credential manager test', function () { - const credentialManager = new JsonCredentialManager(); - - it('test - initiate credential manager', async function () { - if (await credentialManager.read(key) !== null) { - await credentialManager.remove(key); - } - const savedPassword = await credentialManager.read(key); - - assert.strictEqual(await credentialManager.getTokenDir(), path.join(os.homedir(), 'temporary_credential.json')); - assert.strictEqual(savedPassword, null); - }); - - it('test - write the mock credential with the credential manager', async function () { - await credentialManager.write(key, randomPassword); - const result = await credentialManager.read(key); - assert.strictEqual(randomPassword, result); - }); - - it('test - delete the mock credential with the credential manager', async function () { +describe('Json credential manager test', function () { + const credentialManager = new JsonCredentialManager(); + it('test - initiate credential manager', async function () { + if (await credentialManager.read(key) !== null) { await credentialManager.remove(key); - const result = await credentialManager.read(key); - assert.ok(result === null); - }); - - it('test - token saving location when the user sets credentialCacheDir value', async function () { - const credManager = new JsonCredentialManager(os.tmpdir()); - assert.strictEqual(await credManager.getTokenDir(), path.join(os.tmpdir(), 'temporary_credential.json')); - }); + } + const savedPassword = await credentialManager.read(key); + assert.strictEqual(await credentialManager.getTokenDir(), path.join(os.homedir(), 'temporary_credential.json')); + assert.strictEqual(savedPassword, null); + }); + it('test - write the mock credential with the credential manager', async function () { + await credentialManager.write(key, randomPassword); + const result = await credentialManager.read(key); + assert.strictEqual(randomPassword, result); + }); + it('test - delete the mock credential with the credential manager', async function () { + await credentialManager.remove(key); + const result = await credentialManager.read(key); + assert.ok(result === null); + }); + it('test - token saving location when the user sets credentialCacheDir value', async function () { + const credManager = new JsonCredentialManager(os.tmpdir()); + assert.strictEqual(await credManager.getTokenDir(), path.join(os.tmpdir(), 'temporary_credential.json')); }); -} \ No newline at end of file +}); diff --git a/test/unit/connection/statement_test.js b/test/unit/connection/statement_test.js index eaba007bf..7405b64af 100644 --- a/test/unit/connection/statement_test.js +++ b/test/unit/connection/statement_test.js @@ -205,6 +205,16 @@ describe('Statement.execute()', function () { connectionConfig: null }, errorCode: ErrorCodes.ERR_CONN_EXEC_STMT_MISSING_SQL_TEXT + }, + { + name: 'execute() invalid describeOnly', + options: { + statementOptions: { + sqlText: '', + describeOnly: 1, + }, + }, + errorCode: ErrorCodes.ERR_CONN_EXEC_STMT_INVALID_DESCRIBE_ONLY } ]; @@ -404,4 +414,4 @@ describe('Statement.fetchResult()', function () { testCase = testCases[index]; it(testCase.name, createItCallback(testCase)); } -}); \ No newline at end of file +}); diff --git a/test/unit/file_transfer_agent/azure_test.js b/test/unit/file_transfer_agent/azure_test.js index fea6661e4..f99c53147 100644 --- a/test/unit/file_transfer_agent/azure_test.js +++ b/test/unit/file_transfer_agent/azure_test.js @@ -16,6 +16,13 @@ describe('Azure client', function () { const mockKey = 'mockKey'; const mockIv = 'mockIv'; const mockMatDesc = 'mockMatDesc'; + const noProxyConnectionConfig = { + getProxy: function () { + return null; + }, + accessUrl: 'http://fakeaccount.snowflakecomputing.com', + }; + let Azure = null; let client = null; @@ -106,7 +113,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); }); it('extract bucket name and path', async function () { @@ -131,7 +138,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - Azure = new SnowflakeAzureUtil(client); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); @@ -146,7 +153,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - const Azure = new SnowflakeAzureUtil(client); + const Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.NOT_FOUND_FILE); @@ -161,7 +168,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - Azure = new SnowflakeAzureUtil(client); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); @@ -176,7 +183,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - Azure = new SnowflakeAzureUtil(client); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.ERROR); @@ -192,7 +199,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); await Azure.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.UPLOADED); @@ -212,7 +219,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); await Azure.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); @@ -232,7 +239,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); await Azure.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.NEED_RETRY); diff --git a/test/unit/file_transfer_agent/encrypt_util_test.js b/test/unit/file_transfer_agent/encrypt_util_test.js index 5f7e6a7d2..d8668830c 100644 --- a/test/unit/file_transfer_agent/encrypt_util_test.js +++ b/test/unit/file_transfer_agent/encrypt_util_test.js @@ -93,16 +93,13 @@ describe('Encryption util', function () { } return new createWriteStream; }, - closeSync: function () { - return; + close: function (fd, callback) { + callback(null); } }); mock('temp', { - fileSync: function () { - return { - name: mockTmpName, - fd: 0 - }; + file: function (object, callback) { + callback(null, mockTmpName, 0); }, openSync: function () { return; diff --git a/test/unit/file_transfer_agent/gcs_test.js b/test/unit/file_transfer_agent/gcs_test.js index bd2de06e9..6ce0c84d9 100644 --- a/test/unit/file_transfer_agent/gcs_test.js +++ b/test/unit/file_transfer_agent/gcs_test.js @@ -64,6 +64,82 @@ describe('GCS client', function () { GCS = new SnowflakeGCSUtil(httpclient, filestream); }); + describe('GCS client endpoint testing', async function () { + const testCases = [ + { + name: 'when the useRegionalURL is only enabled', + stageInfo: { + endPoint: null, + useRegionalUrl: true, + region: 'mockLocation', + }, + result: 'https://storage.mocklocation.rep.googleapis.com' + }, + { + name: 'when the region is me-central2', + stageInfo: { + endPoint: null, + useRegionalUrl: false, + region: 'me-central2' + }, + result: 'https://storage.me-central2.rep.googleapis.com' + }, + { + name: 'when the region is me-central2 (mixed case)', + stageInfo: { + endPoint: null, + useRegionalUrl: false, + region: 'ME-cEntRal2' + }, + result: 'https://storage.me-central2.rep.googleapis.com' + }, + { + name: 'when the region is me-central2 (uppercase)', + stageInfo: { + endPoint: null, + useRegionalUrl: false, + region: 'ME-CENTRAL2' + }, + result: 'https://storage.me-central2.rep.googleapis.com' + }, + { + name: 'when the endPoint is specified', + stageInfo: { + endPoint: 'https://storage.specialEndPoint.rep.googleapis.com', + useRegionalUrl: false, + region: 'ME-cEntRal1' + }, + result: 'https://storage.specialEndPoint.rep.googleapis.com' + }, + { + name: 'when both the endPoint and the useRegionalUrl are specified', + stageInfo: { + endPoint: 'https://storage.specialEndPoint.rep.googleapis.com', + useRegionalUrl: true, + region: 'ME-cEntRal1' + }, + result: 'https://storage.specialEndPoint.rep.googleapis.com' + }, + { + name: 'when both the endPoint is specified and the region is me-central2', + stageInfo: { + endPoint: 'https://storage.specialEndPoint.rep.googleapis.com', + useRegionalUrl: true, + region: 'ME-CENTRAL2' + }, + result: 'https://storage.specialEndPoint.rep.googleapis.com' + }, + ]; + + testCases.forEach(({ name, stageInfo, result }) => { + it(name, () => { + const client = GCS.createClient({ ...stageInfo, ...meta.stageInfo, creds: { GCS_ACCESS_TOKEN: 'mockToken' } }); + assert.strictEqual(client.gcsClient.apiEndpoint, result); + } ); + + }); + }); + it('extract bucket name and path', async function () { const GCS = new SnowflakeGCSUtil(); diff --git a/test/unit/file_transfer_agent/s3_test.js b/test/unit/file_transfer_agent/s3_test.js index 244a84fbe..64bf0bf07 100644 --- a/test/unit/file_transfer_agent/s3_test.js +++ b/test/unit/file_transfer_agent/s3_test.js @@ -44,7 +44,7 @@ describe('S3 client', function () { before(function () { mock('s3', { - S3: function () { + S3: function (config) { function S3() { this.getObject = function () { function getObject() { @@ -57,6 +57,8 @@ describe('S3 client', function () { return new getObject; }; + + this.config = config; this.putObject = function () { function putObject() { this.then = function (callback) { @@ -82,6 +84,59 @@ describe('S3 client', function () { AWS = new SnowflakeS3Util(noProxyConnectionConfig, s3, filesystem); }); + describe('AWS client endpoint testing', async function () { + const originalStageInfo = meta.stageInfo; + const testCases = [ + { + name: 'when useS3RegionalURL is only enabled', + stageInfo: { + ...originalStageInfo, + useS3RegionalUrl: true, + endPoint: null, + }, + result: null + }, + { + name: 'when useS3RegionalURL and is enabled and domain starts with cn', + stageInfo: { + ...originalStageInfo, + useS3RegionalUrl: true, + endPoint: null, + region: 'cn-mockLocation' + }, + result: 'https://s3.cn-mockLocation.amazonaws.com.cn' + }, + { + name: 'when endPoint is enabled', + stageInfo: { + ...originalStageInfo, + endPoint: 's3.endpoint', + useS3RegionalUrl: false + }, + result: 'https://s3.endpoint' + }, + { + name: 'when both endPoint and useS3PReiongalUrl is valid', + stageInfo: { + ...originalStageInfo, + endPoint: 's3.endpoint', + useS3RegionalUrl: true, + + }, + result: 'https://s3.endpoint' + }, + ]; + + testCases.forEach(({ name, stageInfo, result }) => { + it(name, () => { + const client = AWS.createClient(stageInfo); + assert.strictEqual(client.config.endpoint, result); + } ); + + }); + }); + + it('extract bucket name and path', async function () { let result = extractBucketNameAndPath('sfc-eng-regression/test_sub_dir/'); assert.strictEqual(result.bucketName, 'sfc-eng-regression'); diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index afd2f2a93..73546a7d3 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -36,14 +36,13 @@ MockHttpClient.prototype.request = function (request) { this._mapRequestToOutput = buildRequestToOutputMap(buildRequestOutputMappings(this._clientInfo)); } + removeParamFromRequestUrl(request, 'request_guid'); + removeParamFromRequestParams(request, 'request_guid'); // Closing a connection includes a requestID as a query parameter in the url // Example: http://fake504.snowflakecomputing.com/session?delete=true&requestId=a40454c6-c3bb-4824-b0f3-bae041d9d6a2 if (request.url.includes('session?delete=true') || request.url.includes('session/heartbeat?requestId=')) { - // Offset for the query character preceding the 'requestId=' string in URL (either '?' or '&') - const PRECEDING_QUERY_CHAR_OFFSET = 1; - // Remove the requestID query parameter for the mock HTTP client - request.url = request.url.substring(0, request.url.indexOf('requestId=') - PRECEDING_QUERY_CHAR_OFFSET); + removeParamFromRequestUrl(request, 'requestId'); } // get the output of the specified request from the map @@ -84,14 +83,13 @@ MockHttpClient.prototype.requestAsync = function (request) { this._mapRequestToOutput = buildRequestToOutputMap(buildRequestOutputMappings(this._clientInfo)); } + removeParamFromRequestUrl(request, 'request_guid'); + removeParamFromRequestParams(request, 'request_guid'); // Closing a connection includes a requestID as a query parameter in the url // Example: http://fake504.snowflakecomputing.com/session?delete=true&requestId=a40454c6-c3bb-4824-b0f3-bae041d9d6a2 if (request.url.includes('session?delete=true') || request.url.includes('session/heartbeat?requestId=')) { - // Offset for the query character preceding the 'requestId=' string in URL (either '?' or '&') - const PRECEDING_QUERY_CHAR_OFFSET = 1; - // Remove the requestID query parameter for the mock HTTP client - request.url = request.url.substring(0, request.url.indexOf('requestId=') - PRECEDING_QUERY_CHAR_OFFSET); + removeParamFromRequestUrl(request, 'requestId'); } // get the output of the specified request from the map @@ -175,6 +173,35 @@ function createSortedClone(target) { return sortedClone; } +function removeParamFromRequestUrl(request, paramName) { + try { + // Use the URL constructor to parse the URL + const urlObj = new URL(request.url); + urlObj.searchParams.delete(paramName); + request.url = urlObj.toString(); + } catch (error) { + // Handle invalid URLs or other errors + throw `Invalid URL: ${request.url} Error: ${error}`; + } +} + +/** + * Removes a parameter from the params object of a request. + * If the params object becomes empty after the deletion, removes the params property entirely. + */ +function removeParamFromRequestParams(request, paramName) { + if (request && request.params && typeof request.params === 'object') { + // Delete the specified parameter + delete request.params[paramName]; + + // Check if params is now empty + if (Object.keys(request.params).length === 0) { + // Remove the entire params property + delete request.params; + } + } +} + /** * Returns an array of objects, each of which has a request and output property. * diff --git a/test/unit/ocsp/test_unit_ocsp_mode.js b/test/unit/ocsp/test_unit_ocsp_mode.js index 219aa63fe..a961b83cc 100644 --- a/test/unit/ocsp/test_unit_ocsp_mode.js +++ b/test/unit/ocsp/test_unit_ocsp_mode.js @@ -9,13 +9,13 @@ const assert = require('assert'); describe('OCSP mode', function () { it('getOcspMode', function (done) { // insecure mode - GlobalConfig.setInsecureConnect(true); + GlobalConfig.setDisableOCSPChecks(true); assert.equal(GlobalConfig.getOcspMode(), GlobalConfig.ocspModes.INSECURE); // insecure mode + Fail open GlobalConfig.setOcspFailOpen(true); assert.equal(GlobalConfig.getOcspMode(), GlobalConfig.ocspModes.INSECURE); - GlobalConfig.setInsecureConnect(false); + GlobalConfig.setDisableOCSPChecks(false); assert.equal(GlobalConfig.getOcspMode(), GlobalConfig.ocspModes.FAIL_OPEN); GlobalConfig.setOcspFailOpen(false); diff --git a/test/unit/proxy_util_test.js b/test/unit/proxy_util_test.js new file mode 100644 index 000000000..efc72371b --- /dev/null +++ b/test/unit/proxy_util_test.js @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + +const ProxyUtil = require('./../../lib/proxy_util'); +const Util = require('./../../lib/util'); +const GlobalConfig = require('../../lib/global_config'); + +const assert = require('assert'); + +describe('ProxyUtil Test - removing http or https from string', () => { + const hostAndPortDone = 'my.pro.xy:8080'; + const ipAndPortDone = '10.20.30.40:8080'; + const somethingEntirelyDifferentDone = 'something ENTIRELY different'; + + [ + { name: 'remove http from url', text: 'http://my.pro.xy:8080', shouldMatch: hostAndPortDone }, + { name: 'remove https from url', text: 'https://my.pro.xy:8080', shouldMatch: hostAndPortDone }, + { name: 'remove http from ip and port', text: 'http://10.20.30.40:8080', shouldMatch: ipAndPortDone }, + { name: 'remove https from ip and port', text: 'https://10.20.30.40:8080', shouldMatch: ipAndPortDone }, + { name: 'dont remove http(s) from hostname and port', text: 'my.pro.xy:8080', shouldMatch: hostAndPortDone }, + { name: 'dont remove http(s) from ip and port', text: '10.20.30.40:8080', shouldMatch: ipAndPortDone }, + { name: 'dont remove http(s) from simple string', text: somethingEntirelyDifferentDone, shouldMatch: somethingEntirelyDifferentDone } + ].forEach(({ name, text, shouldMatch }) => { + it(`${name}`, () => { + assert.deepEqual(ProxyUtil.removeScheme(text), shouldMatch); + }); + }); +}); + +describe('ProxyUtil Test - detecting PROXY envvars and compare with the agent proxy settings', () => { + [ + { + name: 'detect http_proxy envvar, no agent proxy', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: '', + agentOptions: { 'keepalive': true }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: .' + }, { + name: 'detect HTTPS_PROXY envvar, no agent proxy', + isWarn: false, + httpproxy: '', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, no agent proxy', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' + }, { + name: 'detect http_proxy envvar, agent proxy set to an unauthenticated proxy, same as the envvar', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: '', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, same as the envvar', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://10.20.30.40:8080', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://10.20.30.40:8080', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass' }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080 user=PRX' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar, with the protocol set', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://10.20.30.40:8080', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass', 'protocol': 'http' }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: protocol=http proxy=10.20.30.40:8080 user=PRX' + }, { + // now some WARN level messages + name: 'detect HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, different from the envvar', + isWarn: true, + httpproxy: '', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, different from each other, agent proxy set to an unauthenticated proxy, different from the envvars', + isWarn: true, + httpproxy: '169.254.169.254:8080', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' Using both the HTTP_PROXY (169.254.169.254:8080) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + } + ].forEach(({ name, isWarn, httpproxy, HTTPSPROXY, agentOptions, shouldLog }) => { + it(`${name}`, () => { + process.env.HTTP_PROXY = httpproxy; + process.env.HTTPS_PROXY = HTTPSPROXY; + + const compareAndLogEnvAndAgentProxies = ProxyUtil.getCompareAndLogEnvAndAgentProxies(agentOptions); + if (!isWarn) { + assert.deepEqual(compareAndLogEnvAndAgentProxies.messages, shouldLog, 'expected log message does not match!'); + } else { + assert.deepEqual(compareAndLogEnvAndAgentProxies.warnings, shouldLog, 'expected warning message does not match!'); + } + }); + }); +}); + +describe('getProxyEnv function test ', function () { + let originalHttpProxy = null; + let originalHttpsProxy = null; + let originalNoProxy = null; + + before(() => { + originalHttpProxy = process.env.HTTP_PROXY; + originalHttpsProxy = process.env.HTTPS_PROXY; + originalNoProxy = process.env.NO_PROXY; + }); + + beforeEach(() => { + delete process.env.HTTP_PROXY; + delete process.env.HTTPS_PROXY; + delete process.env.NO_PROXY; + }); + + after(() => { + originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; + originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; + originalNoProxy ? process.env.NO_PROXY = originalNoProxy : delete process.env.NO_PROXY; + }); + + const testCases = [ + { + name: 'HTTP PROXY without authentication and schema', + isHttps: false, + httpProxy: 'proxy.example.com:8080', + httpsProxy: undefined, + noProxy: '*.amazonaws.com', + result: { + host: 'proxy.example.com', + port: 8080, + protocol: 'http:', + noProxy: '*.amazonaws.com' + } + }, + { + name: 'HTTP PROXY with authentication', + isHttps: false, + httpProxy: 'http://hello:world@proxy.example.com:8080', + httpsProxy: undefined, + noProxy: '*.amazonaws.com,*.my_company.com', + result: { + host: 'proxy.example.com', + user: 'hello', + password: 'world', + port: 8080, + protocol: 'http:', + noProxy: '*.amazonaws.com|*.my_company.com' + } + }, + { + name: 'HTTPS PROXY with authentication without NO proxy', + isHttps: true, + httpsProxy: 'https://user:pass@myproxy.server.com:1234', + result: { + host: 'myproxy.server.com', + user: 'user', + password: 'pass', + port: 1234, + protocol: 'https:', + noProxy: undefined, + }, + }, + { + name: 'HTTPS PROXY with authentication without NO proxy No schema', + isHttps: true, + noProxy: '*.amazonaws.com,*.my_company.com,*.test.com', + httpsProxy: 'myproxy.server.com:1234', + result: { + host: 'myproxy.server.com', + port: 1234, + protocol: 'http:', + noProxy: '*.amazonaws.com|*.my_company.com|*.test.com', + }, + }, + ]; + + testCases.forEach(({ name, isHttps, httpsProxy, httpProxy, noProxy, result }) => { + it(name, function (){ + + if (httpProxy){ + process.env.HTTP_PROXY = httpProxy; + } + if (httpsProxy) { + process.env.HTTPS_PROXY = httpsProxy; + } + if (noProxy) { + process.env.NO_PROXY = noProxy; + } + const proxy = ProxyUtil.getProxyFromEnv(isHttps); + const keys = Object.keys(result); + assert.strictEqual(keys.length, Object.keys(proxy).length); + + for (const key of keys) { + assert.strictEqual(proxy[key], result[key]); + } + }); + }); +}); + +describe('getNoProxyEnv function Test', function () { + let original = null; + + before( function (){ + original = process.env.NO_PROXY; + process.env.NO_PROXY = '*.amazonaws.com,*.my_company.com'; + }); + + after(() => { + process.env.NO_PROXY = original; + }); + + it('test noProxy conversion', function (){ + assert.strictEqual(ProxyUtil.getNoProxyEnv(), '*.amazonaws.com|*.my_company.com'); + }); +}); + +describe('Proxy Util for Azure', function () { + let originalhttpProxy = null; + let originalhttpsProxy = null; + let originalnoProxy = null; + let originalHttpProxy = null; + let originalHttpsProxy = null; + let originalNoProxy = null; + + before(() => { + GlobalConfig.setEnvProxy(false); + originalHttpProxy = process.env.HTTP_PROXY; + originalHttpsProxy = process.env.HTTPS_PROXY; + originalNoProxy = process.env.NO_PROXY; + if (!Util.isWindows()) { + originalhttpProxy = process.env.http_proxy; + originalhttpsProxy = process.env.https_proxy; + originalnoProxy = process.env.no_proxy; + } + }); + + after(() => { + GlobalConfig.setEnvProxy(true); + originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; + originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; + originalNoProxy ? process.env.NO_PROXY = originalNoProxy : delete process.env.NO_PROXY; + if (!Util.isWindows()) { + originalhttpProxy ? process.env['http_proxy'] = originalhttpProxy : delete process.env.http_proxy; + originalhttpsProxy ? process.env['https_proxy'] = originalhttpsProxy : delete process.env.https_proxy; + originalnoProxy ? process.env['no_proxy'] = originalnoProxy : delete process.env.no_proxy; + } + }); + + + it('test hide and restore environment proxy', function () { + const testCases = + { + httpProxy: 'https://user:pass@myproxy.server.com:1234', + httpsProxy: 'https://user:pass@myproxy.server.com:1234', + noProxy: '*.amazonaws.com,*.my_company.com', + HttpProxy: 'https://user:pass@myproxy2.server.com:1234', + HttpsProxy: 'https://user:pass@myproxy2.server.com:1234', + NoProxy: '*.amazonaws2.com,*.my_company2.com', + }; + + process.env.HTTP_PROXY = testCases.HttpProxy; + process.env.HTTPS_PROXY = testCases.HttpsProxy; + process.env.NO_PROXY = testCases.NoProxy; + if (!Util.isWindows()) { + process.env['http_proxy'] = testCases.httpProxy; + process.env['https_proxy'] = testCases.httpsProxy; + process.env['no_proxy'] = testCases.noProxy; + } + + ProxyUtil.hideEnvironmentProxy(); + assert.strictEqual(process.env.HTTP_PROXY, undefined); + assert.strictEqual(process.env.HTTPS_PROXY, undefined); + assert.strictEqual(process.env.NO_PROXY, undefined); + if (!Util.isWindows()) { + assert.strictEqual(process.env['http_proxy'], undefined); + assert.strictEqual(process.env['https_proxy'], undefined); + assert.strictEqual(process.env['no_proxy'], undefined); + } + + ProxyUtil.restoreEnvironmentProxy(); + assert.strictEqual(process.env.HTTP_PROXY, testCases.HttpProxy); + assert.strictEqual(process.env.HTTPS_PROXY, testCases.HttpsProxy); + assert.strictEqual(process.env.NO_PROXY, testCases.NoProxy); + if (!Util.isWindows()) { + assert.strictEqual(process.env.http_proxy, testCases.httpProxy); + assert.strictEqual(process.env.https_proxy, testCases.httpsProxy); + assert.strictEqual(process.env.no_proxy, testCases.noProxy); + } + }); +}); \ No newline at end of file diff --git a/test/unit/secret_detector_test.js b/test/unit/secret_detector_test.js index 472f025a6..ee6a188c3 100644 --- a/test/unit/secret_detector_test.js +++ b/test/unit/secret_detector_test.js @@ -297,4 +297,95 @@ describe('Secret Detector', function () { assert.strictEqual(err.toString(), 'Error: The customPatterns object must have equal length for both \'regex\' and \'mask\''); } }); + + it('test - passcode masking', async function () { + const fourDigitPasscode = 'passcode=1234'; + let result = SecretDetector.maskSecrets(fourDigitPasscode); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'passcode=****'); + assert.strictEqual(result.errstr, null); + + const sixDigitPasscode = 'passcode=654321'; + result = SecretDetector.maskSecrets(sixDigitPasscode); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'passcode=****'); + assert.strictEqual(result.errstr, null); + + const passcodeWithColon = 'otp: 987654'; + result = SecretDetector.maskSecrets(passcodeWithColon); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'otp:****'); + assert.strictEqual(result.errstr, null); + + const pinWithSpaces = 'pin = 4321'; + result = SecretDetector.maskSecrets(pinWithSpaces); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'pin=****'); + assert.strictEqual(result.errstr, null); + + const otacWithSpaces = 'otac= 4321'; + result = SecretDetector.maskSecrets(otacWithSpaces); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'otac=****'); + assert.strictEqual(result.errstr, null); + }); + + it('test - url token masking', async function () { + const TEST_TOKEN_VALUE = 'ETMsDgAAAZNi6aPlABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwEAABAAEExQLlI3h9PIi9TcCRVdwlEAAABQLsgIQdJ0%2B8eQhDMjViFuY5v03Daxt235tNHYVLNoIqM70yLw4zyVdPlkEi208dS88lSqRvPdgQ/RACU7u%2Bn9gWLiTZ79dkZwl4zQactAKJgAFCUrvbxA2tnUP%2BsX6nPBNBzVWnK5'; + const TEST_TOKEN_VERSION_PREFIX = 'ver:1'; + const TEST_TOKEN_HINT_PREFIX = 'hint:1036'; + const TEST_TOKEN_PREFIX = TEST_TOKEN_VERSION_PREFIX + '-' + TEST_TOKEN_HINT_PREFIX + '-'; + + const tokenWithVersionAndHint = 'token=' + TEST_TOKEN_PREFIX + TEST_TOKEN_VALUE; + let result = SecretDetector.maskSecrets(tokenWithVersionAndHint); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token=' + '****'); + assert.strictEqual(result.errstr, null); + + const tokenWithVersionAndHintAndManyEqualsSigns = 'token=====' + TEST_TOKEN_PREFIX + TEST_TOKEN_VALUE; + result = SecretDetector.maskSecrets(tokenWithVersionAndHintAndManyEqualsSigns); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token=====' + '****'); + assert.strictEqual(result.errstr, null); + + const tokenWithVersionAndHintAndColon = 'token:' + TEST_TOKEN_PREFIX + TEST_TOKEN_VALUE; + result = SecretDetector.maskSecrets(tokenWithVersionAndHintAndColon); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token:' + '****'); + assert.strictEqual(result.errstr, null); + + + const TEST_NEXT_PARAMETER_NOT_TO_BE_MASKED = 'jobID=123fdas4-2133212-12'; + const tokenWithVersionAndHintAndAnotherParameterToIgnore = 'token=' + TEST_TOKEN_PREFIX + TEST_TOKEN_VALUE + '&' + TEST_NEXT_PARAMETER_NOT_TO_BE_MASKED; + result = SecretDetector.maskSecrets(tokenWithVersionAndHintAndAnotherParameterToIgnore); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token=' + '****' + '&' + TEST_NEXT_PARAMETER_NOT_TO_BE_MASKED); + assert.strictEqual(result.errstr, null); + + + const tokenWithVersionAndHintAndManySpaces = 'token = ' + TEST_TOKEN_PREFIX + TEST_TOKEN_VALUE; + result = SecretDetector.maskSecrets(tokenWithVersionAndHintAndManySpaces); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token = ' + '****'); + assert.strictEqual(result.errstr, null); + + + const tokenWithVersion = 'token=' + TEST_TOKEN_VERSION_PREFIX + '-' + TEST_TOKEN_VALUE; + result = SecretDetector.maskSecrets(tokenWithVersion); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token=' + '****'); + assert.strictEqual(result.errstr, null); + + const tokenWithHint = 'token=' + TEST_TOKEN_HINT_PREFIX + '-' + TEST_TOKEN_VALUE; + result = SecretDetector.maskSecrets(tokenWithHint); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token=' + '****'); + assert.strictEqual(result.errstr, null); + + const longToken = 'token=' + TEST_TOKEN_VALUE; + result = SecretDetector.maskSecrets(longToken); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'token=****'); + assert.strictEqual(result.errstr, null); + }); }); diff --git a/test/unit/snowflake_config_test.js b/test/unit/snowflake_config_test.js index 7cc9405af..1f10fe3c4 100644 --- a/test/unit/snowflake_config_test.js +++ b/test/unit/snowflake_config_test.js @@ -15,7 +15,7 @@ describe('Snowflake Configure Tests', function () { before(function () { originalConfig = { logLevel: Logger.getInstance().getLevelTag(), - insecureConnect: GlobalConfig.isInsecureConnect(), + disableOCSPChecks: GlobalConfig.isOCSPChecksDisabled(), ocspFailOpen: GlobalConfig.getOcspFailOpen(), keepAlive: GlobalConfig.getKeepAlive(), jsonColumnVariantParser: GlobalConfig.jsonColumnVariantParser, @@ -36,9 +36,9 @@ describe('Snowflake Configure Tests', function () { errorCode: ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_LOG_LEVEL }, { - name: 'invalid insecureConnect', - options: { insecureConnect: 'unsupported' }, - errorCode: ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_INSECURE_CONNECT + name: 'invalid disableOCSPChecks', + options: { disableOCSPChecks: 'unsupported' }, + errorCode: ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_DISABLE_OCSP_CHECKS }, { name: 'invalid ocspMode', @@ -134,17 +134,17 @@ describe('Snowflake Configure Tests', function () { } }, { - name: 'insecureConnect false', + name: 'disableOCSPChecks false', options: { - insecureConnect: false + disableOCSPChecks: false } }, { - name: 'insecureConnect true', + name: 'disableOCSPChecks true', options: { - insecureConnect: true + disableOCSPChecks: true } }, { @@ -213,8 +213,8 @@ describe('Snowflake Configure Tests', function () { let val; if (key === 'logLevel') { val = Logger.getInstance().getLevelTag(); - } else if (key === 'insecureConnect') { - val = GlobalConfig.isInsecureConnect(); + } else if (key === 'disableOCSPChecks') { + val = GlobalConfig.isOCSPChecksDisabled(); } else if (key === 'ocspFailOpen') { val = GlobalConfig.getOcspFailOpen(); } else if (key === 'keepAlive') { diff --git a/test/unit/snowflake_test.js b/test/unit/snowflake_test.js index 0dc44ba64..c14896e4e 100644 --- a/test/unit/snowflake_test.js +++ b/test/unit/snowflake_test.js @@ -815,9 +815,10 @@ describe('connection.execute() statement failure', function () { assert.strictEqual(statement.getColumns(), undefined); assert.strictEqual(statement.getNumRows(), undefined); assert.strictEqual(statement.getSessionState(), undefined); - - assert.strictEqual(statement.getStatementId(), null); - assert.strictEqual(statement.getQueryId(), null); + assert.ok(Util.exists(statement.getStatementId())); + assert.ok(Util.isString(statement.getStatementId())); + assert.ok(Util.exists(statement.getQueryId())); + assert.ok(Util.isString(statement.getQueryId())); callback(); } diff --git a/test/unit/util_test.js b/test/unit/util_test.js index feaa48541..967c424ad 100644 --- a/test/unit/util_test.js +++ b/test/unit/util_test.js @@ -910,262 +910,160 @@ describe('Util', function () { }); }); - describe('Util Test - removing http or https from string', () => { - const hostAndPortDone = 'my.pro.xy:8080'; - const ipAndPortDone = '10.20.30.40:8080'; - const somethingEntirelyDifferentDone = 'something ENTIRELY different'; + describe('Util test - custom credential manager util functions', function () { + const mockUser = 'mockUser'; + const mockHost = 'mockHost'; + const mockCred = 'mockCred'; - [ - { name: 'remove http from url', text: 'http://my.pro.xy:8080', shouldMatch: hostAndPortDone }, - { name: 'remove https from url', text: 'https://my.pro.xy:8080', shouldMatch: hostAndPortDone }, - { name: 'remove http from ip and port', text: 'http://10.20.30.40:8080', shouldMatch: ipAndPortDone }, - { name: 'remove https from ip and port', text: 'https://10.20.30.40:8080', shouldMatch: ipAndPortDone }, - { name: 'dont remove http(s) from hostname and port', text: 'my.pro.xy:8080', shouldMatch: hostAndPortDone }, - { name: 'dont remove http(s) from ip and port', text: '10.20.30.40:8080', shouldMatch: ipAndPortDone }, - { name: 'dont remove http(s) from simple string', text: somethingEntirelyDifferentDone, shouldMatch: somethingEntirelyDifferentDone } - ].forEach(({ name, text, shouldMatch }) => { - it(`${name}`, () => { - assert.deepEqual(Util.removeScheme(text), shouldMatch); - }); - }); - }); - - describe('Util Test - detecting PROXY envvars and compare with the agent proxy settings', () => { - [ - { - name: 'detect http_proxy envvar, no agent proxy', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: '', - agentOptions: { 'keepalive': true }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: .' - }, { - name: 'detect HTTPS_PROXY envvar, no agent proxy', - isWarn: false, - httpproxy: '', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, no agent proxy', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' - }, { - name: 'detect http_proxy envvar, agent proxy set to an unauthenticated proxy, same as the envvar', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: '', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, same as the envvar', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://10.20.30.40:8080', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://10.20.30.40:8080', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass' }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080 user=PRX' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar, with the protocol set', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://10.20.30.40:8080', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass', 'protocol': 'http' }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: protocol=http proxy=10.20.30.40:8080 user=PRX' - }, { - // now some WARN level messages - name: 'detect HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, different from the envvar', - isWarn: true, - httpproxy: '', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, different from each other, agent proxy set to an unauthenticated proxy, different from the envvars', - isWarn: true, - httpproxy: '169.254.169.254:8080', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' Using both the HTTP_PROXY (169.254.169.254:8080) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' - } - ].forEach(({ name, isWarn, httpproxy, HTTPSPROXY, agentOptions, shouldLog }) => { - it(`${name}`, () => { - process.env.HTTP_PROXY = httpproxy; - process.env.HTTPS_PROXY = HTTPSPROXY; - - const compareAndLogEnvAndAgentProxies = Util.getCompareAndLogEnvAndAgentProxies(agentOptions); - if (!isWarn) { - assert.deepEqual(compareAndLogEnvAndAgentProxies.messages, shouldLog, 'expected log message does not match!'); - } else { - assert.deepEqual(compareAndLogEnvAndAgentProxies.warnings, shouldLog, 'expected warning message does not match!'); - } - }); - }); - - describe('Util test - custom credential manager util functions', function () { - const mockUser = 'mockUser'; - const mockHost = 'mockHost'; - const mockCred = 'mockCred'; - - describe('test function build credential key', function () { - const testCases = [ - { - name: 'when all the parameters are null', - user: null, - host: null, - cred: null, - result: null - }, - { - name: 'when two parameters are null or undefined', - user: mockUser, - host: null, - cred: undefined, - result: null - }, - { - name: 'when one parameter is null', - user: mockUser, - host: mockHost, - cred: undefined, - result: null - }, - { - name: 'when one parameter is undefined', - user: mockUser, - host: undefined, - cred: mockCred, - result: null - }, - { - name: 'when all the parameters are valid', - user: mockUser, - host: mockHost, - cred: mockCred, - result: '{mockHost}:{mockUser}:{SF_NODE_JS_DRIVER}:{mockCred}}' - }, - ]; - testCases.forEach((name, user, host, cred, result) => { - it(`${name}`, function () { - if (!result) { - assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), null); - } else { - assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), result); - } - }); - }); - }); - }); - - describe('test valid custom credential manager', function () { - - function sampleManager() { - this.read = function () {}; - - this.write = function () {}; - - this.remove = function () {}; - } - + describe('test function build credential key', function () { const testCases = [ { - name: 'credential manager is an int', - credentialManager: 123, - result: false, + name: 'when all the parameters are null', + user: null, + host: null, + cred: null, + result: null }, { - name: 'credential manager is a string', - credentialManager: 'credential manager', - result: false, + name: 'when two parameters are null or undefined', + user: mockUser, + host: null, + cred: undefined, + result: null }, { - name: 'credential manager is an array', - credentialManager: ['write', 'read', 'remove'], - result: false, + name: 'when one parameter is null', + user: mockUser, + host: mockHost, + cred: undefined, + result: null }, { - name: 'credential manager is an empty obejct', - credentialManager: {}, - result: false, + name: 'when one parameter is undefined', + user: mockUser, + host: undefined, + cred: mockCred, + result: null }, { - name: 'credential manager has property, but invalid types', - credentialManager: { - read: 'read', - write: 1234, - remove: [] - }, - result: false, - }, - { - name: 'credential manager has property, but invalid types', - credentialManager: { - read: 'read', - write: 1234, - remove: [] - }, - result: false, - }, - { - name: 'credential manager has two valid properties, but miss one', - credentialManager: { - read: function () { - - }, - write: function () { - - } - }, - result: false, - }, - { - name: 'credential manager has two valid properties, but miss one', - credentialManager: new sampleManager(), - result: true, + name: 'when all the parameters are valid', + user: mockUser, + host: mockHost, + cred: mockCred, + result: '{mockHost}:{mockUser}:{SF_NODE_JS_DRIVER}:{mockCred}}' }, ]; - - for (const { name, credentialManager, result } of testCases) { - it(name, function () { - assert.strictEqual(Util.checkValidCustomCredentialManager(credentialManager), result); + testCases.forEach((name, user, host, cred, result) => { + it(`${name}`, function () { + if (!result) { + assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), null); + } else { + assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), result); + } }); - } + }); }); + }); - describe('checkParametersDefined function Test', function () { - const testCases = [ - { - name: 'all the parameters are null or undefined', - parameters: [null, undefined, null, null], - result: false + describe('test valid custom credential manager', function () { + + function sampleManager() { + this.read = function () {}; + + this.write = function () {}; + + this.remove = function () {}; + } + + const testCases = [ + { + name: 'credential manager is an int', + credentialManager: 123, + result: false, + }, + { + name: 'credential manager is a string', + credentialManager: 'credential manager', + result: false, + }, + { + name: 'credential manager is an array', + credentialManager: ['write', 'read', 'remove'], + result: false, + }, + { + name: 'credential manager is an empty obejct', + credentialManager: {}, + result: false, + }, + { + name: 'credential manager has property, but invalid types', + credentialManager: { + read: 'read', + write: 1234, + remove: [] }, - { - name: 'one parameter is null', - parameters: ['a', 2, true, null], - result: false + result: false, + }, + { + name: 'credential manager has property, but invalid types', + credentialManager: { + read: 'read', + write: 1234, + remove: [] }, - { - name: 'all the parameter are existing', - parameters: ['a', 123, ['testing'], {}], - result: true + result: false, + }, + { + name: 'credential manager has two valid properties, but miss one', + credentialManager: { + read: function () { + + }, + write: function () { + + } }, - ]; + result: false, + }, + { + name: 'credential manager has two valid properties, but miss one', + credentialManager: new sampleManager(), + result: true, + }, + ]; + + for (const { name, credentialManager, result } of testCases) { + it(name, function () { + assert.strictEqual(Util.checkValidCustomCredentialManager(credentialManager), result); + }); + } + }); + + describe('checkParametersDefined function Test', function () { + const testCases = [ + { + name: 'all the parameters are null or undefined', + parameters: [null, undefined, null, null], + result: false + }, + { + name: 'one parameter is null', + parameters: ['a', 2, true, null], + result: false + }, + { + name: 'all the parameter are existing', + parameters: ['a', 123, ['testing'], {}], + result: true + }, + ]; - for (const { name, parameters, result } of testCases) { - it(name, function () { - assert.strictEqual(Util.checkParametersDefined(...parameters), result); - }); - } - }); + for (const { name, parameters, result } of testCases) { + it(name, function () { + assert.strictEqual(Util.checkParametersDefined(...parameters), result); + }); + } }); if (os.platform() !== 'win32') { @@ -1317,122 +1215,4 @@ describe('Util', function () { } }); - describe('getProxyEnv function test ', function () { - let originalHttpProxy = null; - let originalHttpsProxy = null; - let originalNoProxy = null; - - before(() => { - originalHttpProxy = process.env.HTTP_PROXY; - originalHttpsProxy = process.env.HTTPS_PROXY; - originalNoProxy = process.env.NO_PROXY; - }); - - beforeEach(() => { - delete process.env.HTTP_PROXY; - delete process.env.HTTPS_PROXY; - delete process.env.NO_PROXY; - }); - - after(() => { - originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; - originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; - originalNoProxy ? process.env.NO_PROXY = originalNoProxy : delete process.env.NO_PROXY; - }); - - const testCases = [ - { - name: 'HTTP PROXY without authentication and schema', - isHttps: false, - httpProxy: 'proxy.example.com:8080', - httpsProxy: undefined, - noProxy: '*.amazonaws.com', - result: { - host: 'proxy.example.com', - port: 8080, - protocol: 'http:', - noProxy: '*.amazonaws.com' - } - }, - { - name: 'HTTP PROXY with authentication', - isHttps: false, - httpProxy: 'http://hello:world@proxy.example.com:8080', - httpsProxy: undefined, - noProxy: '*.amazonaws.com,*.my_company.com', - result: { - host: 'proxy.example.com', - user: 'hello', - password: 'world', - port: 8080, - protocol: 'http:', - noProxy: '*.amazonaws.com|*.my_company.com' - } - }, - { - name: 'HTTPS PROXY with authentication without NO proxy', - isHttps: true, - httpsProxy: 'https://user:pass@myproxy.server.com:1234', - result: { - host: 'myproxy.server.com', - user: 'user', - password: 'pass', - port: 1234, - protocol: 'https:', - noProxy: undefined, - }, - }, - { - name: 'HTTPS PROXY with authentication without NO proxy No schema', - isHttps: true, - noProxy: '*.amazonaws.com,*.my_company.com,*.test.com', - httpsProxy: 'myproxy.server.com:1234', - result: { - host: 'myproxy.server.com', - port: 1234, - protocol: 'http:', - noProxy: '*.amazonaws.com|*.my_company.com|*.test.com', - }, - }, - ]; - - testCases.forEach(({ name, isHttps, httpsProxy, httpProxy, noProxy, result }) => { - it(name, function (){ - - if (httpProxy){ - process.env.HTTP_PROXY = httpProxy; - } - if (httpsProxy) { - process.env.HTTPS_PROXY = httpsProxy; - } - if (noProxy) { - process.env.NO_PROXY = noProxy; - } - const proxy = Util.getProxyFromEnv(isHttps); - const keys = Object.keys(result); - assert.strictEqual(keys.length, Object.keys(proxy).length); - - for (const key of keys) { - assert.strictEqual(proxy[key], result[key]); - } - }); - }); - }); - - describe('getNoProxyEnv function Test', function () { - let original = null; - - before( function (){ - original = process.env.NO_PROXY; - process.env.NO_PROXY = '*.amazonaws.com,*.my_company.com'; - }); - - after(() => { - process.env.NO_PROXY = original; - }); - - it('test noProxy conversion', function (){ - assert.strictEqual(Util.getNoProxyEnv(), '*.amazonaws.com|*.my_company.com'); - }); - }); });