SNOW-1781419: when proxy is set in Connection, driver does send traffic through the proxy to S3, but not to Azure blob / GCS bucket (only Snowflake). Works with proxy envvar (Azure)

lib/connection/connection_config.js

@@ -5,6 +5,7 @@
 const os = require('os');
 const url = require('url');
 const Util = require('../util');
+const ProxyUtil = require('../proxy_util');
 const Errors = require('../errors');
 const ConnectionConstants = require('../constants/connection_constants');
 const path = require('path');
@@ -223,7 +224,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
       protocol: proxyProtocol,
       noProxy: noProxy
     };
-    Util.validateProxy(proxy);
+    ProxyUtil.validateProxy(proxy);
   }
 
   const serviceName = options.serviceName;

lib/file_transfer_agent/azure_util.js

@@ -6,6 +6,9 @@
 const FileHeader = require('./file_util').FileHeader;
 const expandTilde = require('expand-tilde');
 const resultStatus = require('./file_util').resultStatus;
+const ProxyUtil = require('../proxy_util');
+const { isBypassProxy } = require('../http/node');
+const Logger = require('../logger');
 
 const EXPIRED_TOKEN = 'ExpiredToken';
 
@@ -26,7 +29,7 @@
  * @returns {Object}
  * @constructor
  */
-function AzureUtil(azure, filestream) {
+function AzureUtil(connectionConfig, azure, filestream) {
   const AZURE = typeof azure !== 'undefined' ? azure : require('@azure/storage-blob');
   const fs = typeof filestream !== 'undefined' ? filestream : require('fs');
 
@@ -42,11 +45,23 @@
     const sasToken = stageCredentials['AZURE_SAS_TOKEN'];
 
     const account = stageInfo['storageAccount'];
-
+    const connectionString = `https://${account}.blob.core.windows.net${sasToken}`;
+    let proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'Azure Util');
+    if (proxy && !isBypassProxy(proxy, connectionString)) {
+      Logger.getInstance().debug(`The destination host is: ${ProxyUtil.getHostFromURL(connectionString)} and the proxy host is: ${proxy.host}`);
+      Logger.getInstance().trace(`Initializing the proxy information for the Azure Client: ${ProxyUtil.describeProxy(proxy)}`);
+
+      proxy = ProxyUtil.getAzureProxy(proxy);
+      Logger.getInstance().trace(connectionConfig.describe);
+    }
+    ProxyUtil.hideEnvironmentProxy();
     const blobServiceClient = new AZURE.BlobServiceClient(
-      `https://${account}.blob.core.windows.net${sasToken}`
+      connectionString, null,
+      {
+        proxyOptions: proxy,
+      }
     );
-
+    ProxyUtil.restoreEnvironmentProxy();
     return blobServiceClient;
   };
 
@@ -203,7 +218,7 @@
           blobContentEncoding: 'UTF-8',
           blobContentType: 'application/octet-stream'
         }
-      });
+      });    
     } catch (err) {
       if (err['statusCode'] === 403 && detectAzureTokenExpireError(err)) {
         meta['lastError'] = err;
@@ -215,7 +230,6 @@
       }
       return;
     }
-
     meta['dstFileSize'] = meta['uploadSize'];
     meta['resultStatus'] = resultStatus.UPLOADED;
   };
@@ -262,7 +276,6 @@
       }
       return;
     }
-
     meta['resultStatus'] = resultStatus.DOWNLOADED;
   };
 
@@ -282,5 +295,4 @@
       errstr.includes('Server failed to authenticate the request.');
   }
 }
-
 module.exports = AzureUtil;

lib/file_transfer_agent/remote_storage_util.js

@@ -44,7 +44,7 @@ function RemoteStorageUtil(connectionConfig) {
     if (type === 'S3') {
       return new SnowflakeS3Util(connectionConfig);
     } else if (type === 'AZURE') {
-      return new SnowflakeAzureUtil();
+      return new SnowflakeAzureUtil(connectionConfig);
     } else if (type === 'GCS') {
       return new SnowflakeGCSUtil();
     } else {

lib/file_transfer_agent/s3_util.js

@@ -7,9 +7,7 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
 const expandTilde = require('expand-tilde');
 const getProxyAgent = require('../http/node').getProxyAgent;
-const Util = require('../util');
-const Logger = require('../logger');
-const GlobalConfig = require('../global_config');
+const ProxyUtil = require('../proxy_util');
 
 const AMZ_IV = 'x-amz-iv';
 const AMZ_KEY = 'x-amz-key';
@@ -79,13 +77,7 @@ function S3Util(connectionConfig, s3, filestream) {
       useAccelerateEndpoint: useAccelerateEndpoint
     };
 
-    let proxy = connectionConfig.getProxy();
-    if (!proxy && GlobalConfig.isEnvProxyActive()) {
-      proxy = Util.getProxyFromEnv();
-      if (proxy) {
-        Logger.getInstance().debug(`S3 Util loads the proxy info from the environment variable host: ${proxy.host}`);
-      }
-    }
+    const proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'S3 Util');
     if (proxy) {
       const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || SNOWFLAKE_S3_DESTINATION);
       config.requestHandler = new NodeHttpHandler({

lib/http/node.js

@@ -3,6 +3,7 @@
  */
 
 const Util = require('../util');
+const ProxyUtil = require('../proxy_util');
 const Base = require('./base');
 const HttpsAgent = require('../agent/https_ocsp_agent');
 const HttpsProxyAgent = require('../agent/https_proxy_agent');
@@ -56,7 +57,7 @@ function getFromCacheOrCreate(agentClass, options, agentId) {
     Logger.getInstance().trace('Agent[id: %s] - new instance stored in cache.', agentId);
 
     // detect and log PROXY envvar + agent proxy settings
-    const compareAndLogEnvAndAgentProxies = Util.getCompareAndLogEnvAndAgentProxies(agentOptions);
+    const compareAndLogEnvAndAgentProxies = ProxyUtil.getCompareAndLogEnvAndAgentProxies(agentOptions);
     Logger.getInstance().debug('Agent[id: %s] - proxy settings used in requests: %s', agentId, compareAndLogEnvAndAgentProxies.messages);
     // if there's anything to warn on (e.g. both envvar + agent proxy used, and they are different)
     // log warnings on them
@@ -109,7 +110,7 @@ NodeHttpClient.prototype.getAgent = function (parsedUrl, proxy, mock) {
   Logger.getInstance().trace('Agent[url: %s] - getting an agent instance.', parsedUrl.href);
   if (!proxy && GlobalConfig.isEnvProxyActive()) {
     const isHttps = parsedUrl.protocol === 'https:';
-    proxy = Util.getProxyFromEnv(isHttps);
+    proxy = ProxyUtil.getProxyFromEnv(isHttps);
     if (proxy) {
       Logger.getInstance().debug('Agent[url: %s] - proxy info loaded from the environment variable. Proxy host: %s', parsedUrl.href, proxy.host);
     }
@@ -133,7 +134,7 @@ function getProxyAgent(proxyOptions, parsedUrl, destination, mock) {
     }
   }
 
-  const destHost = Util.getHostFromURL(destination);
+  const destHost = ProxyUtil.getHostFromURL(destination);
   const agentId = createAgentId(agentOptions.protocol, agentOptions.hostname, destHost, agentOptions.keepAlive);
   Logger.getInstance().debug('Agent[id: %s] - the destination host is: %s.', agentId, destHost);
 
@@ -170,4 +171,4 @@ function getAgentCacheSize() {
   return httpsAgentCache.size;
 }
 
-module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize };
+module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize, isBypassProxy };