add mesc support

ethereumetl/cli/export_all.py

@@ -32,7 +32,7 @@
 from ethereumetl.jobs.export_all_common import export_all_common
 from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.service.eth_service import EthService
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
@@ -109,7 +109,7 @@ def get_partitions(start, end, partition_batch_size, provider_uri):
 @click.option('-e', '--end', required=True, type=str, help='End block/ISO date/Unix time')
 @click.option('-b', '--partition-batch-size', default=10000, show_default=True, type=int,
               help='The number of blocks to export in partition.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output-dir', default='output', show_default=True, type=str, help='Output directory, partitioned in Hive style.')

ethereumetl/cli/export_blocks_and_transactions.py

@@ -28,7 +28,7 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.thread_local_proxy import ThreadLocalProxy
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
@@ -37,7 +37,7 @@
 @click.option('-s', '--start-block', default=0, show_default=True, type=int, help='Start block')
 @click.option('-e', '--end-block', required=True, type=int, help='End block')
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to export at a time.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')

ethereumetl/cli/export_contracts.py

@@ -29,7 +29,7 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.thread_local_proxy import ThreadLocalProxy
 from ethereumetl.providers.auto import get_provider_from_uri
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
@@ -40,7 +40,7 @@
               help='The file containing contract addresses, one per line.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')

ethereumetl/cli/export_receipts_and_logs.py

@@ -29,7 +29,7 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.thread_local_proxy import ThreadLocalProxy
 from ethereumetl.providers.auto import get_provider_from_uri
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
@@ -38,7 +38,7 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of receipts to export at a time.')
 @click.option('-t', '--transaction-hashes', required=True, type=str,
               help='The file containing transaction hashes, one per line.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')

ethereumetl/cli/export_tokens.py

@@ -31,7 +31,7 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.thread_local_proxy import ThreadLocalProxy
 from ethereumetl.providers.auto import get_provider_from_uri
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
@@ -41,7 +41,7 @@
               help='The file containing token addresses, one per line.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')

ethereumetl/cli/extract_tokens.py

@@ -33,14 +33,15 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.thread_local_proxy import ThreadLocalProxy
+from ethereumetl.utils import get_default_provider_uri
 from ethereumetl.web3_utils import build_web3
 
 logging_basic_config()
 
 
 @click.command(context_settings=dict(help_option_names=['-h', '--help']))
 @click.option('-c', '--contracts', type=str, required=True, help='The JSON file containing contracts.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')

ethereumetl/cli/get_block_range_for_date.py

@@ -30,13 +30,13 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.service.eth_service import EthService
 from ethereumetl.providers.auto import get_provider_from_uri
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
 
 @click.command(context_settings=dict(help_option_names=['-h', '--help']))
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-d', '--date', required=True, type=lambda d: datetime.strptime(d, '%Y-%m-%d'),

ethereumetl/cli/get_block_range_for_timestamps.py

@@ -29,13 +29,13 @@
 from blockchainetl.logging_utils import logging_basic_config
 from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.service.eth_service import EthService
-from ethereumetl.utils import check_classic_provider_uri
+from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri
 
 logging_basic_config()
 
 
 @click.command(context_settings=dict(help_option_names=['-h', '--help']))
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-s', '--start-timestamp', required=True, type=int, help='Start unix timestamp, in seconds.')

ethereumetl/cli/stream.py

@@ -29,12 +29,13 @@
 from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.streaming.item_exporter_creator import create_item_exporters
 from ethereumetl.thread_local_proxy import ThreadLocalProxy
+from ethereumetl.utils import get_default_provider_uri
 
 
 @click.command(context_settings=dict(help_option_names=['-h', '--help']))
 @click.option('-l', '--last-synced-block-file', default='last_synced_block.txt', show_default=True, type=str, help='')
 @click.option('--lag', default=0, show_default=True, type=int, help='The number of blocks to lag behind the network.')
-@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
+@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output', type=str,

ethereumetl/providers/auto.py

@@ -23,6 +23,7 @@
 
 from urllib.parse import urlparse
 
+import mesc
 from web3 import IPCProvider, HTTPProvider
 
 from ethereumetl.providers.ipc import BatchIPCProvider
@@ -32,6 +33,15 @@
 
 
 def get_provider_from_uri(uri_string, timeout=DEFAULT_TIMEOUT, batch=False):
+
+    if mesc.is_mesc_enabled():
+        try:
+            endpoint = mesc.get_endpoint_by_query(uri_string, profile='ethereum_etl')
+            if endpoint is not None:
+                uri_string = endpoint['url']
+        except Exception as e:
+            print('MESC configured improperly')
+
     uri = urlparse(uri_string)
     if uri.scheme == 'file':
         if batch:

ethereumetl/utils.py

@@ -24,6 +24,8 @@
 import itertools
 import warnings
 
+import mesc
+
 from ethereumetl.misc.retriable_value_error import RetriableValueError
 
 
@@ -142,3 +144,15 @@ def check_classic_provider_uri(chain, provider_uri):
         warnings.warn("ETC Chain not supported on Infura.io. Using https://ethereumclassic.network instead")
         return 'https://ethereumclassic.network'
     return provider_uri
+
+
+def get_default_provider_uri():
+    if mesc.is_mesc_enabled():
+        try:
+            endpoint = mesc.get_default_endpoint(profile='ethereum_etl')
+            if endpoint is not None:
+                return endpoint['url']
+        except Exception:
+            print('MESC configured improperly')
+
+    return 'https://mainnet.infura.io'

setup.py

@@ -34,6 +34,7 @@ def read(fname):
         'web3>=5.29,<6',
         'eth-utils==1.10',
         'eth-abi>=2.2.0,<3.0.0',
+        'mesc==0.2.0',
         # TODO: This has to be removed when "ModuleNotFoundError: No module named 'eth_utils.toolz'" is fixed at eth-abi
         'python-dateutil>=2.8.0,<3',
         'click>=8.0.4,<9',