Merge remote-tracking branch 'upstream/master' into lexie_int_test

.github/workflows/horizon.yml

@@ -148,32 +148,3 @@ jobs:
       - if: github.ref == 'refs/heads/master'
         name: Push to DockerHub
         run: docker push stellar/horizon-verify-range:latest
-
-  ledger-exporter:
-    name: Test and push the Ledger Exporter images
-    runs-on: ubuntu-latest
-    env:
-      STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal
-      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          # For pull requests, build and test the PR head not a merge of the PR with the destination.
-          ref:  ${{ github.event.pull_request.head.sha || github.ref }}
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
-
-      - name: Run Ledger Exporter test
-        run: go test -v -race ./exp/services/ledgerexporter/... 
-
-      # Push images
-      - if: github.ref == 'refs/heads/master'
-        name: Login to DockerHub
-        uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - if: github.ref == 'refs/heads/master'
-        name: Push to DockerHub
-        run: make -C exp/services/ledgerexporter docker-push

.github/workflows/ledgerexporter-release.yml

@@ -0,0 +1,49 @@
+name: Ledger Exporter release
+
+on:
+  push:
+    tags: ['ledgerexporter-v*']
+
+jobs:
+
+  publish-docker:
+    name: Test and push the Ledger Exporter images
+    runs-on: ubuntu-latest
+    env:
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: stellar/quickstart:testing
+      STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      VERSION: ${GITHUB_REF_NAME#ledgerexporter-v}  
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: github.sha
+      - name: Install captive core
+        run: |
+          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+          # message about why the installation fails)
+          sudo apt list --installed | grep libc++
+          sudo apt-get remove -y libc++1-* libc++abi1-* || true
+
+          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+          sudo apt-get update && sudo apt-get install -y stellar-core="$STELLAR_CORE_VERSION"
+          echo "Using stellar core version $(stellar-core version)"
+
+      - name: Run Ledger Exporter test
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 
+
+      - name: Build Ledger Exporter docker
+        run: make -C exp/services/ledgerexporter docker-build
+
+      # Push images
+      - name: Login to DockerHub
+        uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push to DockerHub
+        run: make -C exp/services/ledgerexporter docker-push

.github/workflows/ledgerexporter.yml

@@ -0,0 +1,36 @@
+name: LedgerExporter
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  ledger-exporter:
+    name: Test Ledger Exporter
+    runs-on: ubuntu-latest
+    env:
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: stellar/quickstart:testing
+    steps:
+      - name: Install captive core
+        run: |
+          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+          # message about why the installation fails)
+          sudo apt list --installed | grep libc++
+          sudo apt-get remove -y libc++1-* libc++abi1-* || true
+
+          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+          sudo apt-get update && sudo apt-get install -y stellar-core="$CAPTIVE_CORE_DEBIAN_PKG_VERSION"
+          echo "Using stellar core version $(stellar-core version)"
+
+      - uses: actions/checkout@v3
+        with:
+          # For pull requests, build and test the PR head not a merge of the PR with the destination.
+          ref:  ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: Run Ledger Exporter test
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 

exp/services/ledgerexporter/DEVELOPER_GUIDE.md

@@ -0,0 +1,65 @@
+
+# Ledger Exporter Developer Guide
+The ledger exporter is a tool to export Stellar network transaction data to cloud storage in a way that is easy to access.
+
+## Prerequisites
+This document assumes that you have installed and can run the ledger exporter, and that you have familiarity with its CLI and configuration. If not, please refer to the [Installation Guide](./README.md).
+
+## Goal
+The goal of the ledger exporter is to build an easy-to-use tool to export Stellar network ledger data to a configurable remote data store, such as cloud blob storage.
+ - Use cloud storage optimally
+ - Minimize network usage to export
+ - Make it easy and fast to search for a specific ledger or ledger range
+
+## Architecture
+To achieve its goals, the ledger exporter uses the following architecture, which consists of the 3 main components:
+- Captive-core to extract raw transaction metadata from the Stellar Network.
+- Export manager to bundles and organizes the ledgers to get them ready for export.
+- The cloud storage plugin writes to the cloud storage. This is specific to the type of cloud storage, GCS in this case.
+
+
+![ledgerexporter-architecture](./architecture.png)
+
+
+## Data Format
+- Ledger exporter uses a compact and efficient data format called [XDR](https://developers.stellar.org/docs/learn/encyclopedia/data-format/xdr) (External Data Representation), which is a compact binary format. A Stellar Captive Core instance emits data in this format and the data structure is referred to as `LedgerCloseMeta`. The exporter bundles multiple `LedgerCloseMeta`'s into a single object using a custom XDR structure called `LedgerCloseMetaBatch` which is defined in [Stellar-exporter.x](https://github.com/stellar/go/blob/master/xdr/Stellar-exporter.x).
+
+- The metadata for the same batch is also stored alongside each exported object. Supported metadata is defined in [metadata.go](https://github.com/stellar/go/blob/master/support/datastore/metadata.go). 
+
+- Objects are compressed before uploading using the [zstd](http://facebook.github.io/zstd/) (zstandard) compression algorithm to optimize network usage and storage needs.
+
+## Data Storage
+- An example implementation of `DataStore` for GCS, Google Cloud Storage. This plugin is located in the [support](https://github.com/stellar/go/tree/master/support/datastore) package. 
+- The ledger exporter currently implements the interface only for Google Cloud Storage (GCS). The [GCS plugin](https://github.com/stellar/go/blob/master/support/datastore/gcs_datastore.go) uses GCS-specific behaviors like conditional puts, automatic retry, metadata, and CRC checksum.
+
+## Build, Run and Test using Docker
+The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) required to run the ledger exporter. 
+
+- Build: To build the Docker container, use the provided [Makefile](./Makefile). Simply run make `make docker-build` to build a new container after making any changes.
+
+- Run: For instructions on running the Docker container, refer to the [Installation Guide](./README.md).
+
+- Test: To test the Docker container, refer to the [docker-test](./Makefile) command for an example of how to use the [GCS emulator](https://github.com/fsouza/fake-gcs-server) for local testing.
+
+## Adding support for a new storage type
+Support for different data storage types are encapsulated as 'plugins', which are implementation of `DataStore` interface in a go package. To add a data storage plugin based on a new storage type (e.g. AWS S3), follow these steps:
+
+- A data storage plugin must implement the [DataStore](https://github.com/stellar/go/blob/master/support/datastore/datastore.go) interface.
+- Add support for new datastore-specific features. Implement any datastore-specific custom logic. Different datastores have different ways of handling 
+  - race conditions
+  - automatic retries
+  - metadata storage, etc.
+- Add the new datastore to the factory function [NewDataStore](https://github.com/stellar/go/blob/master/support/datastore/datastore.go).
+- Add a [config](./config.example.toml) section for the new storage type. This may include configurations like destination, authentication information etc.
+- An emulator such as a GCS emulator [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) can be used for testing without connecting to real cloud storage.
+
+### Design DOs and DONTs
+- Multiple exporters should be able to run in parallel without the need for explicit locking or synchronization.
+- Exporters when restarted do not have any memory of prior operation and rely on the already exported data as much as possible to decide where to resume.
+
+## Using exported data
+The exported data in storage can be used in the ETL pipeline to gather analytics and reporting. To write a tool that consumes exported data you can use Stellar ingestion library's `ledgerbackend` package. This package includes a ledger backend called [BufferedStorageBackend](https://github.com/stellar/go/blob/master/ingest/ledgerbackend/buffered_storage_backend.go),
+which imports data from the storage and validates it. For more details, refer to the ledgerbackend [documentation](https://github.com/stellar/go/tree/master/ingest/ledgerbackend).
+
+## Contributing
+For information on how to contribute, please refer to our [Contribution Guidelines](https://github.com/stellar/go/blob/master/CONTRIBUTING.md).

exp/services/ledgerexporter/Makefile

@@ -2,7 +2,7 @@ SUDO := $(shell docker version >/dev/null 2>&1 || echo "sudo")
 
 # https://github.com/opencontainers/image-spec/blob/master/annotations.md
 BUILD_DATE := $(shell date -u +%FT%TZ)
-VERSION ?= 1.0.0-$(shell git rev-parse --short HEAD)
+VERSION ?= $(shell git rev-parse --short HEAD)
 DOCKER_IMAGE := stellar/ledger-exporter
 
 docker-build: