Merge pull request #3 from KYVENetwork/supervisor

Support supervised sync mode

Makefile

@@ -1,4 +1,4 @@
 #!/usr/bin/make -f
 
 ksync:
-	go build -mod=readonly -o ./build/ksync ./cmd/ksync/main.go
+	go build -mod=readonly -o ./build/ksync ./cmd/ksync/main.go

README.md

@@ -22,8 +22,9 @@
     - [Requirements](#db-requirements)
     - [Sync node](#sync-node-with-db)
 - [Examples](#examples)
-  - [1. Sync Osmosis on Kaon with DB-SYNC](#1-sync-osmosis-on-kaon-with-db-sync)
-  - [2. Sync Cosmos Hub on Mainnet with P2P-SYNC](#2-sync-cosmos-hub-on-mainnet-over-p2p-sync)
+  - [1. Sync Osmosis (Kaon) with DB-SYNC](#1-sync-osmosis-(kaon)-with-db-sync)
+  - [2. Sync Cosmos Hub (Mainnet) with P2P-SYNC](#2-sync-cosmos-hub-(mainnet)-with-p2p-sync)
+  - [3. Sync Cosmos Hub (Mainnet) with AUTO-SYNC](#3-sync-cosmos-hub-(mainnet)-with-auto-sync)
 
 ## What is KSYNC?
 
@@ -35,7 +36,7 @@ to sync blocks and join the network.
 
 ## How does it work?
 
-KSYNC comes with two sync modes which can be applied depending on the type of application. There is DB-SYNC
+KSYNC comes with three sync modes which can be applied depending on the type of application. There is DB-SYNC
 which syncs blocks by directly communicating with the app and writing the data directly to the database and then there
 P2P-SYNC where KSYNC mocks a peer in the network which has all the required blocks, streaming them over
 the dedicated block channels over to the node.
@@ -97,6 +98,8 @@ message size limitation in the Tendermint Socket Protocol (TSP).
 Currently, p2p sync is only supported for nodes using `github.com/tendermint/tendermint`. If nodes use CometBFT db sync
 has to be used. CometBFT support will be added in the future.
 
+> **_NOTE:_** [AUTO-SYNC](#auto-sync) wil manage the P2P or DB decision on its own.
+
 ### P2P-SYNC
 
 In this sync mode this tool mocks a peer which has all the blocks the actual peer node needs. The
@@ -151,22 +154,16 @@ When you see that the  node is trying to search for peers but is unable to find
 > above
 
 You can then start KSYNC in a **new** terminal with the following command. Please make sure to replace `<user>` and 
-`<chain>` with your specific values. This of course is also true for `<pool>` and `<network-api-endpoint>`.
+`<chain>` with your specific values. This of course is also true for `<pool>` and `<chain-id>`.
 
 ```bash
-ksync start mode=p2p --home="/Users/<user>/.<chain>" --pool-id=<pool> --rest=<network-api-endpoint>
+ksync start mode=p2p --home="/Users/<user>/.<chain>" --pool-id=<pool> --chain-id=<chain-id>
 ```
 
-Available rest endpoints for every network maintained by KYVE:
+Available chain ids are `kyve-1` for Mainnet, `kaon-1` for Kaon Testnet and `korellia` for Korellia Devnet
 
-- **KYVE (Mainnet)**
-  - https://api-eu-1.kyve.network
-  - https://api-us-1.kyve.network
-- **Kaon (Testnet)**
-  - https://api-eu-1.kaon.kyve.network
-  - https://api-us-1.kaon.kyve.network
-- **Korellia (Devent)**
-  - https://api.korellia.kyve.network
+> **_TIP:_** If you want to use your own rest endpoint for syncing, because you are running your own KYVE node
+> for example or want to use a different geolocated endpoint, simply overwrite it by adding the `--rest-endpoint=https://api-us-1.kyve.network`
 
 Once KSYNC starts it automatically continues from the latest height found in the node and starts downloading
 the blocks from the storage provider and validates the checksum. You should see blocks streaming over and the node
@@ -188,6 +185,21 @@ to the DB directly.
 It does not matter if you want to sync a node from genesis or from an existing height, the following settings have
 to be changed in order to run DB sync.
 
+Make sure that `persistent_peers` are empty in the `config.toml` config file:
+
+`~/.<chain>/config/config.toml`
+```toml
+[p2p]
+
+persistent_peers = ""
+```
+
+Make sure that your `addrbook.json` is empty or delete it entirely:
+
+```bash
+rm ~/.<chain>/config/addrbook.json
+```
+
 Make sure that `proxy_app` and `abci` have the following default values in the `config.toml` config file:
 
 `~/.<chain>/config/config.toml`
@@ -210,35 +222,48 @@ Now you can start your node with a special flag, so it does not start with tende
 
 If you see that the abci server is waiting for new connections you can proceed with starting KSYNC in a **new** 
 terminal with the following command. Please make sure to replace `<user>` and
-`<chain>` with your specific values. This of course is also true for `<pool>` and `<network-api-endpoint>`.
+`<chain>` with your specific values. This of course is also true for `<pool>` and `<chain-id>`.
 
-```bash
-ksync start mode=db --home="/Users/<user>/.<chain>" --pool-id=<pool> --rest=<network-api-endpoint>
-```
+Available chain ids are `kyve-1` for Mainnet, `kaon-1` for Kaon Testnet and `korellia` for Korellia Devnet
 
-Available rest endpoints for every network maintained by KYVE:
-
-- **KYVE (Mainnet)**
-  - https://api-eu-1.kyve.network
-  - https://api-us-1.kyve.network
-- **Kaon (Testnet)**
-  - https://api-eu-1.kaon.kyve.network
-  - https://api-us-1.kaon.kyve.network
-- **Korellia (Devent)**
-  - https://api.korellia.kyve.network
+> **_TIP:_** If you want to use your own rest endpoint for syncing, because you are running your own KYVE node
+> for example or want to use a different geolocated endpoint, simply overwrite it by adding the `--rest-endpoint=https://api-us-1.kyve.network`
 
 Once KSYNC starts it automatically continues from the latest height found in the node and starts downloading
 the blocks from the storage provider and validates the checksum. You should KSYNC committing blocks against the app.
 If you run this command without a `--target-height` it will sync all blocks which are
 available in the pool. KSYNC will automatically exit once a target height is reached, or you can simply exit the sync 
 process by killing KSYNC with CMD+C.
 
+### AUTO-SYNC
+
+Due to the [100MB limitation](#limitations), P2P-Sync was implemented to support KSYNC for any type of Tendermint blockchain.
+However, in comparison to DB-Sync it has some disadvantages, which is why it's recommended to use P2P-Sync only as long as the first block was synced successfully before switching back to DB-SYNC.
+On top of that, you need to run the node to be synced and KSYNC in two different terminals, which isn't that comfortable for developers.
+AUTO-SYNC consists of a process manager, running P2P-Sync or DB-Sync on the one hand and the node to be synced on the other hand.
+Based on the requirements, the process manager will manage which syncing process is required, thereby enabling the best way of syncing the node with the validated blocks.
+If the node is completely synced with the corresponding KYVE pool, it will start normally to find peers through the provided seeds.
+This resolves in the ability to sync a node completely with KYVE data requiring just one command.
+
+#### AUTO-SYNC requirements
+
+The requirements are similar to the [DB-SYNC requirements](#db-requirements). Everything else will be set up automatically.
+
+#### Sync node with AUTO-SYNC
+
+To start the syncing process, simply run
+
+````bash
+ksync start --home="/Users/<user>/.<chain>" --daemon-path="/Users/<user>/<daemon>" --pool-id=<pool> --chain-id=<chain-id> --seeds <p2p.seeds>
+````
+
+> **_TIP:_** Since the "auto" is the default syncing mode you don't have to specifiy with a flag
+
 ## Examples
 
-All examples below use test data from a KYVE test chain running on `http://35.158.99.65:26657`. This should not be
-used in production and is only intended for demonstration purposes.
+Below are two example of how to use the different recommended sync modes for specific chains.
 
-### 1. Sync Osmosis on Kaon with DB-SYNC
+### 1. Sync Osmosis (Kaon) with DB-SYNC
 
 To sync osmosis you have to download and set up the correct osmosis binary. To sync from genesis the version `v3.1.0` has
 to be used. You can download them [here](https://github.com/osmosis-labs/osmosis/releases/tag/v3.1.0) or build them from source: [https://github.com/osmosis-labs/osmosis](https://github.com/osmosis-labs/osmosis)
@@ -262,8 +287,8 @@ download the genesis
 wget -O ~/.osmosisd/config/genesis.json https://github.com/osmosis-labs/networks/raw/main/osmosis-1/genesis.json
 ```
 
-Important: Don't include an addrbook.json and make sure persistent_peers and etc. are empty for now or else the node will connect to other peers. It should only connect
-to our peer.
+Important: Don't include an addrbook.json and make sure persistent_peers and etc. (e.g. unconditional_peer_ids, private_peer_ids
+and seeds) are empty for now or else the node will connect to other peers. It should only connect to our peer.
 
 when the config is done the node can be started
 
@@ -275,16 +300,16 @@ After you see that the node is waiting for incoming connections you can open a *
 the sync.
 
 ```bash
-ksync start --mode=db --home="/Users/<user>/.osmosisd" --pool-id=1 --rest=https://api-eu-1.kaon.kyve.network
+ksync start --mode=db --home="/Users/<user>/.osmosisd" --pool-id=1 --chain-id=kaon-1
 ```
 
-You should see KSYNC connecting to Osmosis and applying the blocks against the app. After the ~600 blocks were
-applied KSYNC automatically exits.
+You should see KSYNC connecting to Osmosis and applying the blocks against the app. You can exit anytime with CMD+C
+if you wish to abort the syncing process.
 
 When you want to continue to sync normally you can now add an addrbook or add peers in `persistent_peers`. When you start
 the node again with the normal start command `./osmosisd start` the node should continue normally and tries to sync the remaining blocks.
 
-### 2. Sync Cosmos Hub on Mainnet over P2P-SYNC
+### 2. Sync Cosmos Hub (Mainnet) with P2P-SYNC
 
 Since we want to sync Cosmos Hub from genesis and the genesis file is bigger than 100MB we have to use P2P sync.
 
@@ -320,11 +345,11 @@ pex = false
 allow_duplicate_ip = true
 ```
 
-Important: Don't include an addrbook.json and make sure persistent_peers and etc. are empty for now or else the node
-will connect to other peers. It should only connect to our peer.
+Important: Don't include an addrbook.json and make sure persistent_peers and etc. (e.g. unconditional_peer_ids, private_peer_ids
+and seeds) are empty for now or else the node will connect to other peers. It should only connect to our peer.
 
 When the config is done the node can be started. NOTE: this can take a while (~5mins) since the genesis file is
-quite big. You can skip invariants checks to boot even fast, but it still takes a long time until the gaia node starts.
+quite big. You can skip invariants checks to boot even faster, but it still takes a long time until the gaia node starts.
 
 ```bash
 ./gaiad start --x-crisis-skip-assert-invariants
@@ -334,11 +359,47 @@ After you see that the node is searching for peers you can start the tool. For t
 5000 blocks of Cosmos Hub, so after that height is reached the sync will be done.
 
 ```bash
-ksync start --mode=p2p --home="/Users/<user>/.gaia" --pool-id=0 --rest=https://api-eu-1.kyve.network
+ksync start --mode=p2p --home="/Users/<user>/.gaia" --pool-id=0 --chain-id=kyve-1
 ```
 
 You should see the peer connecting and sending over blocks to the gaia node. After all the blocks have been applied
 the tool shows _Done_ and you can safely exit the process with CMD+C.
 
 When you want to continue to sync normally you can now add an addrbook or add peers in `persistent_peers`.
 When you start  the node again the node should continue normally and tries to sync the remaining blocks.
+
+### 3. Sync Cosmos Hub (Mainnet) with AUTO-SYNC
+
+Cosmos Hub requires a start with P2P sync due to the >100MB genesis file.
+To simplify the syncing process, we can use AUTO-SYNC to let KSYNC manage both processes independently.
+
+To start successfully, you need to download and set up the correct binary with the version `v.4.2.1`. You can download them [here](https://github.com/cosmos/gaia/releases/tag/v4.2.1) or build them from source:
+[https://github.com/cosmos/gaia](https://github.com/cosmos/gaia)
+
+Verify installation with
+
+```bash
+./gaiad version
+4.2.1
+```
+
+After the installation init the project
+
+```bash
+./gaiad init <your-moniker> --chain-id cosmoshub-4
+```
+
+download the genesis
+
+```bash
+wget https://raw.githubusercontent.com/cosmos/mainnet/master/genesis/genesis.cosmoshub-4.json.gz
+gzip -d genesis.cosmoshub-4.json.gz
+mv genesis.cosmoshub-4.json ~/.gaia/config/genesis.json
+```
+
+Don't include an addrbook.json and KSYNC will manage your config file itself.
+It should only connect to our peer. The supervised KSYNC process can be started with
+
+`````bash
+ ksync start --daemon-path /<daemon-path>/gaiad --home /Users/<user>/.gaia --pool-id 0 --chain-id=kyve-1
+`````

cmd/ksync/commands/root.go

@@ -2,9 +2,14 @@ package commands
 
 import (
 	"fmt"
+	log "github.com/KYVENetwork/ksync/logger"
 	"github.com/spf13/cobra"
 )
 
+var (
+	logger = log.Logger("commands")
+)
+
 // RootCmd is the root command for KSYNC.
 var rootCmd = &cobra.Command{
 	Use:   "ksync",

cmd/ksync/commands/start.go

@@ -2,65 +2,94 @@ package commands
 
 import (
 	"fmt"
+	"github.com/KYVENetwork/ksync/executor/auto"
 	"github.com/KYVENetwork/ksync/executor/db"
 	"github.com/KYVENetwork/ksync/executor/p2p"
-	log "github.com/KYVENetwork/ksync/logger"
 	"github.com/KYVENetwork/ksync/utils"
 	"github.com/spf13/cobra"
+	"strings"
 )
 
 var (
-	logger = log.Logger()
-)
-
-var (
+	daemonPath   string
+	flags        string
 	mode         string
 	home         string
 	poolId       int64
+	seeds        string
 	targetHeight int64
+	chainId      string
 	restEndpoint string
 
 	quitCh = make(chan int)
 )
 
 func init() {
-	startCmd.Flags().StringVar(&mode, "mode", "", "sync mode (\"p2p\",\"db\")")
-	if err := startCmd.MarkFlagRequired("mode"); err != nil {
-		panic(fmt.Errorf("flag 'sync-mode' should be required: %w", err))
-	}
+	startCmd.Flags().StringVar(&mode, "mode", utils.DefaultMode, fmt.Sprintf("sync mode (\"auto\",\"db\",\"p2p\"), [default = %s]", utils.DefaultMode))
 
 	startCmd.Flags().StringVar(&home, "home", "", "home directory")
 	if err := startCmd.MarkFlagRequired("home"); err != nil {
 		panic(fmt.Errorf("flag 'home' should be required: %w", err))
 	}
 
-	startCmd.Flags().StringVar(&restEndpoint, "rest", utils.DefaultRestEndpoint, fmt.Sprintf("kyve chain rest endpoint [default = %s]", utils.DefaultRestEndpoint))
+	// Optional AUTO-MODE flags.
+	startCmd.Flags().StringVar(&daemonPath, "daemon-path", "", "daemon path of node to be synced")
+
+	startCmd.Flags().StringVar(&chainId, "chain-id", utils.DefaultChainId, fmt.Sprintf("kyve chain id (\"kyve-1\",\"kaon-1\",\"korellia\"), [default = %s]", utils.DefaultChainId))
 
 	startCmd.Flags().Int64Var(&poolId, "pool-id", 0, "pool id")
 	if err := startCmd.MarkFlagRequired("pool-id"); err != nil {
 		panic(fmt.Errorf("flag 'pool-id' should be required: %w", err))
 	}
 
+	startCmd.Flags().StringVar(&restEndpoint, "rest-endpoint", "", "Overwrite default rest endpoint from chain")
+
 	startCmd.Flags().Int64Var(&targetHeight, "target-height", 0, "target height (including)")
 
+	startCmd.Flags().StringVar(&seeds, "seeds", "", "P2P seeds to continue syncing process after KSYNC")
+
+	startCmd.Flags().StringVar(&flags, "flags", "", "Flags for starting the node to be synced; excluding --home and --with-tendermint")
+
 	rootCmd.AddCommand(startCmd)
 }
 
 var startCmd = &cobra.Command{
 	Use:   "start",
-	Short: "Start fast syncing blocks",
+	Short: "Start fast syncing blocks with KSYNC",
 	Run: func(cmd *cobra.Command, args []string) {
-		if mode != "p2p" && mode != "db" {
-			logger.Error("flag sync-mode has to be either \"p2p\" or \"db\"")
+		// if no custom rest endpoint was given we take it from the chainId
+		if restEndpoint == "" {
+			switch chainId {
+			case "kyve-1":
+				restEndpoint = utils.RestEndpointMainnet
+			case "kaon-1":
+				restEndpoint = utils.RestEndpointKaon
+			case "korellia":
+				restEndpoint = utils.RestEndpointKorellia
+			default:
+				panic("flag --chain-id has to be either \"kyve-1\", \"kaon-1\" or \"korellia\"")
+			}
 		}
 
-		if mode == "p2p" {
-			go p2p.StartP2PExecutor(quitCh, home, poolId, restEndpoint, targetHeight)
-		} else {
+		// trim trailing slash
+		restEndpoint = strings.TrimSuffix(restEndpoint, "/")
+
+		// start block executor based on sync mode
+		switch mode {
+		case "auto":
+			if daemonPath == "" {
+				panic("flag --daemon-path is required for mode \"auto\"")
+			}
+			auto.StartAutoExecutor(quitCh, home, daemonPath, seeds, flags, poolId, restEndpoint, targetHeight)
+		case "db":
 			go db.StartDBExecutor(quitCh, home, poolId, restEndpoint, targetHeight)
+		case "p2p":
+			go p2p.StartP2PExecutor(quitCh, home, poolId, restEndpoint, targetHeight)
+		default:
+			panic("flag --mode has to be either \"auto\", \"db\" or \"p2p\"")
 		}
 
-		// wait for executor to finish
+		// only exit process if executor has finished
 		<-quitCh
 	},
 }