Support Quorum import export chain (#946)

Co-authored-by: Sai V <[email protected]>
Co-authored-by: Samer Falah <[email protected]>

Author: 3 people

cmd/geth/chaincmd.go

@@ -109,7 +109,7 @@ be gzipped.`,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-	The import-preimages command imports hash preimages from an RLP encoded stream.`,
+The import-preimages command imports hash preimages from an RLP encoded stream.`,
 	}
 	exportPreimagesCommand = cli.Command{
 		Action:    utils.MigrateFlags(exportPreimages),
@@ -232,7 +232,7 @@ func importChain(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	chain, chainDb := utils.MakeChain(ctx, stack)
+	chain, chainDb := utils.MakeChain(ctx, stack, true)
 	defer chainDb.Close()
 
 	// Start periodically gathering memory profiles
@@ -326,7 +326,7 @@ func exportChain(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	chain, _ := utils.MakeChain(ctx, stack)
+	chain, _ := utils.MakeChain(ctx, stack, true)
 	start := time.Now()
 
 	var err error
@@ -392,7 +392,7 @@ func copyDb(ctx *cli.Context) error {
 	}
 	// Initialize a new chain for the running node to sync into
 	stack := makeFullNode(ctx)
-	chain, chainDb := utils.MakeChain(ctx, stack)
+	chain, chainDb := utils.MakeChain(ctx, stack, false)
 
 	syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
 	dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
@@ -464,7 +464,7 @@ func removeDB(ctx *cli.Context) error {
 
 func dump(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
-	chain, chainDb := utils.MakeChain(ctx, stack)
+	chain, chainDb := utils.MakeChain(ctx, stack, false)
 	for _, arg := range ctx.Args() {
 		var block *types.Block
 		if hashish(arg) {

cmd/utils/flags.go

@@ -29,10 +29,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-
-	"github.com/ethereum/go-ethereum/permission"
-	"github.com/ethereum/go-ethereum/plugin"
-
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
@@ -42,7 +38,10 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/consensus/istanbul"
+	istanbulBackend "github.com/ethereum/go-ethereum/consensus/istanbul/backend"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -63,6 +62,8 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/permission"
+	"github.com/ethereum/go-ethereum/plugin"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -609,7 +610,7 @@ var (
 		Value: 50400,
 	}
 	RaftDNSEnabledFlag = cli.BoolFlag{
-		Name: "raftdnsenable",
+		Name:  "raftdnsenable",
 		Usage: "Enable DNS resolution of peers",
 	}
 
@@ -1292,7 +1293,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	}
 	cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
 
-
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
 		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
@@ -1545,17 +1545,41 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 }
 
 // MakeChain creates a chain manager from set command line flags.
-func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) {
-	var err error
+func MakeChain(ctx *cli.Context, stack *node.Node, useExist bool) (chain *core.BlockChain, chainDb ethdb.Database) {
+	var (
+		config *params.ChainConfig
+		err    error
+	)
 	chainDb = MakeChainDatabase(ctx, stack)
 
-	config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
-	if err != nil {
-		Fatalf("%v", err)
+	if useExist {
+		stored := rawdb.ReadCanonicalHash(chainDb, 0)
+		if (stored == common.Hash{}) {
+			Fatalf("No existing genesis")
+		}
+		config = rawdb.ReadChainConfig(chainDb, stored)
+	} else {
+		config, _, err = core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
+		if err != nil {
+			Fatalf("%v", err)
+		}
 	}
+
 	var engine consensus.Engine
 	if config.Clique != nil {
 		engine = clique.New(config.Clique, chainDb)
+	} else if config.Istanbul != nil {
+		// for IBFT
+		istanbulConfig := istanbul.DefaultConfig
+		if config.Istanbul.Epoch != 0 {
+			istanbulConfig.Epoch = config.Istanbul.Epoch
+		}
+		istanbulConfig.ProposerPolicy = istanbul.ProposerPolicy(config.Istanbul.ProposerPolicy)
+		istanbulConfig.Ceil2Nby3Block = config.Istanbul.Ceil2Nby3Block
+		engine = istanbulBackend.New(istanbulConfig, stack.GetNodeKey(), chainDb)
+	} else if config.IsQuorum {
+		// for Raft
+		engine = ethash.NewFullFaker()
 	} else {
 		engine = ethash.NewFaker()
 		if !ctx.GlobalBool(FakePoWFlag.Name) {

docs/Features/import-export.md

@@ -0,0 +1,48 @@
+# Backup & Restore of Quorum Nodes
+
+Quorum supports export and import of chain data with built in tooling. This is an effective node backup mechanism 
+adapted for the specific needs of Quorum such as private transactions, permissioning, and supported consensus 
+algorithms.
+
+
+!!! note
+    Quorum chain data import and export must run after `geth` process is stopped.
+
+### Node Backup (Export)
+
+Backup functionality mimics original `geth export` command. Quorum export accepts 3 arguments:
+
+1. Export file name **required**
+3. First block
+4. Last block *are optional but must be provided together when used*
+
+##### Sample command
+
+`geth export <export file name> --datadir <geth data dir>`
+
+### Node Restore (Import)
+
+Restore functionality mimics original `geth import` command but requires transaction manager environment variable.
+Quorum import must run on a new node with an initialized `--datadir` after `geth init` has been executed. Restore 
+supports arbitrary number of import files (at least 1).
+
+!!! warning
+    If private transactions are used in the chain data, Private Transaction Manager process for the original exported 
+    node must be running on the PTM ipc endpoint during import chain. Otherwise, nil pointer exceptions will be raised.
+
+##### Sample command
+
+`PRIVATE_CONFIG=<PTM ipc endpoint> geth import <import file names...> --datadir <geth data dir>`
+
+### Special Consensus Considerations
+
+##### IBFT
+
+IBFT block data contains sealer information in the header, to restore a copy of exported chain data, the new node must 
+be initialized use an IBFT genesis file with exact same validator set encoded in extra data field as original exported 
+node's genesis.
+
+##### Raft
+
+Raft backup do not account for current Raft state. An exported chain data from a Raft cluster can only be used by 
+new nodes being added to that same cluster only.

mkdocs.yml

@@ -96,6 +96,7 @@ nav:
     - How-To Guides:
         - Adding new nodes: How-To-Guides/adding_nodes.md
         - Adding IBFT validators: How-To-Guides/add_ibft_validator.md
+        - Backup & Restore: Features/import-export.md
     - Product Roadmap: roadmap.md
     - FAQ: FAQ.md