FAB-827 Add config option in core.yaml for couchDB

Add config option in core.yaml to switch state database from goleveldb to CouchDB

Added the following to core.yaml

    # stateDatabase - options are "goleveldb", "CouchDB"
    # goleveldb - default state database stored in goleveldb.
    # CouchDB - store state database in CouchDB
    stateDatabase: goleveldb
    couchDBConfig:
       couchDBAddress: 127.0.0.1:5984
       username:
       password:

Changes were also made to couchdb_txmgr to support the config option.

Changes were made to couchdb_test.go and marble_example.go to support the
config option in core.yaml and environment variables.

Marbles example can now be run with couchDb with the following command:

CORE_LEDGER_STATE_STATEDATABASE="CouchDB" go run marble_example.go

Adding a declaration for CORE_LEDGER_STATE_STATEDATABASE will work for peer,
couchdb_test and marbles example.

Change-Id: If02ac4f7c1c234d5f4c869875f3c331e35c9a066
Signed-off-by: Chris Elder <chris.elder@us.ibm.com>

Author: Chris Elder

core/ledger/kvledger/example/main/example.go

@@ -23,7 +23,7 @@ import (
 	"github.com/hyperledger/fabric/core/ledger"
 	"github.com/hyperledger/fabric/core/ledger/kvledger"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/example"
-
+	"github.com/hyperledger/fabric/core/ledger/testutil"
 	pb "github.com/hyperledger/fabric/protos/peer"
 )
 
@@ -40,6 +40,9 @@ var accounts = []string{"account1", "account2", "account3", "account4"}
 
 func init() {
 
+	//call a helper method to load the core.yaml
+	testutil.SetupCoreYAMLConfig("./../../../../../peer")
+
 	// Initialization will get a handle to the ledger at the specified path
 	// Note, if subledgers are supported in the future,
 	// the various ledgers could be created/managed at this level

core/ledger/kvledger/kv_ledger.go

@@ -79,13 +79,15 @@ func NewKVLedger(conf *Conf) (*KVLedger, error) {
 	if kvledgerconfig.IsCouchDBEnabled() == true {
 		//By default we can talk to CouchDB with empty id and pw (""), or you can add your own id and password to talk to a secured CouchDB
 		logger.Debugf("===COUCHDB=== NewKVLedger() Using CouchDB instead of RocksDB...hardcoding and passing connection config for now")
-		//TODO Hardcoding and passing connection config for now, eventually this will be passed from external config
+
+		couchDBDef := kvledgerconfig.GetCouchDBDefinition()
+
+		//create new transaction manager based on couchDB
 		txmgmt := couchdbtxmgmt.NewCouchDBTxMgr(&couchdbtxmgmt.Conf{DBPath: conf.txMgrDBPath},
-			"127.0.0.1", //couchDB host
-			5984,        //couchDB port
-			"system",    //couchDB db name matches ledger name, TODO for now use system ledger, eventually allow passing in subledger name
-			"",          //enter couchDB id here
-			"")          //enter couchDB pw here
+			couchDBDef.URL,      //couchDB connection URL
+			"system",            //couchDB db name matches ledger name, TODO for now use system ledger, eventually allow passing in subledger name
+			couchDBDef.Username, //enter couchDB id here
+			couchDBDef.Password) //enter couchDB pw here
 		return &KVLedger{blockStore, txmgmt, nil}, nil
 	}
 

core/ledger/kvledger/kvledgerconfig/kv_ledger_config.go

@@ -16,12 +16,35 @@ limitations under the License.
 
 package kvledgerconfig
 
-// Change this feature toggle to true to use CouchDB for state database
-// TODO Eventually this feature toggle will be externalized via a real
-//      config option on the peer
-var useCouchDB = false
+import "github.com/spf13/viper"
+
+var stateDatabase = "goleveldb"
+var couchDBAddress = "127.0.0.1:5984"
+var username = ""
+var password = ""
+
+// CouchDBDef contains parameters
+type CouchDBDef struct {
+	URL      string
+	Username string
+	Password string
+}
 
 //IsCouchDBEnabled exposes the useCouchDB variable
 func IsCouchDBEnabled() bool {
-	return useCouchDB
+	stateDatabase = viper.GetString("ledger.state.stateDatabase")
+	if stateDatabase == "CouchDB" {
+		return true
+	}
+	return false
+}
+
+//GetCouchDBDefinition exposes the useCouchDB variable
+func GetCouchDBDefinition() *CouchDBDef {
+
+	couchDBAddress = viper.GetString("ledger.state.couchDBConfig.couchDBAddress")
+	username = viper.GetString("ledger.state.couchDBConfig.username")
+	password = viper.GetString("ledger.state.couchDBConfig.password")
+
+	return &CouchDBDef{couchDBAddress, username, password}
 }

core/ledger/kvledger/marble_example/main/marble_example.go

@@ -23,10 +23,9 @@ import (
 	"github.com/hyperledger/fabric/core/ledger"
 	"github.com/hyperledger/fabric/core/ledger/kvledger"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/example"
-
-	logging "github.com/op/go-logging"
-
+	"github.com/hyperledger/fabric/core/ledger/testutil"
 	pb "github.com/hyperledger/fabric/protos/peer"
+	logging "github.com/op/go-logging"
 )
 
 var logger = logging.MustGetLogger("main")
@@ -47,6 +46,9 @@ func init() {
 	// the various ledgers could be created/managed at this level
 	logger.Debugf("===COUCHDB=== Marble Example main init()")
 
+	//call a helper method to load the core.yaml
+	testutil.SetupCoreYAMLConfig("./../../../../../peer")
+
 	os.RemoveAll(ledgerPath)
 	ledgerConf := kvledger.NewConf(ledgerPath, 0)
 	var err error

core/ledger/kvledger/txmgmt/couchdbtxmgmt/couchdb/couchdb.go

@@ -30,7 +30,6 @@ import (
 	"net/textproto"
 	"net/url"
 	"regexp"
-	"strconv"
 	"strings"
 
 	logging "github.com/op/go-logging"
@@ -104,21 +103,14 @@ type FileDetails struct {
 }
 
 //CreateConnectionDefinition for a new client connection
-func CreateConnectionDefinition(host string, port int, databaseName, username, password string) (*CouchDBConnectionDef, error) {
+func CreateConnectionDefinition(couchDBAddress string, databaseName, username, password string) (*CouchDBConnectionDef, error) {
 
 	logger.Debugf("===COUCHDB=== Entering CreateConnectionDefinition()")
 
-	connectURI := []string{}
-	connectURI = append(connectURI, "http://")
-	connectURI = append(connectURI, host)
-	connectURI = append(connectURI, ":")
-
-	connectURI = append(connectURI, strconv.Itoa(port))
-
-	urlConcat := strings.Join(connectURI, "")
+	connectURL := fmt.Sprintf("%s//%s", "http:", couchDBAddress)
 
 	//parse the constructed URL to verify no errors
-	finalURL, err := url.Parse(urlConcat)
+	finalURL, err := url.Parse(connectURL)
 	if err != nil {
 		logger.Errorf("===COUCHDB=== URL parse error: %s", err.Error())
 		return nil, err

core/ledger/kvledger/txmgmt/couchdbtxmgmt/couchdb/couchdb_test.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"testing"
 
-	kvledgerconfig "github.com/hyperledger/fabric/core/ledger/kvledger/kvledgerconfig"
+	"github.com/hyperledger/fabric/core/ledger/kvledger/kvledgerconfig"
 	"github.com/hyperledger/fabric/core/ledger/testutil"
 )
 
@@ -34,8 +34,8 @@ type Asset struct {
 	Owner     string `json:"owner"`
 }
 
-var hostname = "localhost"
-var port = 5984
+var connectURL = "localhost:5984"
+var badConnectURL = "localhost:5990"
 var database = "testdb1"
 var username = ""
 var password = ""
@@ -44,16 +44,19 @@ var assetJSON = []byte(`{"asset_name":"marble1","color":"blue","size":"35","owne
 
 func TestDBConnectionDef(t *testing.T) {
 
+	//call a helper method to load the core.yaml
+	testutil.SetupCoreYAMLConfig("./../../../../../../peer")
+
 	//create a new connection
-	_, err := CreateConnectionDefinition("localhost", 5984, "database", "", "")
+	_, err := CreateConnectionDefinition(connectURL, "database", "", "")
 	testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 }
 
 func TestDBBadConnectionDef(t *testing.T) {
 
 	//create a new connection
-	_, err := CreateConnectionDefinition("^^^localhost", 5984, "database", "", "")
+	_, err := CreateConnectionDefinition("^^^localhost:5984", "database", "", "")
 	testutil.AssertError(t, err, fmt.Sprintf("Did not receive error when trying to create database connection definition with a bad hostname"))
 
 }
@@ -66,7 +69,7 @@ func TestDBCreateSaveWithoutRevision(t *testing.T) {
 		defer cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -85,7 +88,7 @@ func TestDBBadConnection(t *testing.T) {
 	if kvledgerconfig.IsCouchDBEnabled() == true {
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port+5, database, username, password)
+		db, err := CreateConnectionDefinition(badConnectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -110,7 +113,7 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) {
 		cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -177,7 +180,7 @@ func TestDBBadJSON(t *testing.T) {
 		cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -216,7 +219,7 @@ func TestDBSaveAttachment(t *testing.T) {
 		attachments = append(attachments, attachment)
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -242,7 +245,7 @@ func TestDBRetrieveNonExistingDocument(t *testing.T) {
 		defer cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -264,7 +267,7 @@ func TestDBTestExistingDB(t *testing.T) {
 		defer cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//create a new database
@@ -286,7 +289,7 @@ func TestDBTestDropNonExistDatabase(t *testing.T) {
 		defer cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port, database, username, password)
+		db, err := CreateConnectionDefinition(connectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//Attempt to drop the database without creating first
@@ -304,7 +307,7 @@ func TestDBTestDropDatabaseBadConnection(t *testing.T) {
 		defer cleanup()
 
 		//create a new connection
-		db, err := CreateConnectionDefinition(hostname, port+4, database, username, password)
+		db, err := CreateConnectionDefinition(badConnectURL, database, username, password)
 		testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create database connection definition"))
 
 		//Attempt to drop the database without creating first
@@ -317,7 +320,7 @@ func TestDBTestDropDatabaseBadConnection(t *testing.T) {
 func cleanup() {
 
 	//create a new connection
-	db, _ := CreateConnectionDefinition(hostname, port, database, username, password)
+	db, _ := CreateConnectionDefinition(connectURL, database, username, password)
 
 	//drop the test database
 	db.DropDatabase()

core/ledger/kvledger/txmgmt/couchdbtxmgmt/couchdb_txmgmt_test.go

@@ -28,31 +28,35 @@ import (
 
 type testEnv struct {
 	conf              *Conf
-	couchHost         string
-	couchPort         int
+	couchDBAddress    string
 	couchDatabaseName string
 	couchUsername     string
 	couchPassword     string
 }
 
 func newTestEnv(t testing.TB) *testEnv {
+
+	//call a helper method to load the core.yaml
+	testutil.SetupCoreYAMLConfig("./../../../../../peer")
+
+	couchDBDef := kvledgerconfig.GetCouchDBDefinition()
+
 	conf := &Conf{"/tmp/tests/ledger/kvledger/txmgmt/couchdbtxmgmt"}
 	os.RemoveAll(conf.DBPath)
 	return &testEnv{
 		conf:              conf,
-		couchHost:         "127.0.0.1",
-		couchPort:         5984,
+		couchDBAddress:    couchDBDef.URL,
 		couchDatabaseName: "system_test",
-		couchUsername:     "",
-		couchPassword:     "",
+		couchUsername:     couchDBDef.Username,
+		couchPassword:     couchDBDef.Password,
 	}
 }
 
 func (env *testEnv) Cleanup() {
 	os.RemoveAll(env.conf.DBPath)
 
 	//create a new connection
-	couchDB, _ := couchdb.CreateConnectionDefinition(env.couchHost, env.couchPort, env.couchDatabaseName, env.couchUsername, env.couchPassword)
+	couchDB, _ := couchdb.CreateConnectionDefinition(env.couchDBAddress, env.couchDatabaseName, env.couchUsername, env.couchPassword)
 
 	//drop the test database if it already existed
 	couchDB.DropDatabase()
@@ -72,8 +76,7 @@ func TestDatabaseAutoCreate(t *testing.T) {
 		defer env.Cleanup() //and cleanup at the end
 
 		txMgr := NewCouchDBTxMgr(env.conf,
-			env.couchHost,         //couchDB host
-			env.couchPort,         //couchDB port
+			env.couchDBAddress,    //couchDB Address
 			env.couchDatabaseName, //couchDB db name
 			env.couchUsername,     //enter couchDB id
 			env.couchPassword)     //enter couchDB pw
@@ -88,8 +91,7 @@ func TestDatabaseAutoCreate(t *testing.T) {
 
 		//Call NewCouchDBTxMgr again, this time the database will already exist from last time
 		txMgr2 := NewCouchDBTxMgr(env.conf,
-			env.couchHost,         //couchDB host
-			env.couchPort,         //couchDB port
+			env.couchDBAddress,    //couchDB Address
 			env.couchDatabaseName, //couchDB db name
 			env.couchUsername,     //enter couchDB id
 			env.couchPassword)     //enter couchDB pw

core/ledger/kvledger/txmgmt/couchdbtxmgmt/couchdb_txmgr.go

@@ -82,14 +82,13 @@ type CouchConnection struct {
 }
 
 // NewCouchDBTxMgr constructs a `CouchDBTxMgr`
-func NewCouchDBTxMgr(conf *Conf, host string, port int, dbName string, id string, pw string) *CouchDBTxMgr {
+func NewCouchDBTxMgr(conf *Conf, couchDBConnectURL string, dbName string, id string, pw string) *CouchDBTxMgr {
 
 	// TODO cleanup this RocksDB handle
 	db := db.CreateDB(&db.Conf{DBPath: conf.DBPath})
 	db.Open()
 
-	couchDB, err := couchdb.CreateConnectionDefinition(host,
-		port,
+	couchDB, err := couchdb.CreateConnectionDefinition(couchDBConnectURL,
 		dbName,
 		id,
 		pw)

core/ledger/testutil/test_util.go

@@ -72,6 +72,19 @@ func SetupTestConfig() {
 	logging.SetFormatter(formatter)
 }
 
+// SetupCoreYAMLConfig sets up configurations for tetsing
+func SetupCoreYAMLConfig(coreYamlPath string) {
+	viper.SetConfigName("core")
+	viper.SetEnvPrefix("CORE")
+	viper.AddConfigPath(coreYamlPath)
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	viper.AutomaticEnv()
+	err := viper.ReadInConfig()
+	if err != nil { // Handle errors reading the config file
+		panic(fmt.Errorf("Fatal error config file: %s \n", err))
+	}
+}
+
 // SetLogLevel sets up log level
 func SetLogLevel(level logging.Level, module string) {
 	logging.SetLevel(level, module)

peer/core.yaml

@@ -343,7 +343,7 @@ chaincode:
     # A value <= 0 turns keepalive off
     keepalive: 0
 
-    # system chaincodes whitelist. To add system chaincode "myscc" to the  
+    # system chaincodes whitelist. To add system chaincode "myscc" to the
     # whitelist, add "myscc: enable" to the list
     system:
         lccc: enable
@@ -397,6 +397,14 @@ ledger:
         # configurations for 'trie'
         # 'tire' has no additional configurations exposed as yet
 
+    # stateDatabase - options are "goleveldb", "CouchDB"
+    # goleveldb - default state database stored in goleveldb.
+    # CouchDB - store state database in CouchDB
+    stateDatabase: goleveldb
+    couchDBConfig:
+       couchDBAddress: 127.0.0.1:5984
+       username:
+       password:
 
 ###############################################################################
 #