[FAB-9130] fix race on metrics.RootScope

Introduced lock around RootScope.

Change-Id: I85c5289c369c9a75c6b03e88a8fa76428aa79408
Signed-off-by: Saad Karim <skarim@us.ibm.com>
Signed-off-by: Matthew Sykes <sykesmat@us.ibm.com>

Author: Saad Karim

common/metrics/server.go

@@ -9,7 +9,6 @@ package metrics
 import (
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/spf13/viper"
@@ -31,7 +30,8 @@ const (
 
 var RootScope Scope
 var once sync.Once
-var started uint32
+var rootScopeMutex = &sync.Mutex{}
+var running bool
 
 // NewOpts create metrics options based config file
 func NewOpts() Opts {
@@ -84,21 +84,33 @@ func Init(opts Opts) (err error) {
 
 //Start starts metrics server
 func Start() error {
-	if atomic.CompareAndSwapUint32(&started, 0, 1) {
-		return RootScope.Start()
+	rootScopeMutex.Lock()
+	defer rootScopeMutex.Unlock()
+	if running {
+		return nil
 	}
-	return nil
+	running = true
+	return RootScope.Start()
 }
 
 //Shutdown closes underlying resources used by metrics server
 func Shutdown() error {
-	if atomic.CompareAndSwapUint32(&started, 1, 0) {
-		err := RootScope.Close()
-		RootScope = nil
-		return err
+	rootScopeMutex.Lock()
+	defer rootScopeMutex.Unlock()
+	if !running {
+		return nil
 	}
 
-	return nil
+	err := RootScope.Close()
+	RootScope = nil
+	running = false
+	return err
+}
+
+func isRunning() bool {
+	rootScopeMutex.Lock()
+	defer rootScopeMutex.Unlock()
+	return running
 }
 
 type StatsdReporterOpts struct {

common/metrics/server_test.go

@@ -9,11 +9,11 @@ package metrics
 import (
 	"fmt"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/hyperledger/fabric/core/config/configtest"
+	. "github.com/onsi/gomega"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 )
@@ -155,6 +155,7 @@ func TestStartInvalidReporter(t *testing.T) {
 
 func TestStartAndClose(t *testing.T) {
 	t.Parallel()
+	gt := NewGomegaWithT(t)
 	defer Shutdown()
 	opts := Opts{
 		Enabled:  true,
@@ -166,10 +167,9 @@ func TestStartAndClose(t *testing.T) {
 			FlushBytes:    512,
 		}}
 	Init(opts)
-	go Start()
-	time.Sleep(1 * time.Second)
 	assert.NotNil(t, RootScope)
-	assert.Equal(t, uint32(1), atomic.LoadUint32(&started))
+	go Start()
+	gt.Eventually(isRunning).Should(BeTrue())
 }
 
 func TestNoOpScopeMetrics(t *testing.T) {