Fix SQL NPE after connection is closed (#3829)

* fix npe for closed sql connections

* connection test

* cleanup

* unexport

* comments

Author: pdoerner

common/persistence/sql/factory.go

@@ -187,9 +187,7 @@ func (c *DbConn) Close() error {
 	defer c.Unlock()
 	c.refCnt--
 	if c.refCnt == 0 {
-		err := c.DB.Close()
-		c.DB = nil
-		return err
+		return c.DB.Close()
 	}
 	return nil
 }

common/persistence/tests/mysql_test.go

@@ -574,3 +574,11 @@ func TestMySQLVisibilitySuite(t *testing.T) {
 	s := sqltests.NewVisibilitySuite(t, store)
 	suite.Run(t, s)
 }
+
+func TestMySQLClosedConnectionError(t *testing.T) {
+	testData, tearDown := setUpMySQLTest(t)
+	defer tearDown()
+
+	s := newConnectionSuite(t, testData.Factory)
+	suite.Run(t, s)
+}

common/persistence/tests/persistence_connection_suite.go

@@ -0,0 +1,100 @@
+// The MIT License
+//
+// Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
+//
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package tests
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	p "go.temporal.io/server/common/persistence"
+	"go.temporal.io/server/common/persistence/serialization"
+	"go.temporal.io/server/common/persistence/sql"
+)
+
+type (
+	connectionSuite struct {
+		suite.Suite
+		*require.Assertions
+
+		factory *sql.Factory
+	}
+)
+
+func newConnectionSuite(
+	t *testing.T,
+	factory *sql.Factory,
+) *connectionSuite {
+	return &connectionSuite{
+		Assertions: require.New(t),
+		factory:    factory,
+	}
+}
+
+func (s *connectionSuite) SetupSuite() {
+
+}
+
+func (s *connectionSuite) TearDownSuite() {
+
+}
+
+func (s *connectionSuite) SetupTest() {
+	s.Assertions = require.New(s.T())
+}
+
+func (s *connectionSuite) TearDownTest() {
+
+}
+
+// Tests that SQL operations do not panic if the underlying connection has been closed and that the persistence layer
+// returns a useful error message.
+// Currently only run against MySQL and Postgresql (SQLite always maintains at least one open connection)
+func (s *connectionSuite) TestClosedConnectionError() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+	defer cancel()
+
+	shardID := (int32)(1)
+	rangeID := rand.Int63()
+	shardInfo := RandomShardInfo(shardID, rangeID)
+
+	store, err := s.factory.NewShardStore()
+	s.NoError(err)
+
+	store.Close() // Connection will be closed by this call
+	manager := p.NewShardManager(store, serialization.NewSerializer())
+
+	resp, err := manager.GetOrCreateShard(ctx, &p.GetOrCreateShardRequest{
+		ShardID:          shardID,
+		InitialShardInfo: shardInfo,
+	})
+
+	s.Nil(resp)
+	s.Error(err)
+	s.ErrorContains(err, "closed")
+}

common/persistence/tests/postgresql_test.go

@@ -581,3 +581,11 @@ func TestPostgreSQLVisibilitySuite(t *testing.T) {
 	s := sqltests.NewVisibilitySuite(t, store)
 	suite.Run(t, s)
 }
+
+func TestPostgreSQLClosedConnectionError(t *testing.T) {
+	testData, tearDown := setUpPostgreSQLTest(t)
+	defer tearDown()
+
+	s := newConnectionSuite(t, testData.Factory)
+	suite.Run(t, s)
+}