Skip to content

Commit fa429a5

Browse files
authored
[Advanced Visibility with SQL] Adding MySQL 8 schema and interface (#3552)
* Create base schema for MySQL 8 * MySQL 8 schema changes to support advanced visibility * Create base MySQL 8 db interface and configs
1 parent 416abfb commit fa429a5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1520
-19
lines changed

Makefile

+14
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,17 @@ install-schema-mysql: temporal-sql-tool
361361
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) setup-schema -v 0.0
362362
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) update-schema -d ./schema/mysql/v57/visibility/versioned
363363

364+
install-schema-mysql8: temporal-sql-tool
365+
@printf $(COLOR) "Install MySQL schema..."
366+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) drop -f
367+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) create
368+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) setup-schema -v 0.0
369+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(TEMPORAL_DB) update-schema -d ./schema/mysql/v8/temporal/versioned
370+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) drop -f
371+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) create
372+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) setup-schema -v 0.0
373+
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) --db $(VISIBILITY_DB) update-schema -d ./schema/mysql/v8/visibility/versioned
374+
364375
install-schema-postgresql: temporal-sql-tool
365376
@printf $(COLOR) "Install Postgres schema..."
366377
./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres --db $(TEMPORAL_DB) drop -f
@@ -430,6 +441,9 @@ start-es: temporal-server
430441
start-mysql: temporal-server
431442
./temporal-server --env development-mysql --allow-no-auth start
432443

444+
start-mysql8: temporal-server
445+
./temporal-server --env development-mysql8 --allow-no-auth start
446+
433447
start-mysql-es: temporal-server
434448
./temporal-server --env development-mysql-es --allow-no-auth start
435449

common/persistence/persistence-tests/persistenceTestBase.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func NewTestBaseWithSQL(options *TestBaseOptions) TestBase {
136136

137137
if options.DBPort == 0 {
138138
switch options.SQLDBPluginName {
139-
case mysql.PluginName:
139+
case mysql.PluginName, mysql.PluginNameV8:
140140
options.DBPort = environment.GetMySQLPort()
141141
case postgresql.PluginName:
142142
options.DBPort = environment.GetPostgreSQLPort()
@@ -148,7 +148,7 @@ func NewTestBaseWithSQL(options *TestBaseOptions) TestBase {
148148
}
149149
if options.DBHost == "" {
150150
switch options.SQLDBPluginName {
151-
case mysql.PluginName:
151+
case mysql.PluginName, mysql.PluginNameV8:
152152
options.DBHost = environment.GetMySQLAddress()
153153
case postgresql.PluginName:
154154
options.DBHost = environment.GetPostgreSQLAddress()

common/persistence/persistence-tests/setup.go

+17-3
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,10 @@ import (
3333
)
3434

3535
const (
36-
testMySQLUser = "temporal"
37-
testMySQLPassword = "temporal"
38-
testMySQLSchemaDir = "schema/mysql/v57"
36+
testMySQLUser = "temporal"
37+
testMySQLPassword = "temporal"
38+
testMySQLSchemaDir = "schema/mysql/v57"
39+
testMySQL8SchemaDir = "schema/mysql/v8"
3940

4041
testPostgreSQLUser = "temporal"
4142
testPostgreSQLPassword = "temporal"
@@ -61,6 +62,19 @@ func GetMySQLTestClusterOption() *TestBaseOptions {
6162
}
6263
}
6364

65+
// GetMySQL8TestClusterOption return test options
66+
func GetMySQL8TestClusterOption() *TestBaseOptions {
67+
return &TestBaseOptions{
68+
SQLDBPluginName: mysql.PluginNameV8,
69+
DBUsername: testMySQLUser,
70+
DBPassword: testMySQLPassword,
71+
DBHost: environment.GetMySQLAddress(),
72+
DBPort: environment.GetMySQLPort(),
73+
SchemaDir: testMySQL8SchemaDir,
74+
StoreType: config.StoreTypeSQL,
75+
}
76+
}
77+
6478
// GetPostgreSQLTestClusterOption return test options
6579
func GetPostgreSQLTestClusterOption() *TestBaseOptions {
6680
return &TestBaseOptions{

common/persistence/sql/sqlplugin/mysql/db.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333

3434
"go.temporal.io/server/common/persistence/schema"
3535
"go.temporal.io/server/common/persistence/sql/sqlplugin"
36-
mysqlschema "go.temporal.io/server/schema/mysql"
36+
mysqlschemaV57 "go.temporal.io/server/schema/mysql/v57"
3737
)
3838

3939
// db represents a logical connection to mysql database
@@ -120,9 +120,9 @@ func (mdb *db) DbName() string {
120120
func (mdb *db) ExpectedVersion() string {
121121
switch mdb.dbKind {
122122
case sqlplugin.DbKindMain:
123-
return mysqlschema.Version
123+
return mysqlschemaV57.Version
124124
case sqlplugin.DbKindVisibility:
125-
return mysqlschema.VisibilityVersion
125+
return mysqlschemaV57.VisibilityVersion
126126
default:
127127
panic(fmt.Sprintf("unknown db kind %v", mdb.dbKind))
128128
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// The MIT License
2+
//
3+
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
4+
//
5+
// Copyright (c) 2020 Uber Technologies, Inc.
6+
//
7+
// Permission is hereby granted, free of charge, to any person obtaining a copy
8+
// of this software and associated documentation files (the "Software"), to deal
9+
// in the Software without restriction, including without limitation the rights
10+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
// copies of the Software, and to permit persons to whom the Software is
12+
// furnished to do so, subject to the following conditions:
13+
//
14+
// The above copyright notice and this permission notice shall be included in
15+
// all copies or substantial portions of the Software.
16+
//
17+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
// THE SOFTWARE.
24+
25+
package mysql
26+
27+
import (
28+
"context"
29+
"fmt"
30+
31+
"github.com/jmoiron/sqlx"
32+
33+
"go.temporal.io/server/common/persistence/schema"
34+
"go.temporal.io/server/common/persistence/sql/sqlplugin"
35+
mysqlschemaV8 "go.temporal.io/server/schema/mysql/v8"
36+
)
37+
38+
// db represents a logical connection to mysql database
39+
type dbV8 struct {
40+
db
41+
}
42+
43+
var _ sqlplugin.AdminDB = (*dbV8)(nil)
44+
var _ sqlplugin.DB = (*dbV8)(nil)
45+
var _ sqlplugin.Tx = (*dbV8)(nil)
46+
47+
// newDB returns an instance of DB, which is a logical
48+
// connection to the underlying mysql database
49+
func newDBV8(
50+
dbKind sqlplugin.DbKind,
51+
dbName string,
52+
xdb *sqlx.DB,
53+
tx *sqlx.Tx,
54+
) *dbV8 {
55+
mdb := &dbV8{
56+
db: db{
57+
dbKind: dbKind,
58+
dbName: dbName,
59+
db: xdb,
60+
tx: tx,
61+
},
62+
}
63+
mdb.conn = xdb
64+
if tx != nil {
65+
mdb.conn = tx
66+
}
67+
mdb.converter = &converter{}
68+
return mdb
69+
}
70+
71+
// BeginTx starts a new transaction and returns a reference to the Tx object
72+
func (mdb *dbV8) BeginTx(ctx context.Context) (sqlplugin.Tx, error) {
73+
xtx, err := mdb.db.db.BeginTxx(ctx, nil)
74+
if err != nil {
75+
return nil, err
76+
}
77+
return newDBV8(mdb.dbKind, mdb.dbName, mdb.db.db, xtx), nil
78+
}
79+
80+
// PluginName returns the name of the mysql plugin
81+
func (mdb *dbV8) PluginName() string {
82+
return PluginNameV8
83+
}
84+
85+
// ExpectedVersion returns expected version.
86+
func (mdb *dbV8) ExpectedVersion() string {
87+
switch mdb.dbKind {
88+
case sqlplugin.DbKindMain:
89+
return mysqlschemaV8.Version
90+
case sqlplugin.DbKindVisibility:
91+
return mysqlschemaV8.VisibilityVersion
92+
default:
93+
panic(fmt.Sprintf("unknown db kind %v", mdb.dbKind))
94+
}
95+
}
96+
97+
// VerifyVersion verify schema version is up to date
98+
func (mdb *dbV8) VerifyVersion() error {
99+
expectedVersion := mdb.ExpectedVersion()
100+
return schema.VerifyCompatibleVersion(mdb, mdb.dbName, expectedVersion)
101+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// The MIT License
2+
//
3+
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
4+
//
5+
// Copyright (c) 2020 Uber Technologies, Inc.
6+
//
7+
// Permission is hereby granted, free of charge, to any person obtaining a copy
8+
// of this software and associated documentation files (the "Software"), to deal
9+
// in the Software without restriction, including without limitation the rights
10+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
// copies of the Software, and to permit persons to whom the Software is
12+
// furnished to do so, subject to the following conditions:
13+
//
14+
// The above copyright notice and this permission notice shall be included in
15+
// all copies or substantial portions of the Software.
16+
//
17+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
// THE SOFTWARE.
24+
25+
package mysql
26+
27+
import (
28+
"go.temporal.io/server/common/config"
29+
"go.temporal.io/server/common/persistence/sql"
30+
"go.temporal.io/server/common/persistence/sql/sqlplugin"
31+
"go.temporal.io/server/common/resolver"
32+
)
33+
34+
const (
35+
// PluginName is the name of the plugin
36+
PluginNameV8 = "mysql8"
37+
)
38+
39+
type pluginV8 struct {
40+
plugin
41+
}
42+
43+
var _ sqlplugin.Plugin = (*pluginV8)(nil)
44+
45+
func init() {
46+
sql.RegisterPlugin(PluginNameV8, &pluginV8{})
47+
}
48+
49+
// CreateDB initialize the db object
50+
func (p *pluginV8) CreateDB(
51+
dbKind sqlplugin.DbKind,
52+
cfg *config.SQL,
53+
r resolver.ServiceResolver,
54+
) (sqlplugin.DB, error) {
55+
conn, err := p.createDBConnection(cfg, r)
56+
if err != nil {
57+
return nil, err
58+
}
59+
db := newDBV8(dbKind, cfg.DatabaseName, conn, nil)
60+
return db, nil
61+
}
62+
63+
// CreateAdminDB initialize the db object
64+
func (p *pluginV8) CreateAdminDB(
65+
dbKind sqlplugin.DbKind,
66+
cfg *config.SQL,
67+
r resolver.ServiceResolver,
68+
) (sqlplugin.AdminDB, error) {
69+
conn, err := p.createDBConnection(cfg, r)
70+
if err != nil {
71+
return nil, err
72+
}
73+
db := newDBV8(dbKind, cfg.DatabaseName, conn, nil)
74+
return db, nil
75+
}

common/persistence/tests/mysql_test.go

+35
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,13 @@ func TestMySQLVisibilityPersistenceSuite(t *testing.T) {
152152
suite.Run(t, s)
153153
}
154154

155+
func TestMySQL8VisibilityPersistenceSuite(t *testing.T) {
156+
s := &VisibilityPersistenceSuite{
157+
TestBase: persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption()),
158+
}
159+
suite.Run(t, s)
160+
}
161+
155162
// TODO: Merge persistence-tests into the tests directory.
156163

157164
func TestMySQLHistoryV2PersistenceSuite(t *testing.T) {
@@ -182,6 +189,34 @@ func TestMySQLClusterMetadataPersistence(t *testing.T) {
182189
suite.Run(t, s)
183190
}
184191

192+
func TestMySQL8HistoryV2PersistenceSuite(t *testing.T) {
193+
s := new(persistencetests.HistoryV2PersistenceSuite)
194+
s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption())
195+
s.TestBase.Setup(nil)
196+
suite.Run(t, s)
197+
}
198+
199+
func TestMySQL8MetadataPersistenceSuiteV2(t *testing.T) {
200+
s := new(persistencetests.MetadataPersistenceSuiteV2)
201+
s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption())
202+
s.TestBase.Setup(nil)
203+
suite.Run(t, s)
204+
}
205+
206+
func TestMySQL8QueuePersistence(t *testing.T) {
207+
s := new(persistencetests.QueuePersistenceSuite)
208+
s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption())
209+
s.TestBase.Setup(nil)
210+
suite.Run(t, s)
211+
}
212+
213+
func TestMySQL8ClusterMetadataPersistence(t *testing.T) {
214+
s := new(persistencetests.ClusterMetadataManagerSuite)
215+
s.TestBase = persistencetests.NewTestBaseWithSQL(persistencetests.GetMySQL8TestClusterOption())
216+
s.TestBase.Setup(nil)
217+
suite.Run(t, s)
218+
}
219+
185220
// SQL Store tests
186221

187222
func TestMySQLNamespaceSuite(t *testing.T) {

common/persistence/tests/mysql_test_util.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import (
3838
p "go.temporal.io/server/common/persistence"
3939
"go.temporal.io/server/common/persistence/sql"
4040
"go.temporal.io/server/common/persistence/sql/sqlplugin"
41-
_ "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql"
41+
"go.temporal.io/server/common/persistence/sql/sqlplugin/mysql"
4242
"go.temporal.io/server/common/resolver"
4343
"go.temporal.io/server/common/shuffle"
4444
"go.temporal.io/server/environment"
@@ -56,8 +56,8 @@ const (
5656

5757
// TODO hard code this dir for now
5858
// need to merge persistence test config / initialization in one place
59-
testMySQLExecutionSchema = "../../../schema/mysql/v57/temporal/schema.sql"
60-
testMySQLVisibilitySchema = "../../../schema/mysql/v57/visibility/schema.sql"
59+
testMySQLExecutionSchema = "../../../schema/mysql/v8/temporal/schema.sql"
60+
testMySQLVisibilitySchema = "../../../schema/mysql/v8/visibility/schema.sql"
6161
)
6262

6363
type (
@@ -100,7 +100,7 @@ func NewMySQLConfig() *config.SQL {
100100
strconv.Itoa(environment.GetMySQLPort()),
101101
),
102102
ConnectProtocol: testMySQLConnectionProtocol,
103-
PluginName: "mysql",
103+
PluginName: mysql.PluginNameV8,
104104
DatabaseName: testMySQLDatabaseNamePrefix + shuffle.String(testMySQLDatabaseNameSuffix),
105105
}
106106
}

0 commit comments

Comments
 (0)