Skip to content

Commit 09653f9

Browse files
ssongliussongliu
authored
fix: 修改远程数据库依赖 (#1794)
1 parent e0ca950 commit 09653f9

File tree

11 files changed

+577
-35
lines changed

11 files changed

+577
-35
lines changed

‎backend/utils/mysql/client/remote.go

+12-10
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"database/sql"
66
"fmt"
77
"os"
8-
"path"
8+
"os/exec"
99
"strings"
1010
"time"
1111

@@ -14,8 +14,7 @@ import (
1414
"github.com/1Panel-dev/1Panel/backend/global"
1515
"github.com/1Panel-dev/1Panel/backend/utils/common"
1616
"github.com/1Panel-dev/1Panel/backend/utils/files"
17-
18-
"github.com/jarvanstack/mysqldump"
17+
"github.com/1Panel-dev/1Panel/backend/utils/mysql/helper"
1918
)
2019

2120
type Remote struct {
@@ -222,23 +221,26 @@ func (r *Remote) Backup(info BackupInfo) error {
222221

223222
f, _ := os.OpenFile(fileNameItem, os.O_RDWR|os.O_CREATE, 0755)
224223
defer f.Close()
225-
if err := mysqldump.Dump(dns, mysqldump.WithData(), mysqldump.WithDropTable(), mysqldump.WithWriter(f)); err != nil {
224+
if err := helper.Dump(dns, helper.WithData(), helper.WithDropTable(), helper.WithWriter(f)); err != nil {
226225
return err
227226
}
228227

229-
if err := fileOp.Compress([]string{fileNameItem}, info.TargetDir, info.FileName, files.Gz); err != nil {
230-
return err
228+
gzipCmd := exec.Command("gzip", fileNameItem)
229+
stdout, err := gzipCmd.CombinedOutput()
230+
if err != nil {
231+
return fmt.Errorf("gzip file %s failed, stdout: %v, err: %v", strings.TrimSuffix(info.FileName, ".gz"), string(stdout), err)
231232
}
232233
return nil
233234
}
234235

235236
func (r *Remote) Recover(info RecoverInfo) error {
236-
fileOp := files.NewFileOp()
237237
fileName := info.SourceFile
238238
if strings.HasSuffix(info.SourceFile, ".sql.gz") {
239239
fileName = strings.TrimSuffix(info.SourceFile, ".gz")
240-
if err := fileOp.Decompress(info.SourceFile, path.Dir(fileName), files.Gz); err != nil {
241-
return err
240+
gzipCmd := exec.Command("gunzip", info.SourceFile)
241+
stdout, err := gzipCmd.CombinedOutput()
242+
if err != nil {
243+
return fmt.Errorf("gunzip file %s failed, stdout: %v, err: %v", info.SourceFile, string(stdout), err)
242244
}
243245
}
244246
dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F")
@@ -247,7 +249,7 @@ func (r *Remote) Recover(info RecoverInfo) error {
247249
return err
248250
}
249251
defer f.Close()
250-
if err := mysqldump.Source(dns, f, mysqldump.WithMergeInsert(1000)); err != nil {
252+
if err := helper.Source(dns, f, helper.WithMergeInsert(1000)); err != nil {
251253
return err
252254
}
253255
return nil

‎backend/utils/mysql/helper/dump.go

+327
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
package helper
2+
3+
import (
4+
"bufio"
5+
"database/sql"
6+
"fmt"
7+
"io"
8+
"os"
9+
"strings"
10+
"time"
11+
12+
"github.com/1Panel-dev/1Panel/backend/global"
13+
_ "github.com/go-sql-driver/mysql"
14+
)
15+
16+
func init() {}
17+
18+
type dumpOption struct {
19+
isData bool
20+
21+
tables []string
22+
isAllTable bool
23+
isDropTable bool
24+
writer io.Writer
25+
}
26+
27+
type DumpOption func(*dumpOption)
28+
29+
func WithDropTable() DumpOption {
30+
return func(option *dumpOption) {
31+
option.isDropTable = true
32+
}
33+
}
34+
35+
func WithData() DumpOption {
36+
return func(option *dumpOption) {
37+
option.isData = true
38+
}
39+
}
40+
41+
func WithTables(tables ...string) DumpOption {
42+
return func(option *dumpOption) {
43+
option.tables = tables
44+
}
45+
}
46+
47+
func WithAllTable() DumpOption {
48+
return func(option *dumpOption) {
49+
option.isAllTable = true
50+
}
51+
}
52+
53+
func WithWriter(writer io.Writer) DumpOption {
54+
return func(option *dumpOption) {
55+
option.writer = writer
56+
}
57+
}
58+
59+
func Dump(dns string, opts ...DumpOption) error {
60+
start := time.Now()
61+
global.LOG.Infof("dump start at %s\n", start.Format("2006-01-02 15:04:05"))
62+
defer func() {
63+
end := time.Now()
64+
global.LOG.Infof("dump end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start))
65+
}()
66+
67+
var err error
68+
69+
var o dumpOption
70+
71+
for _, opt := range opts {
72+
opt(&o)
73+
}
74+
75+
if len(o.tables) == 0 {
76+
o.isAllTable = true
77+
}
78+
79+
if o.writer == nil {
80+
o.writer = os.Stdout
81+
}
82+
83+
buf := bufio.NewWriter(o.writer)
84+
defer buf.Flush()
85+
86+
_, _ = buf.WriteString("-- ----------------------------\n")
87+
_, _ = buf.WriteString("-- MySQL Database Dump\n")
88+
_, _ = buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n")
89+
_, _ = buf.WriteString("-- ----------------------------\n")
90+
_, _ = buf.WriteString("\n\n")
91+
92+
db, err := sql.Open("mysql", dns)
93+
if err != nil {
94+
global.LOG.Errorf("open mysql db failed, err: %v", err)
95+
return err
96+
}
97+
defer db.Close()
98+
99+
dbName, err := getDBNameFromDNS(dns)
100+
if err != nil {
101+
global.LOG.Errorf("get db name from dns failed, err: %v", err)
102+
return err
103+
}
104+
_, err = db.Exec(fmt.Sprintf("USE `%s`", dbName))
105+
if err != nil {
106+
global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err)
107+
return err
108+
}
109+
110+
var tables []string
111+
if o.isAllTable {
112+
tmp, err := getAllTables(db)
113+
if err != nil {
114+
global.LOG.Errorf("get all tables failed, err: %v", err)
115+
return err
116+
}
117+
tables = tmp
118+
} else {
119+
tables = o.tables
120+
}
121+
122+
for _, table := range tables {
123+
if o.isDropTable {
124+
_, _ = buf.WriteString(fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table))
125+
}
126+
127+
err = writeTableStruct(db, table, buf)
128+
if err != nil {
129+
global.LOG.Errorf("write table struct failed, err: %v", err)
130+
return err
131+
}
132+
133+
if o.isData {
134+
err = writeTableData(db, table, buf)
135+
if err != nil {
136+
global.LOG.Errorf("write table data failed, err: %v", err)
137+
return err
138+
}
139+
}
140+
}
141+
142+
_, _ = buf.WriteString("-- ----------------------------\n")
143+
_, _ = buf.WriteString("-- Dumped by mysqldump\n")
144+
_, _ = buf.WriteString("-- Cost Time: " + time.Since(start).String() + "\n")
145+
_, _ = buf.WriteString("-- ----------------------------\n")
146+
_ = buf.Flush()
147+
148+
return nil
149+
}
150+
151+
func getCreateTableSQL(db *sql.DB, table string) (string, error) {
152+
var createTableSQL string
153+
err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL)
154+
if err != nil {
155+
return "", err
156+
}
157+
createTableSQL = strings.Replace(createTableSQL, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)
158+
return createTableSQL, nil
159+
}
160+
161+
func getAllTables(db *sql.DB) ([]string, error) {
162+
var tables []string
163+
rows, err := db.Query("SHOW TABLES")
164+
if err != nil {
165+
return nil, err
166+
}
167+
defer rows.Close()
168+
169+
for rows.Next() {
170+
var table string
171+
err = rows.Scan(&table)
172+
if err != nil {
173+
return nil, err
174+
}
175+
tables = append(tables, table)
176+
}
177+
return tables, nil
178+
}
179+
180+
func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error {
181+
_, _ = buf.WriteString("-- ----------------------------\n")
182+
_, _ = buf.WriteString(fmt.Sprintf("-- Table structure for %s\n", table))
183+
_, _ = buf.WriteString("-- ----------------------------\n")
184+
185+
createTableSQL, err := getCreateTableSQL(db, table)
186+
if err != nil {
187+
global.LOG.Errorf("get create table sql failed, err: %v", err)
188+
return err
189+
}
190+
_, _ = buf.WriteString(createTableSQL)
191+
_, _ = buf.WriteString(";")
192+
193+
_, _ = buf.WriteString("\n\n")
194+
_, _ = buf.WriteString("\n\n")
195+
return nil
196+
}
197+
198+
func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error {
199+
_, _ = buf.WriteString("-- ----------------------------\n")
200+
_, _ = buf.WriteString(fmt.Sprintf("-- Records of %s\n", table))
201+
_, _ = buf.WriteString("-- ----------------------------\n")
202+
203+
lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table))
204+
if err != nil {
205+
global.LOG.Errorf("exec `select * from %s` failed, err: %v", table, err)
206+
return err
207+
}
208+
defer lineRows.Close()
209+
210+
var columns []string
211+
columns, err = lineRows.Columns()
212+
if err != nil {
213+
global.LOG.Errorf("get columes falied, err: %v", err)
214+
return err
215+
}
216+
columnTypes, err := lineRows.ColumnTypes()
217+
if err != nil {
218+
global.LOG.Errorf("get colume types failed, err: %v", err)
219+
return err
220+
}
221+
222+
var values [][]interface{}
223+
for lineRows.Next() {
224+
row := make([]interface{}, len(columns))
225+
rowPointers := make([]interface{}, len(columns))
226+
for i := range columns {
227+
rowPointers[i] = &row[i]
228+
}
229+
err = lineRows.Scan(rowPointers...)
230+
if err != nil {
231+
global.LOG.Errorf("scan row data failed, err: %v", err)
232+
return err
233+
}
234+
values = append(values, row)
235+
}
236+
237+
for _, row := range values {
238+
ssql := "INSERT INTO `" + table + "` VALUES ("
239+
240+
for i, col := range row {
241+
if col == nil {
242+
ssql += "NULL"
243+
} else {
244+
Type := columnTypes[i].DatabaseTypeName()
245+
Type = strings.Replace(Type, "UNSIGNED", "", -1)
246+
Type = strings.Replace(Type, " ", "", -1)
247+
switch Type {
248+
case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT":
249+
if bs, ok := col.([]byte); ok {
250+
ssql += string(bs)
251+
} else {
252+
ssql += fmt.Sprintf("%d", col)
253+
}
254+
case "FLOAT", "DOUBLE":
255+
if bs, ok := col.([]byte); ok {
256+
ssql += string(bs)
257+
} else {
258+
ssql += fmt.Sprintf("%f", col)
259+
}
260+
case "DECIMAL", "DEC":
261+
ssql += fmt.Sprintf("%s", col)
262+
263+
case "DATE":
264+
t, ok := col.(time.Time)
265+
if !ok {
266+
global.LOG.Errorf("the DATE type conversion failed., err: %v", err)
267+
return err
268+
}
269+
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02"))
270+
case "DATETIME":
271+
t, ok := col.(time.Time)
272+
if !ok {
273+
global.LOG.Errorf("the DATETIME type conversion failed., err: %v", err)
274+
return err
275+
}
276+
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05"))
277+
case "TIMESTAMP":
278+
t, ok := col.(time.Time)
279+
if !ok {
280+
global.LOG.Errorf("the TIMESTAMP type conversion failed., err: %v", err)
281+
return err
282+
}
283+
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05"))
284+
case "TIME":
285+
t, ok := col.([]byte)
286+
if !ok {
287+
global.LOG.Errorf("the TIME type conversion failed., err: %v", err)
288+
return err
289+
}
290+
ssql += fmt.Sprintf("'%s'", string(t))
291+
case "YEAR":
292+
t, ok := col.([]byte)
293+
if !ok {
294+
global.LOG.Errorf("the YEAR type conversion failed., err: %v", err)
295+
return err
296+
}
297+
ssql += string(t)
298+
case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT":
299+
ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1))
300+
case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB":
301+
ssql += fmt.Sprintf("0x%X", col)
302+
case "ENUM", "SET":
303+
ssql += fmt.Sprintf("'%s'", col)
304+
case "BOOL", "BOOLEAN":
305+
if col.(bool) {
306+
ssql += "true"
307+
} else {
308+
ssql += "false"
309+
}
310+
case "JSON":
311+
ssql += fmt.Sprintf("'%s'", col)
312+
default:
313+
global.LOG.Errorf("unsupported colume type: %s", Type)
314+
return fmt.Errorf("unsupported colume type: %s", Type)
315+
}
316+
}
317+
if i < len(row)-1 {
318+
ssql += ","
319+
}
320+
}
321+
ssql += ");\n"
322+
_, _ = buf.WriteString(ssql)
323+
}
324+
325+
_, _ = buf.WriteString("\n\n")
326+
return nil
327+
}

0 commit comments

Comments
 (0)