Tokenize and build FTS query string for postgres and sqlite (#3933)

Author: rodrigozhou

common/persistence/visibility/store/sql/query_converter_postgresql.go

@@ -156,6 +156,23 @@ func (c *pgQueryConverter) convertTextComparisonExpr(
 	if !isSupportedTextOperator(expr.Operator) {
 		return nil, query.NewConverterError("invalid query")
 	}
+	valueExpr, ok := expr.Right.(*unsafeSQLString)
+	if !ok {
+		return nil, query.NewConverterError(
+			"%s: unexpected value type (expected string, got %s)",
+			query.InvalidExpressionErrMessage,
+			sqlparser.String(expr.Right),
+		)
+	}
+	tokens := tokenizeTextQueryString(valueExpr.Val)
+	if len(tokens) == 0 {
+		return nil, query.NewConverterError(
+			"%s: unexpected value for Text type search attribute (no tokens found in %s)",
+			query.InvalidExpressionErrMessage,
+			sqlparser.String(expr.Right),
+		)
+	}
+	valueExpr.Val = strings.Join(tokens, " | ")
 	var newExpr sqlparser.Expr = &sqlparser.ComparisonExpr{
 		Operator: ftsMatchOp,
 		Left:     expr.Left,

common/persistence/visibility/store/sql/query_converter_sqlite.go

@@ -179,22 +179,23 @@ func (c *sqliteQueryConverter) convertTextComparisonExpr(
 	}
 	colNameStr := sqlparser.String(colName)
 
-	value, err := query.ParseSqlValue(sqlparser.String(expr.Right))
-	if err != nil {
-		return nil, err
+	valueExpr, ok := expr.Right.(*unsafeSQLString)
+	if !ok {
+		return nil, query.NewConverterError(
+			"%s: unexpected value type (expected string, got %s)",
+			query.InvalidExpressionErrMessage,
+			sqlparser.String(expr.Right),
+		)
 	}
-
-	var ftsQuery string
-	switch v := value.(type) {
-	case string:
-		ftsQuery = fmt.Sprintf(`%s:(%s)`, colNameStr, v)
-	default:
+	tokens := tokenizeTextQueryString(valueExpr.Val)
+	if len(tokens) == 0 {
 		return nil, query.NewConverterError(
-			"%s: unexpected value type %T",
+			"%s: unexpected value for Text type search attribute (no tokens found in %s)",
 			query.InvalidExpressionErrMessage,
-			expr,
+			sqlparser.String(expr.Right),
 		)
 	}
+	ftsQuery := fmt.Sprintf("%s:(%s)", colNameStr, strings.Join(tokens, " OR "))
 
 	var oper string
 	switch expr.Operator {

common/persistence/visibility/store/sql/query_converter_util.go

@@ -25,6 +25,7 @@
 package sql
 
 import (
+	"strings"
 	"time"
 
 	"github.com/xwb1989/sqlparser"
@@ -94,3 +95,16 @@ func getMaxDatetimeValue() time.Time {
 	t, _ := time.Parse(time.RFC3339, "9999-12-31T23:59:59Z")
 	return t
 }
+
+// Simple tokenizer by spaces. It's a temporary solution as it doesn't cover tokenizer used by
+// PostgreSQL or SQLite.
+func tokenizeTextQueryString(s string) []string {
+	tokens := strings.Split(s, " ")
+	nonEmptyTokens := make([]string, 0, len(tokens))
+	for _, token := range tokens {
+		if token != "" {
+			nonEmptyTokens = append(nonEmptyTokens, token)
+		}
+	}
+	return nonEmptyTokens
+}