elasticapm: sanitize cookies and POST form fields

Sanitize cookies and POST form fields by matching
their names against a configurable regular expression.
By default, we match against:

  (?i:password|passwd|pwd|secret|.*key|.*token|.*session.*|.*credit.*|.*card.*)

The pattern can be overridden by specifying the
environment variable ELASTIC_APM_SANITIZE_FIELD_NAMES.
The value is treated as a regular expression, and
is wrapped as "(?i:<pattern>)" to match case-insensitively
by default. If a key should be matched case-sensitively,
the flag can be unset with the syntax "(?-i:<pattern>)".

Author: axw

README.md

@@ -44,21 +44,33 @@ The two most critical configuration attributes are the server URL and
 the secret token. All other attributes have default values, and are not
 required to enable tracing.
 
-Environment variable                    | Default | Description
-----------------------------------------|---------|------------------------------------------
-ELASTIC\_APM\_SERVER\_URL               |         | Base URL of the Elastic APM server. If unspecified, no tracing will take place.
-ELASTIC\_APM\_SECRET\_TOKEN             |         | The secret token for Elastic APM server.
-ELASTIC\_APM\_VERIFY\_SERVER\_CERT      | true    | Verify certificates when using https.
-ELASTIC\_APM\_FLUSH\_INTERVAL           | 10s     | Time to wait before sending transactions to the Elastic APM server. Transactions will be batched up and sent periodically.
-ELASTIC\_APM\_MAX\_QUEUE\_SIZE          | 500     | Maximum number of transactions to queue before sending to the Elastic APM server. Once this number is reached, any new transactions will replace old ones until the queue is flushed.
-ELASTIC\_APM\_TRANSACTION\_MAX\_SPANS   | 500     | Maximum number of spans to capture per transaction. After this is reached, new spans will not be created, and a dropped count will be incremented.
-ELASTIC\_APM\_TRANSACTION\_SAMPLE\_RATE | 1.0     | Number in the range 0.0-1.0 inclusive, controlling how many transactions should be sampled (i.e. include full detail.)
-ELASTIC\_APM\_ENVIRONMENT               |         | Environment name, e.g. "production".
-ELASTIC\_APM\_FRAMEWORK\_NAME           |         | Framework name, e.g. "gin".
-ELASTIC\_APM\_FRAMEWORK\_VERSION        |         | Framework version, e.g. "1.0".
-ELASTIC\_APM\_SERVICE\_NAME             |         | Service name, e.g. "my-service". If this is unspecified, the agent will report the program binary name as the service name.
-ELASTIC\_APM\_SERVICE\_VERSION          |         | Service version, e.g. "1.0".
-ELASTIC\_APM\_HOSTNAME                  |         | Override for the hostname.
+Environment variable                    | Default   | Description
+----------------------------------------|-----------|------------------------------------------
+ELASTIC\_APM\_SERVER\_URL               |           | Base URL of the Elastic APM server. If unspecified, no tracing will take place.
+ELASTIC\_APM\_SECRET\_TOKEN             |           | The secret token for Elastic APM server.
+ELASTIC\_APM\_VERIFY\_SERVER\_CERT      | true      | Verify certificates when using https.
+ELASTIC\_APM\_FLUSH\_INTERVAL           | 10s       | Time to wait before sending transactions to the Elastic APM server. Transactions will be batched up and sent periodically.
+ELASTIC\_APM\_MAX\_QUEUE\_SIZE          | 500       | Maximum number of transactions to queue before sending to the Elastic APM server. Once this number is reached, any new transactions will replace old ones until the queue is flushed.
+ELASTIC\_APM\_TRANSACTION\_MAX\_SPANS   | 500       | Maximum number of spans to capture per transaction. After this is reached, new spans will not be created, and a dropped count will be incremented.
+ELASTIC\_APM\_TRANSACTION\_SAMPLE\_RATE | 1.0       | Number in the range 0.0-1.0 inclusive, controlling how many transactions should be sampled (i.e. include full detail.)
+ELASTIC\_APM\_ENVIRONMENT               |           | Environment name, e.g. "production".
+ELASTIC\_APM\_FRAMEWORK\_NAME           |           | Framework name, e.g. "gin".
+ELASTIC\_APM\_FRAMEWORK\_VERSION        |           | Framework version, e.g. "1.0".
+ELASTIC\_APM\_SERVICE\_NAME             |           | Service name, e.g. "my-service". If this is unspecified, the agent will report the program binary name as the service name.
+ELASTIC\_APM\_SERVICE\_VERSION          |           | Service version, e.g. "1.0".
+ELASTIC\_APM\_HOSTNAME                  |           | Override for the hostname.
+ELASTIC\_APM\_SANITIZE\_FIELD\_NAMES    |[(1)](#(1))| A pattern to match names of cookies and form fields that should be redacted.
+
+<a name="(1)">(1)</a> ELASTIC\_APM\_SANITIZE\_FIELD\_NAMES
+
+By default, we redact the values of cookies and POST form fields that match the following regular expression:
+
+  `password|passwd|pwd|secret|.*key|.*token|.*session.*|.*credit.*|.*card.*`
+
+The pattern specified in ELASTIC\_APM\_SANITIZE\_FIELD\_NAMES is treated
+case-insensitively by default. To override this behavior and match case-sensitively,
+wrap the value like `(?-i:<value>)`. For a full definition of Go's regular
+expression syntax, see https://golang.org/pkg/regexp/syntax/.
 
 ## Instrumentation
 

env.go

@@ -1,9 +1,12 @@
 package elasticapm
 
 import (
+	"fmt"
 	"math/rand"
 	"os"
+	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -14,12 +17,27 @@ const (
 	envMaxQueueSize          = "ELASTIC_APM_MAX_QUEUE_SIZE"
 	envMaxSpans              = "ELASTIC_APM_TRANSACTION_MAX_SPANS"
 	envTransactionSampleRate = "ELASTIC_APM_TRANSACTION_SAMPLE_RATE"
+	envSanitizeFieldNames    = "ELASTIC_APM_SANITIZE_FIELD_NAMES"
 
 	defaultFlushInterval           = 10 * time.Second
 	defaultMaxTransactionQueueSize = 500
 	defaultMaxSpans                = 500
 )
 
+var (
+	defaultSanitizedFieldNames = regexp.MustCompile(fmt.Sprintf("(?i:%s)", strings.Join([]string{
+		"password",
+		"passwd",
+		"pwd",
+		"secret",
+		".*key",
+		".*token",
+		".*session.*",
+		".*credit.*",
+		".*card.*",
+	}, "|")))
+)
+
 func initialFlushInterval() (time.Duration, error) {
 	value := os.Getenv(envFlushInterval)
 	if value == "" {
@@ -85,3 +103,16 @@ func initialSampler() (Sampler, error) {
 	source := rand.NewSource(time.Now().Unix())
 	return NewRatioSampler(ratio, source), nil
 }
+
+func initialSanitizedFieldNamesRegexp() (*regexp.Regexp, error) {
+	value := os.Getenv(envSanitizeFieldNames)
+	if value == "" {
+		return defaultSanitizedFieldNames, nil
+	}
+	re, err := regexp.Compile(fmt.Sprintf("(?i:%s)", value))
+	if err != nil {
+		_, err = regexp.Compile(value)
+		return nil, errors.Wrapf(err, "invalid %s value", envSanitizeFieldNames)
+	}
+	return re, nil
+}

env_test.go

@@ -2,6 +2,8 @@ package elasticapm_test
 
 import (
 	"context"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"testing"
@@ -11,6 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/apm-agent-go"
+	"github.com/elastic/apm-agent-go/contrib/apmhttp"
 	"github.com/elastic/apm-agent-go/model"
 	"github.com/elastic/apm-agent-go/transport/transporttest"
 )
@@ -91,6 +94,39 @@ func testTracerTransactionRateEnv(t *testing.T, envValue string, ratio float64)
 	assert.InDelta(t, N*ratio, sampled, N*0.02) // allow 2% error
 }
 
+func TestTracerSanitizeFieldNamesEnvInvalid(t *testing.T) {
+	os.Setenv("ELASTIC_APM_SANITIZE_FIELD_NAMES", "oy(")
+	defer os.Unsetenv("ELASTIC_APM_SANITIZE_FIELD_NAMES")
+
+	_, err := elasticapm.NewTracer("tracer_testing", "")
+	assert.EqualError(t, err, "invalid ELASTIC_APM_SANITIZE_FIELD_NAMES value: error parsing regexp: missing closing ): `oy(`")
+}
+
+func TestTracerSanitizeFieldNamesEnv(t *testing.T) {
+	testTracerSanitizeFieldNamesEnv(t, "secRet", "[REDACTED]")
+	testTracerSanitizeFieldNamesEnv(t, "nada", "top")
+}
+
+func testTracerSanitizeFieldNamesEnv(t *testing.T, envValue, expect string) {
+	os.Setenv("ELASTIC_APM_SANITIZE_FIELD_NAMES", envValue)
+	defer os.Unsetenv("ELASTIC_APM_SANITIZE_FIELD_NAMES")
+
+	tracer, transport := transporttest.NewRecorderTracer()
+	defer tracer.Close()
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "http://server.testing/", nil)
+	req.AddCookie(&http.Cookie{Name: "secret", Value: "top"})
+	h := &apmhttp.Handler{Handler: http.NotFoundHandler(), Tracer: tracer}
+	h.ServeHTTP(w, req)
+	tracer.Flush(nil)
+
+	tx := transport.Payloads()[0].Transactions()[0]
+	assert.Equal(t, tx.Context.Request.Cookies, model.Cookies{
+		{Name: "secret", Value: expect},
+	})
+}
+
 func TestTracerServiceNameEnvSanitizationSpecified(t *testing.T) {
 	testTracerServiceNameSanitization(
 		t, "TestTracerServiceNameEnvSanitizationSpecified",

sanitizer.go

@@ -0,0 +1,44 @@
+package elasticapm
+
+import (
+	"bytes"
+	"regexp"
+
+	"github.com/elastic/apm-agent-go/model"
+)
+
+const redacted = "[REDACTED]"
+
+// sanitizeRequest sanitizes HTTP request data, redacting
+// the values of cookies and forms whose corresponding keys
+// match the given regular expression.
+func sanitizeRequest(r *model.Request, re *regexp.Regexp) {
+	var anyCookiesRedacted bool
+	for _, c := range r.Cookies {
+		if !re.MatchString(c.Name) {
+			continue
+		}
+		c.Value = redacted
+		anyCookiesRedacted = true
+	}
+	if anyCookiesRedacted && r.Headers != nil {
+		var b bytes.Buffer
+		for i, c := range r.Cookies {
+			if i != 0 {
+				b.WriteRune(';')
+			}
+			b.WriteString(c.String())
+		}
+		r.Headers.Cookie = b.String()
+	}
+	if r.Body != nil && r.Body.Form != nil {
+		for key, values := range r.Body.Form {
+			if !re.MatchString(key) {
+				continue
+			}
+			for i := range values {
+				values[i] = redacted
+			}
+		}
+	}
+}

sanitizer_test.go

@@ -0,0 +1,102 @@
+package elasticapm_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/apm-agent-go/contrib/apmhttp"
+	"github.com/elastic/apm-agent-go/model"
+	"github.com/elastic/apm-agent-go/transport/transporttest"
+)
+
+func TestSanitizeRequest(t *testing.T) {
+	tracer, transport := transporttest.NewRecorderTracer()
+	defer tracer.Close()
+
+	mux := http.NewServeMux()
+	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(http.StatusTeapot)
+	}))
+	h := &apmhttp.Handler{
+		Handler: mux,
+		Tracer:  tracer,
+	}
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "http://server.testing/", nil)
+	for _, c := range []*http.Cookie{
+		{Name: "secret", Value: "top"},
+		{Name: "Custom-Credit-Card-Number", Value: "top"},
+		{Name: "sessionid", Value: "123"},
+		{Name: "user_id", Value: "456"},
+	} {
+		req.AddCookie(c)
+	}
+	h.ServeHTTP(w, req)
+	tracer.Flush(nil)
+
+	payloads := transport.Payloads()
+	require.Len(t, payloads, 1)
+	transactions := payloads[0].Transactions()
+	require.Len(t, transactions, 1)
+
+	tx := transactions[0]
+	assert.Equal(t, tx.Context.Request.Cookies, model.Cookies{
+		{Name: "Custom-Credit-Card-Number", Value: "[REDACTED]"},
+		{Name: "secret", Value: "[REDACTED]"},
+		{Name: "sessionid", Value: "[REDACTED]"},
+		{Name: "user_id", Value: "456"},
+	})
+	assert.Equal(t,
+		"secret=[REDACTED];Custom-Credit-Card-Number=[REDACTED];sessionid=[REDACTED];user_id=456",
+		tx.Context.Request.Headers.Cookie,
+	)
+}
+
+func TestSetSanitizedFieldNamesNone(t *testing.T) {
+	testSetSanitizedFieldNames(t, "top")
+}
+
+func TestSetSanitizedFieldNamesCaseSensitivity(t *testing.T) {
+	// patterns are matched case-insensitively by default
+	testSetSanitizedFieldNames(t, "[REDACTED]", "Secret")
+
+	// patterns can be made case-sensitive by clearing the "i" flag.
+	testSetSanitizedFieldNames(t, "top", "(?-i:Secret)")
+}
+
+func testSetSanitizedFieldNames(t *testing.T, expect string, sanitized ...string) {
+	tracer, transport := transporttest.NewRecorderTracer()
+	defer tracer.Close()
+	tracer.SetSanitizedFieldNames(sanitized...)
+
+	mux := http.NewServeMux()
+	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(http.StatusTeapot)
+	}))
+	h := &apmhttp.Handler{
+		Handler: mux,
+		Tracer:  tracer,
+	}
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "http://server.testing/", nil)
+	req.AddCookie(&http.Cookie{Name: "secret", Value: "top"})
+	h.ServeHTTP(w, req)
+	tracer.Flush(nil)
+
+	payloads := transport.Payloads()
+	require.Len(t, payloads, 1)
+	transactions := payloads[0].Transactions()
+	require.Len(t, transactions, 1)
+
+	tx := transactions[0]
+	assert.Equal(t, tx.Context.Request.Cookies, model.Cookies{
+		{Name: "secret", Value: expect},
+	})
+	assert.Equal(t, "secret="+expect, tx.Context.Request.Headers.Cookie)
+}