Fix flaky TestReload (#2624)

Signed-off-by: albertteoh <albert.teoh@logz.io>

Author: albertteoh

pkg/config/tlscfg/cert_watcher.go

@@ -112,7 +112,12 @@ func (w *certWatcher) watchChangesLoop(rootCAs, clientCAs *x509.CertPool) {
 				w.mu.Unlock()
 				err = e
 			}
-			if err != nil {
+			if err == nil {
+				w.logger.Info("Loaded modified certificate",
+					zap.String("certificate", event.Name),
+					zap.String("event", event.Op.String()))
+
+			} else {
 				w.logger.Error("Failed to load certificate",
 					zap.String("certificate", event.Name),
 					zap.String("event", event.Op.String()),

pkg/config/tlscfg/cert_watcher_test.go

@@ -17,6 +17,7 @@ package tlscfg
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -92,9 +93,24 @@ func TestReload(t *testing.T) {
 	require.NoError(t, err)
 
 	waitUntil(func() bool {
-		return logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0
+		// Logged when both matching public and private keys are modified in the cert.
+		// If mismatched keys are present in the cert, the "Failed to load certificate" error will be logged instead.
+		return logObserver.FilterMessage("Loaded modified certificate").Len() > 0
 	}, 100, time.Millisecond*200)
-	assert.True(t, logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0)
+
+	// Logged when the cert is modified with the client's public key due to
+	// a mismatch with the existing server private key.
+	assert.True(t, logObserver.
+		FilterMessage("Failed to load certificate").
+		FilterField(zap.String("certificate", certFile.Name())).Len() > 0,
+		"Failed to find wanted logs. All logs: "+fmt.Sprint(logObserver.All()))
+
+	// Logged when the cert is modified with the client's private key,
+	// resulting in both public and private keys matching (from the client).
+	assert.True(t, logObserver.
+		FilterMessage("Loaded modified certificate").
+		FilterField(zap.String("certificate", keyFile.Name())).Len() > 0,
+		"Failed to find wanted logs. All logs: "+fmt.Sprint(logObserver.All()))
 
 	cert, err = tls.LoadX509KeyPair(filepath.Clean(clientCert), clientKey)
 	require.NoError(t, err)