implement signing up by sending us an email

this is preferred over putting an email address in the web form: by letting the
user send us the first message, our address will be a known correpondent in
their email account. that should help their junk filter realize our signup
message and later notification messages are legit. it also makes it impossible
for users to mistype their own email. we make it easy for users: they just
click our link to compose a message and hit send in their email application.

we'll now monitor the inbox for messages with "signup" as subject.
some metrics related to incoming message handling were changed.
messages are only processed if they don't have headers that indicate they are
coming from a mailing list or are automated in any way. we also check are
either have a dmarc pass, or aligned relaxed spf or aligned relaxed dkim pass,
based on the authentication-results header the mail server must add to the
message.

this adds config options to disable signup through email or website (or both to
disable entirely).

this changes config option SubjectPrefix. it must now be the full prefix,
no ": " is added anymore.

this also increases likelyhood of being able to deliver messages to address
with non-ascii localparts by setting the smtputf8 & 8bitmime extension when
needed.

for issue #25

Author: mjl-

api.go

@@ -200,14 +200,51 @@ func (API) Signup(ctx context.Context, email string) {
 		logCheck(err, "closing smtp connectiong")
 	}()
 
-	var user User
-	var msg []byte
-	var mailFrom, sendID string
-	var eightbit, smtputf8 bool
-	var m Message
+	user, msg, mailFrom, eightbit, smtputf8, m, err := signup(ctx, email, "", true)
+	if serr, ok := err.(*sherpa.Error); ok {
+		panic(serr)
+	}
+	xcheckf(err, "adding user to database")
+
+	// Send the message.
+	submitctx, submitcancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer submitcancel()
+	if err := smtpSubmit(submitctx, smtpconn, true, mailFrom, email, msg, eightbit, smtputf8); err != nil {
+		logErrorx("submission for signup/passwordreset", err, "userid", user.ID)
+		if err := database.Delete(context.Background(), &m); err != nil {
+			logErrorx("removing metamessage added before submission error", err)
+		}
+		xcheckf(err, "submitting verification/password reset email")
+	}
+	slog.Info("submitted signup/passwordreset email", "userid", user.ID)
+}
+
+func signup(ctx context.Context, email string, origMessageID string, viaWebsite bool) (user User, msg []byte, mailFrom string, eightbit, smtputf8 bool, m Message, err error) {
+	var sendID string
+
+	// Code below can raise panics with sherpa.Error. Catch them an return as regular error.
+	defer func() {
+		x := recover()
+		if x == nil {
+			return
+		}
+		serr, ok := x.(*sherpa.Error)
+		if !ok || err != nil {
+			panic(x)
+		}
+		err = serr
+	}()
+
 	err = database.Write(ctx, func(tx *bstore.Tx) error {
 		user, err = bstore.QueryTx[User](tx).FilterNonzero(User{Email: email}).Get()
 		if err == bstore.ErrAbsent || err == nil && user.VerifyToken != "" {
+			if viaWebsite && config.SignupWebsiteDisabled {
+				return fmt.Errorf("signup via website currently disabled")
+			}
+			if !viaWebsite && config.SignupEmailDisabled {
+				return fmt.Errorf("signup via email currently disabled")
+			}
+
 			metaUnsubToken := user.MetaUnsubscribeToken
 			if metaUnsubToken == "" {
 				metaUnsubToken = xrandomID(16)
@@ -239,9 +276,9 @@ func (API) Signup(ctx context.Context, email string) {
 				return fmt.Errorf("adding user to database: %v", err)
 			}
 
-			subject, text, html, err := composeSignup(user)
+			subject, text, html, err := composeSignup(user, viaWebsite)
 			xcheckf(err, "composing signup text")
-			mailFrom, sendID, msg, eightbit, smtputf8, err = compose(true, user, subject, text, html)
+			mailFrom, sendID, msg, eightbit, smtputf8, err = compose(true, user, origMessageID, subject, text, html)
 			xcheckf(err, "composing signup message")
 
 			m = Message{
@@ -252,7 +289,11 @@ func (API) Signup(ctx context.Context, email string) {
 			if err := tx.Insert(&m); err != nil {
 				return fmt.Errorf("adding outgoing message to database: %v", err)
 			}
-			xaddUserLogf(tx, user.ID, "Signup through website")
+			msg := "Signup through email"
+			if viaWebsite {
+				msg = "Signup through website"
+			}
+			xaddUserLogf(tx, user.ID, msg)
 
 			return nil
 		}
@@ -266,9 +307,9 @@ func (API) Signup(ctx context.Context, email string) {
 			return fmt.Errorf("updating user in database: %v", err)
 		}
 
-		subject, text, html, err := composePasswordReset(user)
+		subject, text, html, err := composePasswordReset(user, viaWebsite)
 		xcheckf(err, "composing password reset text")
-		mailFrom, sendID, msg, eightbit, smtputf8, err = compose(true, user, subject, text, html)
+		mailFrom, sendID, msg, eightbit, smtputf8, err = compose(true, user, origMessageID, subject, text, html)
 		xcheckf(err, "composing password reset message")
 
 		m = Message{
@@ -279,23 +320,15 @@ func (API) Signup(ctx context.Context, email string) {
 		if err := tx.Insert(&m); err != nil {
 			return fmt.Errorf("adding outgoing message to database: %v", err)
 		}
-		xaddUserLogf(tx, user.ID, "Signup for existing account, sending password reset.")
+		msg := "Signup through email for existing account, sending password reset."
+		if viaWebsite {
+			msg = "Signup through website for existing account, sending password reset."
+		}
+		xaddUserLogf(tx, user.ID, msg)
 
 		return nil
 	})
-	xcheckf(err, "adding user to database")
-
-	// Send the message.
-	submitctx, submitcancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer submitcancel()
-	if err := smtpSubmit(submitctx, smtpconn, true, mailFrom, email, msg, eightbit, smtputf8); err != nil {
-		logErrorx("submission for signup/passwordreset", err, "userid", user.ID)
-		if err := database.Delete(context.Background(), &m); err != nil {
-			logErrorx("removing metamessage added before submission error", err)
-		}
-		xcheckf(err, "submitting verification/password reset email")
-	}
-	slog.Info("submitted signup/passwordreset email", "userid", user.ID)
+	return
 }
 
 // SignupEmail returns the email address for a verify token. So we can show it, and
@@ -464,10 +497,10 @@ func (API) RequestPasswordReset(ctx context.Context, prepToken, email string) {
 	}
 	xcheckf(err, "requesting password reset")
 
-	subject, text, html, err := composePasswordReset(user)
+	subject, text, html, err := composePasswordReset(user, true)
 	xcheckf(err, "composing password reset text")
 
-	mailFrom, sendID, msg, eightbit, smtputf8, err := compose(true, user, subject, text, html)
+	mailFrom, sendID, msg, eightbit, smtputf8, err := compose(true, user, "", subject, text, html)
 	xcheckf(err, "composing password reset message")
 
 	smtpTake()
@@ -981,17 +1014,20 @@ type Recent struct {
 }
 
 type Home struct {
-	Version            string
-	GoVersion          string
-	GoOS               string
-	GoArch             string
-	ServiceName        string
-	AdminName          string
-	AdminEmail         string
-	Note               string
-	SignupNote         string
-	SkipModulePrefixes []string
-	Recents            []Recent
+	Version               string
+	GoVersion             string
+	GoOS                  string
+	GoArch                string
+	ServiceName           string
+	AdminName             string
+	AdminEmail            string
+	Note                  string
+	SignupNote            string
+	SkipModulePrefixes    []string
+	SignupEmailDisabled   bool
+	SignupWebsiteDisabled bool
+	SignupAddress         string
+	Recents               []Recent
 }
 
 func _recents(ctx context.Context, n int) (recents []Recent) {
@@ -1015,12 +1051,15 @@ func _recents(ctx context.Context, n int) (recents []Recent) {
 // Home returns data for the home page.
 func (API) Home(ctx context.Context) (home Home) {
 	home = Home{
-		Version:            version,
-		GoVersion:          runtime.Version(),
-		GoOS:               runtime.GOOS,
-		GoArch:             runtime.GOARCH,
-		ServiceName:        config.ServiceName,
-		SkipModulePrefixes: config.SkipModulePrefixes,
+		Version:               version,
+		GoVersion:             runtime.Version(),
+		GoOS:                  runtime.GOOS,
+		GoArch:                runtime.GOARCH,
+		ServiceName:           config.ServiceName,
+		SkipModulePrefixes:    config.SkipModulePrefixes,
+		SignupEmailDisabled:   config.SignupEmailDisabled,
+		SignupWebsiteDisabled: config.SignupWebsiteDisabled,
+		SignupAddress:         smtp.Address{Localpart: config.Submission.From.ParsedLocalpartBase, Domain: config.Submission.From.DNSDomain}.String(),
 	}
 
 	home.Recents = _recents(ctx, 15)
@@ -1144,7 +1183,7 @@ func (API) TestSend(ctx context.Context, secret, kind, email string) {
 	subject, text, html, err := composeSample(kind, u, loginToken)
 	xcheckf(err, "compose text")
 
-	mailFrom, sendID, msg, eightbit, smtputf8, err := compose(kind != "moduleupdates", u, subject, text, html)
+	mailFrom, sendID, msg, eightbit, smtputf8, err := compose(kind != "moduleupdates", u, "", config.SubjectPrefix+subject, text, html)
 	xcheckf(err, "compose message")
 	slog.Info("composed test message", "sendid", sendID)
 

api.json

@@ -773,6 +773,27 @@
 						"string"
 					]
 				},
+				{
+					"Name": "SignupEmailDisabled",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "SignupWebsiteDisabled",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "SignupAddress",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
 				{
 					"Name": "Recents",
 					"Docs": "",

api.ts

@@ -69,6 +69,9 @@ export interface Home {
 	Note: string
 	SignupNote: string
 	SkipModulePrefixes?: string[] | null
+	SignupEmailDisabled: boolean
+	SignupWebsiteDisabled: boolean
+	SignupAddress: string
 	Recents?: Recent[] | null
 }
 
@@ -98,7 +101,7 @@ export const types: TypenameMap = {
 	"ModuleUpdateURLs": {"Name":"ModuleUpdateURLs","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UserID","Docs":"","Typewords":["int64"]},{"Name":"SubscriptionID","Docs":"","Typewords":["int64"]},{"Name":"LogRecordID","Docs":"","Typewords":["int64"]},{"Name":"Discovered","Docs":"","Typewords":["timestamp"]},{"Name":"Module","Docs":"","Typewords":["string"]},{"Name":"Version","Docs":"","Typewords":["string"]},{"Name":"MessageID","Docs":"","Typewords":["int64"]},{"Name":"RepoURL","Docs":"","Typewords":["string"]},{"Name":"TagURL","Docs":"","Typewords":["string"]},{"Name":"DocURL","Docs":"","Typewords":["string"]}]},
 	"UserLog": {"Name":"UserLog","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UserID","Docs":"","Typewords":["int64"]},{"Name":"Time","Docs":"","Typewords":["timestamp"]},{"Name":"Text","Docs":"","Typewords":["string"]}]},
 	"SubscriptionImport": {"Name":"SubscriptionImport","Docs":"","Fields":[{"Name":"GoMod","Docs":"","Typewords":["string"]},{"Name":"BelowModule","Docs":"","Typewords":["bool"]},{"Name":"OlderVersions","Docs":"","Typewords":["bool"]},{"Name":"Prerelease","Docs":"","Typewords":["bool"]},{"Name":"Pseudo","Docs":"","Typewords":["bool"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"Indirect","Docs":"","Typewords":["bool"]}]},
-	"Home": {"Name":"Home","Docs":"","Fields":[{"Name":"Version","Docs":"","Typewords":["string"]},{"Name":"GoVersion","Docs":"","Typewords":["string"]},{"Name":"GoOS","Docs":"","Typewords":["string"]},{"Name":"GoArch","Docs":"","Typewords":["string"]},{"Name":"ServiceName","Docs":"","Typewords":["string"]},{"Name":"AdminName","Docs":"","Typewords":["string"]},{"Name":"AdminEmail","Docs":"","Typewords":["string"]},{"Name":"Note","Docs":"","Typewords":["string"]},{"Name":"SignupNote","Docs":"","Typewords":["string"]},{"Name":"SkipModulePrefixes","Docs":"","Typewords":["[]","string"]},{"Name":"Recents","Docs":"","Typewords":["[]","Recent"]}]},
+	"Home": {"Name":"Home","Docs":"","Fields":[{"Name":"Version","Docs":"","Typewords":["string"]},{"Name":"GoVersion","Docs":"","Typewords":["string"]},{"Name":"GoOS","Docs":"","Typewords":["string"]},{"Name":"GoArch","Docs":"","Typewords":["string"]},{"Name":"ServiceName","Docs":"","Typewords":["string"]},{"Name":"AdminName","Docs":"","Typewords":["string"]},{"Name":"AdminEmail","Docs":"","Typewords":["string"]},{"Name":"Note","Docs":"","Typewords":["string"]},{"Name":"SignupNote","Docs":"","Typewords":["string"]},{"Name":"SkipModulePrefixes","Docs":"","Typewords":["[]","string"]},{"Name":"SignupEmailDisabled","Docs":"","Typewords":["bool"]},{"Name":"SignupWebsiteDisabled","Docs":"","Typewords":["bool"]},{"Name":"SignupAddress","Docs":"","Typewords":["string"]},{"Name":"Recents","Docs":"","Typewords":["[]","Recent"]}]},
 	"Recent": {"Name":"Recent","Docs":"","Fields":[{"Name":"Module","Docs":"","Typewords":["string"]},{"Name":"Version","Docs":"","Typewords":["string"]},{"Name":"Discovered","Docs":"","Typewords":["timestamp"]},{"Name":"RepoURL","Docs":"","Typewords":["string"]},{"Name":"TagURL","Docs":"","Typewords":["string"]},{"Name":"DocURL","Docs":"","Typewords":["string"]}]},
 	"Interval": {"Name":"Interval","Docs":"","Values":[{"Name":"IntervalImmediate","Value":"immediate","Docs":""},{"Name":"IntervalHour","Value":"hour","Docs":""},{"Name":"IntervalDay","Value":"day","Docs":""},{"Name":"IntervalWeek","Value":"week","Docs":""}]},
 }

compose.go

@@ -32,7 +32,7 @@ import (
 //
 // If configured, the message subject prefix is prepended to the subject before
 // adding it to the message.
-func compose(meta bool, user User, subject, text, html string) (mailFrom, sendID string, msg []byte, eightbit, smtputf8 bool, rerr error) {
+func compose(meta bool, user User, origMessageID, subject, text, html string) (mailFrom, sendID string, msg []byte, eightbit, smtputf8 bool, rerr error) {
 	// message.Composer uses panic for error handling...
 	defer func() {
 		x := recover()
@@ -48,9 +48,22 @@ func compose(meta bool, user User, subject, text, html string) (mailFrom, sendID
 	var buf bytes.Buffer
 	c := message.NewComposer(&buf, 0)
 
+	rcptToAddr, err := smtp.ParseAddress(user.Email)
+	if err != nil {
+		// This shouldn't fail, we've validated the address before.
+		return "", "", nil, false, false, fmt.Errorf("parsing recipient address: %v", err)
+	}
+	for _, ch := range string(rcptToAddr.Localpart + config.Submission.From.ParsedLocalpartBase) {
+		if ch >= 0x80 {
+			c.SMTPUTF8 = true
+			c.Has8bit = true
+			break
+		}
+	}
+
 	sendID = xrandomID(16)
 	mailFromAddr := smtp.Address{
-		Localpart: smtp.Localpart(config.Submission.From.LocalpartBase),
+		Localpart: config.Submission.From.ParsedLocalpartBase,
 		Domain:    config.Submission.From.DNSDomain,
 	}
 	// We send from our address that uses "+<id>" in the SMTP MAIL FROM address.
@@ -64,19 +77,14 @@ func compose(meta bool, user User, subject, text, html string) (mailFrom, sendID
 		{Address: config.Admin.AddressParsed},
 	})
 
-	rcptToAddr, err := smtp.ParseAddress(user.Email)
-	if err != nil {
-		// This shouldn't fail, we've validated the address before.
-		return "", "", nil, false, false, fmt.Errorf("parsing recipient address: %v", err)
-	}
 	c.HeaderAddrs("To", []message.NameAddress{{Address: rcptToAddr}})
-	var prefix string
-	if config.SubjectPrefix != "" {
-		prefix = config.SubjectPrefix + ": "
-	}
-	c.Subject(prefix + subject)
+	c.Subject(subject)
 	c.Header("Date", time.Now().Format(RFC5322Z))
 	c.Header("Message-ID", fmt.Sprintf("<%s@%s>", sendID, hostname))
+	if origMessageID != "" {
+		c.Header("In-Reply-To", fmt.Sprintf("<%s>", origMessageID))
+		c.Header("References", fmt.Sprintf("<%s>", origMessageID))
+	}
 	c.Header("User-Agent", "gopherwatch/"+version)
 
 	// Try to prevent out-of-office notifications. RFC 3834.
@@ -152,8 +160,11 @@ func composeRender(templText *texttemplate.Template, templHTML *htmltemplate.Tem
 	return textBuf.String(), pageBuf.String(), nil
 }
 
-func composeSignup(u User) (subject, text, html string, err error) {
-	subject = "Verify new account"
+func composeSignup(u User, fromWebsite bool) (subject, text, html string, err error) {
+	subject = config.SubjectPrefix + "Verify new account"
+	if !fromWebsite {
+		subject = "re: signup for " + config.ServiceName
+	}
 	args := struct {
 		BaseURL string
 		Subject string
@@ -163,8 +174,11 @@ func composeSignup(u User) (subject, text, html string, err error) {
 	return subject, text, html, err
 }
 
-func composePasswordReset(u User) (subject, text, html string, err error) {
-	subject = "Password reset requested"
+func composePasswordReset(u User, fromWebsite bool) (subject, text, html string, err error) {
+	subject = config.SubjectPrefix + "Password reset requested"
+	if !fromWebsite {
+		subject = "re: signup for " + config.ServiceName
+	}
 	args := struct {
 		BaseURL string
 		Subject string
@@ -244,7 +258,7 @@ func composeModuleUpdates(u User, loginToken string, updates []ModuleUpdate) (su
 		return l[i].Module < l[j].Module
 	})
 
-	subject = fmt.Sprintf("%d modules with %d new versions", len(l), len(updates))
+	subject = fmt.Sprintf("%s%d modules with %d new versions", config.SubjectPrefix, len(l), len(updates))
 
 	var truncated bool
 	if len(l) > 1100 {
@@ -267,9 +281,9 @@ func composeModuleUpdates(u User, loginToken string, updates []ModuleUpdate) (su
 func composeSample(kind string, user User, loginToken string) (subject, text, html string, err error) {
 	switch kind {
 	case "signup":
-		return composeSignup(user)
+		return composeSignup(user, true)
 	case "passwordreset":
-		return composePasswordReset(user)
+		return composePasswordReset(user, true)
 	case "moduleupdates":
 		updates := []ModuleUpdate{
 			{

go.mod

@@ -4,7 +4,7 @@ go 1.22.0
 
 require (
 	github.com/mjl-/bstore v0.0.4
-	github.com/mjl-/mox v0.0.10
+	github.com/mjl-/mox v0.0.11-0.20240313163553-2c9cb5b847a7
 	github.com/mjl-/sconf v0.0.5
 	github.com/mjl-/sherpa v0.6.7
 	github.com/mjl-/sherpadoc v0.0.12

go.sum

@@ -28,8 +28,8 @@ github.com/mjl-/adns v0.0.0-20240309142737-2a1aacf346dc h1:ghTx3KsrO0hSJW0bCFCGw
 github.com/mjl-/adns v0.0.0-20240309142737-2a1aacf346dc/go.mod h1:v47qUMJnipnmDTRGaHwpCwzE6oypa5K33mUvBfzZBn8=
 github.com/mjl-/bstore v0.0.4 h1:q+R1oAr8+E9yf9q+zxkVjQ18VFqD/E9KmGVoe4FIOBA=
 github.com/mjl-/bstore v0.0.4/go.mod h1:/cD25FNBaDfvL/plFRxI3Ba3E+wcB0XVOS8nJDqndg0=
-github.com/mjl-/mox v0.0.10 h1:n/PamNQyRScHI5rtPBSdD+EeMjkyV36hDLoJZ+NTyD0=
-github.com/mjl-/mox v0.0.10/go.mod h1:6YYUzVv+DBNoUmP8OHa3vKKHvQ4N4jwvjjcVVbA1xRc=
+github.com/mjl-/mox v0.0.11-0.20240313163553-2c9cb5b847a7 h1:Nbu7NY+XFZVveS7s/jX1cTTmIZuXRiDtu8yHCXaDS4w=
+github.com/mjl-/mox v0.0.11-0.20240313163553-2c9cb5b847a7/go.mod h1:6YYUzVv+DBNoUmP8OHa3vKKHvQ4N4jwvjjcVVbA1xRc=
 github.com/mjl-/sconf v0.0.5 h1:4CMUTENpSnaeP2g6RKtrs8udTxnJgjX2MCCovxGId6s=
 github.com/mjl-/sconf v0.0.5/go.mod h1:uF8OdWtLT8La3i4ln176i1pB0ps9pXGCaABEU55ZkE0=
 github.com/mjl-/sherpa v0.6.7 h1:C5F8XQdV5nCuS4fvB+ye/ziUQrajEhOoj/t2w5T14BY=