feat: adding robots.txt override via admin control panel

locales/en-US.yml

@@ -1472,6 +1472,8 @@ trustedLinkUrlPatterns: "Link to external site warning exclusion list"
 trustedLinkUrlPatternsDescription: "Separate with spaces for an AND condition or with line breaks for an OR condition. Using surrounding keywords with slashes will turn them into a regular expression. If you write only the domain name, it will be a backward match."
 open: "Open"
 keyboardShortcuts: "Keyboard shortcuts"
+overrideRobotsTxt: "Override robots.txt"
+overrideRobotsTxtDescription: "Override the default robots.txt. Maximum 2048 characters."
 _nsfwOpenBehavior:
   click: "Click to open"
   doubleClick: "Double click to open"

locales/index.d.ts

@@ -5951,6 +5951,14 @@ export interface Locale extends ILocale {
      * キーボードショートカット
      */
     "keyboardShortcuts": string;
+    /**
+     * robots.txtの上書き
+     */
+    "overrideRobotsTxt": string;
+    /**
+     * robots.txtの内容を上書きします。最大2048文字です。
+     */
+    "overrideRobotsTxtDescription": string;
     "_nsfwOpenBehavior": {
         /**
          * タップして開く

locales/ja-JP.yml

@@ -1480,6 +1480,8 @@ trustedLinkUrlPatterns: "外部サイトへのリンク警告 除外リスト"
 trustedLinkUrlPatternsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。スラッシュで囲むと正規表現になります。ドメイン名だけ書くと後方一致になります。"
 open: "開く"
 keyboardShortcuts: "キーボードショートカット"
+overrideRobotsTxt: "robots.txtの上書き"
+overrideRobotsTxtDescription: "robots.txtの内容を上書きします。最大2048文字です。"
 
 _nsfwOpenBehavior:
   click: "タップして開く"

packages/backend/migration/1738098171990-robotsTxt.js

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: marie and other Sharkey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class RobotsTxt1738098171990 {
+    name = 'RobotsTxt1738098171990'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" character varying(2048)`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`);
+    }
+}

packages/backend/src/core/entities/MetaEntityService.ts

@@ -97,6 +97,7 @@ export class MetaEntityService {
 			mcaptchaSiteKey: instance.mcaptchaSitekey,
 			mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl,
 			enableRecaptcha: instance.enableRecaptcha,
+			robotsTxt: instance.robotsTxt,
 			recaptchaSiteKey: instance.recaptchaSiteKey,
 			enableTurnstile: instance.enableTurnstile,
 			turnstileSiteKey: instance.turnstileSiteKey,

packages/backend/src/models/Meta.ts

@@ -659,6 +659,11 @@ export class MiMeta {
 	})
 	public enableIdenticonGeneration: boolean;
 
+	@Column('varchar', {
+		length: 2048, nullable: true,
+	})
+	public robotsTxt: string | null;
+
 	@Column('jsonb', {
 		default: { },
 	})

packages/backend/src/models/json-schema/meta.ts

@@ -119,6 +119,10 @@ export const packedMetaLiteSchema = {
 			type: 'string',
 			optional: false, nullable: true,
 		},
+		robotsTxt: {
+			type: 'string',
+			optional: false, nullable: true,
+		},
 		enableTestcaptcha: {
 			type: 'boolean',
 			optional: false, nullable: false,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -396,6 +396,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			robotsTxt: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableIdenticonGeneration: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -774,6 +778,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				enableChartsForFederatedInstances: instance.enableChartsForFederatedInstances,
 				enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances,
 				enableServerMachineStats: instance.enableServerMachineStats,
+				robotsTxt: instance.robotsTxt,
 				enableIdenticonGeneration: instance.enableIdenticonGeneration,
 				bannedEmailDomains: instance.bannedEmailDomains,
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -159,6 +159,7 @@ export const paramDef = {
 		enableChartsForFederatedInstances: { type: 'boolean' },
 		enableStatsForFederatedInstances: { type: 'boolean' },
 		enableServerMachineStats: { type: 'boolean' },
+		robotsTxt: { type: 'string', nullable: true },
 		enableIdenticonGeneration: { type: 'boolean' },
 		serverRules: { type: 'array', items: { type: 'string' } },
 		bannedEmailDomains: { type: 'array', items: { type: 'string' } },
@@ -709,6 +710,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableServerMachineStats = ps.enableServerMachineStats;
 			}
 
+			if (ps.robotsTxt !== undefined) {
+				set.robotsTxt = ps.robotsTxt;
+			}
+
 			if (ps.enableIdenticonGeneration !== undefined) {
 				set.enableIdenticonGeneration = ps.enableIdenticonGeneration;
 			}

packages/backend/src/server/web/ClientServerService.ts

@@ -471,7 +471,14 @@ export class ClientServerService {
 		});
 
 		fastify.get('/robots.txt', async (request, reply) => {
-			return await reply.sendFile('/robots.txt', staticAssets);
+			if (this.meta.robotsTxt) {
+				let content = '';
+				content += this.meta.robotsTxt;
+				reply.header('Content-Type', 'text/plain');
+				return await reply.send(content);
+			} else {
+				return await reply.sendFile('/robots.txt', staticAssets);
+			}
 		});
 
 		// OpenSearch XML

packages/cherrypick-js/src/autogen/types.ts

@@ -5249,6 +5249,7 @@ export type components = {
       recaptchaSiteKey: string | null;
       enableTurnstile: boolean;
       turnstileSiteKey: string | null;
+      robotsTxt: string | null;
       enableTestcaptcha: boolean;
       swPublickey: string | null;
       /** @default /assets/ai.png */
@@ -9036,6 +9037,7 @@ export type operations = {
             enableChartsForFederatedInstances: boolean;
             enableStatsForFederatedInstances: boolean;
             enableServerMachineStats: boolean;
+            robotsTxt: string | null;
             enableIdenticonGeneration: boolean;
             manifestJsonOverride: string;
             policies: Record<string, never>;
@@ -11537,6 +11539,7 @@ export type operations = {
           enableChartsForFederatedInstances?: boolean;
           enableStatsForFederatedInstances?: boolean;
           enableServerMachineStats?: boolean;
+          robotsTxt?: string | null;
           enableIdenticonGeneration?: boolean;
           serverRules?: string[];
           bannedEmailDomains?: string[];

packages/frontend/src/pages/admin/settings.vue

@@ -271,6 +271,21 @@
 					</div>
 				</MkFolder>
 
+				<MkFolder>
+					<template #icon><i class="ti ti-adjustments"></i></template>
+					<template #label>{{ i18n.ts.otherSettings }}</template>
+					<template v-if="otherSettingsForm.modified.value" #footer>
+						<MkFormFooter :form="otherSettingsForm"/>
+					</template>
+
+					<div class="_gaps_m">
+						<MkTextarea v-model="otherSettingsForm.state.robotTxt">
+							<template #label>{{ i18n.ts.overrideRobotsTxt }}<span v-if="otherSettingsForm.modifiedStates.robotTxt" class="_modified">{{ i18n.ts.modified }}</span></template>
+							<template #caption>{{ i18n.ts.overrideRobotsTxtDescription }}</template>
+						</MkTextarea>
+					</div>
+				</MkFolder>
+
 				<MkFolder>
 					<template #icon><i class="ti ti-file-search"></i></template>
 					<template #label>{{ i18n.ts.search }}</template>
@@ -419,6 +434,15 @@
 	fetchInstance(true);
 });
 
+const otherSettingsForm = useForm({
+	robotTxt: meta.robotsTxt,
+}, async (state) => {
+	await os.apiWithDialog('admin/update-meta', {
+		robotsTxt: state.robotTxt,
+	});
+	fetchInstance(true);
+});
+
 function chooseProxyAccount() {
 	os.selectUser({ localOnly: true }).then(user => {
 		proxyAccount.value = user;
@@ -427,7 +451,7 @@
 		}).then(() => {
 			fetchInstance(true);
 		});
 	});
 
 }