Improve URL check (#4613)

* Improve URL check

* production only

Author: mei23

src/misc/check-allowed-url.ts

@@ -0,0 +1,20 @@
+export function checkAllowedUrl(url: string | URL | undefined): boolean {
+	if (process.env.NODE_ENV !== 'production') return true;
+
+	try {
+		if (url == null) return false;
+
+		const u = typeof url === 'string' ? new URL(url) : url;
+		if (!u.protocol.match(/^https?:$/) || u.hostname === 'unix') {
+			return false;
+		}
+
+		if (u.port !== '' && !['80', '443'].includes(u.port)) {
+			return false;
+		}
+
+		return true;
+	} catch {
+		return false;
+	}
+}

src/misc/download-url.ts

@@ -7,13 +7,13 @@ import config from '../config';
 import * as chalk from 'chalk';
 import Logger from '../services/logger';
 import { checkPrivateIp } from './check-private-ip';
+import { checkAllowedUrl } from './check-allowed-url';
 
 const pipeline = util.promisify(stream.pipeline);
 
 export async function downloadUrl(url: string, path: string) {
-	const u = new URL(url);
-	if (!u.protocol.match(/^https?:$/) || u.hostname === 'unix') {
-		throw new StatusError('Invalid protocol', 400);
+	if (!checkAllowedUrl(url)) {
+		throw new StatusError('Invalid URL', 400);
 	}
 
 	const logger = new Logger('download-url');
@@ -43,6 +43,11 @@ export async function downloadUrl(url: string, path: string) {
 		},
 		http2: false,	// default
 		retry: 0,
+	}).on('redirect', (res: Got.Response, opts: Got.NormalizedOptions) => {
+		if (!checkAllowedUrl(opts.url)) {
+			logger.warn(`Invalid URL: ${opts.url}`);
+			req.destroy();
+		}
 	}).on('response', (res: Got.Response) => {
 		if (checkPrivateIp(res.ip)) {
 			logger.warn(`Blocked address: ${res.ip}`);

src/misc/fetch.ts

@@ -6,6 +6,7 @@ import { HttpProxyAgent } from 'http-proxy-agent';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import config from '../config';
 import { checkPrivateIp } from './check-private-ip';
+import { checkAllowedUrl } from './check-allowed-url';
 
 export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<any> {
 	const res = await getResponse({
@@ -42,9 +43,8 @@ const OPERATION_TIMEOUT = 60 * 1000;
 const MAX_RESPONSE_SIZE = 10 * 1024 * 1024;
 
 export async function getResponse(args: { url: string, method: 'GET' | 'POST', body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
-	const u = new URL(args.url);
-	if (!u.protocol.match(/^https?:$/) || u.hostname === 'unix') {
-		throw new StatusError('Invalid protocol', 400);
+	if (!checkAllowedUrl(args.url)) {
+		throw new StatusError('Invalid URL', 400);
 	}
 
 	const timeout = args.timeout || RESPONSE_TIMEOUT;
@@ -71,6 +71,12 @@ export async function getResponse(args: { url: string, method: 'GET' | 'POST', b
 		retry: 0,
 	});
 
+	req.on('redirect', (res, opts) => {
+		if (!checkAllowedUrl(opts.url)) {
+			req.cancel(`Invalid url: ${opts.url}`);
+		}
+	});
+
 	return await receiveResponce(req, args.size || MAX_RESPONSE_SIZE);
 }