From 3dd3b65c3988808513a1f5fe2720dafdd5126fa5 Mon Sep 17 00:00:00 2001 From: maksadbek Date: Wed, 7 Aug 2024 12:20:42 +0200 Subject: [PATCH] feat(proxy): add --proxy-domains flag for domain-specific proxying with wildcard support (#582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename “proxy-internal” to “proxy-target”. - Rename “proxy-external” to “proxy-bright”. - Add a new “proxy-domains” flag as an optional flag. It can be accepted when the user is setting either “proxy-target” or “proxy”. It can receive a comma-separated list of domains which will be proxied. Any domain that is not part of the list won’t be proxied. Notes: Support for wildcard is required. If I specify “*.foo.bar” as one of the domains, “a.foo.bar”, “b.foo.bar”, etc. will all be proxied. - Bright domains will always be proxied when using with the “proxy” flag, without a need to have them explicitly listed as part of the “proxy-domains” list. if the user doesn’t want to proxy bright domains, he should use only "proxy-target" --- src/Commands/GetEntryPoints.ts | 2 +- src/Commands/PollingScanStatus.ts | 2 +- src/Commands/RetestScan.ts | 2 +- src/Commands/RunRepeater.ts | 13 ++++++++++--- src/Commands/RunScan.ts | 2 +- src/Commands/StopScan.ts | 2 +- src/Commands/UploadArchive.ts | 2 +- src/Config/CliBuilder.ts | 6 ++++-- src/EntryPoint/RestEntryPoints.ts | 1 + src/Repeater/DefaultRepeaterServer.ts | 1 + src/RequestExecutor/HttpRequestExecutor.ts | 18 +++++++++++++++++- src/RequestExecutor/RequestExecutorOptions.ts | 1 + src/Scan/RestScans.ts | 1 + 13 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/Commands/GetEntryPoints.ts b/src/Commands/GetEntryPoints.ts index a152b4ed..f9e1469d 100644 --- a/src/Commands/GetEntryPoints.ts +++ b/src/Commands/GetEntryPoints.ts @@ -32,7 +32,7 @@ export class GetEntryPoints implements CommandModule { insecure: args.insecure as boolean, baseURL: args.api as string, apiKey: args.token as string, - proxyURL: (args.proxyExternal ?? args.proxy) as string + proxyURL: (args.proxyBright ?? args.proxy) as string } }) ); diff --git a/src/Commands/PollingScanStatus.ts b/src/Commands/PollingScanStatus.ts index 99862671..2a1a0868 100644 --- a/src/Commands/PollingScanStatus.ts +++ b/src/Commands/PollingScanStatus.ts @@ -53,7 +53,7 @@ export class PollingScanStatus implements CommandModule { insecure: args.insecure as boolean, baseURL: args.api as string, apiKey: args.token as string, - proxyURL: (args.proxyExternal ?? args.proxy) as string + proxyURL: (args.proxyBright ?? args.proxy) as string } }) ); diff --git a/src/Commands/RetestScan.ts b/src/Commands/RetestScan.ts index 411d11ad..deee8b2a 100644 --- a/src/Commands/RetestScan.ts +++ b/src/Commands/RetestScan.ts @@ -27,7 +27,7 @@ export class RetestScan implements CommandModule { insecure: args.insecure as boolean, baseURL: args.api as string, apiKey: args.token as string, - proxyURL: (args.proxyExternal ?? args.proxy) as string + proxyURL: (args.proxyBright ?? args.proxy) as string } }) ); diff --git a/src/Commands/RunRepeater.ts b/src/Commands/RunRepeater.ts index 52fd65f0..f1b13721 100644 --- a/src/Commands/RunRepeater.ts +++ b/src/Commands/RunRepeater.ts @@ -131,6 +131,12 @@ export class RunRepeater implements CommandModule { alias: ['rm', 'remove'], describe: 'Stop and remove repeater daemon' }) + .option('proxy-domains', { + requiresArg: true, + array: true, + describe: + 'Comma-separated list of domains that should be routed through the proxy. This option is only applicable when using the --proxy option' + }) .conflicts({ daemon: 'remove-daemon', ntlm: [ @@ -165,7 +171,7 @@ export class RunRepeater implements CommandModule { useValue: { headers: (args.header ?? args.headers) as Record, timeout: args.timeout as number, - proxyUrl: (args.proxyInternal ?? args.proxy) as string, + proxyUrl: (args.proxyTarget ?? args.proxy) as string, certs: args.cert as Cert[], maxContentLength: 100, reuseConnection: @@ -185,7 +191,8 @@ export class RunRepeater implements CommandModule { 'application/msgpack', 'application/ld+json', 'application/graphql' - ] + ], + proxyDomains: args.proxyDomains as string[] } }) .register( @@ -195,7 +202,7 @@ export class RunRepeater implements CommandModule { uri: args.repeaterServer as string, token: args.token as string, connectTimeout: 10000, - proxyUrl: (args.proxyExternal ?? args.proxy) as string, + proxyUrl: (args.proxyBright ?? args.proxy) as string, insecure: args.insecure as boolean } } diff --git a/src/Commands/RunScan.ts b/src/Commands/RunScan.ts index 8d8ce19c..5a55b8b0 100644 --- a/src/Commands/RunScan.ts +++ b/src/Commands/RunScan.ts @@ -172,7 +172,7 @@ export class RunScan implements CommandModule { insecure: args.insecure as boolean, baseURL: args.api as string, apiKey: args.token as string, - proxyURL: (args.proxyExternal ?? args.proxy) as string + proxyURL: (args.proxyBright ?? args.proxy) as string } }) ); diff --git a/src/Commands/StopScan.ts b/src/Commands/StopScan.ts index a4bc2598..9cb5c53b 100644 --- a/src/Commands/StopScan.ts +++ b/src/Commands/StopScan.ts @@ -27,7 +27,7 @@ export class StopScan implements CommandModule { insecure: args.insecure as boolean, baseURL: args.api as string, apiKey: args.token as string, - proxyURL: (args.proxyExternal ?? args.proxy) as string + proxyURL: (args.proxyBright ?? args.proxy) as string } }) ); diff --git a/src/Commands/UploadArchive.ts b/src/Commands/UploadArchive.ts index 21bd84c2..68cda0de 100644 --- a/src/Commands/UploadArchive.ts +++ b/src/Commands/UploadArchive.ts @@ -86,7 +86,7 @@ export class UploadArchive implements CommandModule { insecure: args.insecure as boolean, baseURL: args.api as string, apiKey: args.token as string, - proxyURL: (args.proxyExternal ?? args.proxy) as string + proxyURL: (args.proxyBright ?? args.proxy) as string } }); }); diff --git a/src/Config/CliBuilder.ts b/src/Config/CliBuilder.ts index 8e63e482..d827301d 100644 --- a/src/Config/CliBuilder.ts +++ b/src/Config/CliBuilder.ts @@ -64,12 +64,14 @@ export class CliBuilder { describe: 'Specify a proxy URL to route all traffic through. This should be an HTTP(S), SOCKS4, or SOCKS5 URL. By default, if you specify SOCKS://, then SOCKS5h is applied.' }) - .option('proxy-external', { + .option('proxy-bright', { + alias: 'proxy-external', requiresArg: true, describe: 'Specify a proxy URL to route all outbound traffic through. For more information, see the --proxy option.' }) - .option('proxy-internal', { + .option('proxy-target', { + alias: 'proxy-internal', requiresArg: true, describe: 'Specify a proxy URL to route all inbound traffic through. For more information, see the --proxy option.' diff --git a/src/EntryPoint/RestEntryPoints.ts b/src/EntryPoint/RestEntryPoints.ts index 77c8f42a..da36a30e 100644 --- a/src/EntryPoint/RestEntryPoints.ts +++ b/src/EntryPoint/RestEntryPoints.ts @@ -11,6 +11,7 @@ export interface RestProjectsOptions { timeout?: number; insecure?: boolean; proxyURL?: string; + proxyDomains?: string[]; } export const RestProjectsOptions: unique symbol = Symbol('RestProjectsOptions'); diff --git a/src/Repeater/DefaultRepeaterServer.ts b/src/Repeater/DefaultRepeaterServer.ts index f17531b7..f1273474 100644 --- a/src/Repeater/DefaultRepeaterServer.ts +++ b/src/Repeater/DefaultRepeaterServer.ts @@ -32,6 +32,7 @@ export interface DefaultRepeaterServerOptions { readonly connectTimeout?: number; readonly proxyUrl?: string; readonly insecure?: boolean; + readonly proxyDomains?: string[]; } export const DefaultRepeaterServerOptions: unique symbol = Symbol( diff --git a/src/RequestExecutor/HttpRequestExecutor.ts b/src/RequestExecutor/HttpRequestExecutor.ts index 191bef68..2b840fbe 100644 --- a/src/RequestExecutor/HttpRequestExecutor.ts +++ b/src/RequestExecutor/HttpRequestExecutor.ts @@ -1,7 +1,7 @@ import { RequestExecutor } from './RequestExecutor'; import { Response } from './Response'; import { Request, RequestOptions } from './Request'; -import { logger, ProxyFactory } from '../Utils'; +import { Helpers, logger, ProxyFactory } from '../Utils'; import { VirtualScripts } from '../Scripts'; import { Protocol } from './Protocol'; import { RequestExecutorOptions } from './RequestExecutorOptions'; @@ -39,6 +39,7 @@ export class HttpRequestExecutor implements RequestExecutor { private readonly httpsProxyAgent?: https.Agent; private readonly httpAgent?: http.Agent; private readonly httpsAgent?: https.Agent; + private readonly proxyDomains?: RegExp[]; get protocol(): Protocol { return Protocol.HTTP; @@ -65,6 +66,12 @@ export class HttpRequestExecutor implements RequestExecutor { this.httpsAgent = new https.Agent(agentOptions); this.httpAgent = new http.Agent(agentOptions); } + + if (this.options.proxyDomains) { + this.proxyDomains = this.options.proxyDomains.map((domain) => + Helpers.wildcardToRegExp(domain) + ); + } } public async execute(options: Request): Promise { @@ -190,6 +197,15 @@ export class HttpRequestExecutor implements RequestExecutor { } private getRequestAgent(options: Request) { + if ( + this.proxyDomains && + !this.proxyDomains.some((domain) => + domain.test(parseUrl(options.url).hostname) + ) + ) { + return options.secureEndpoint ? this.httpsAgent : this.httpAgent; + } + return options.secureEndpoint ? this.httpsProxyAgent ?? this.httpsAgent : this.httpProxyAgent ?? this.httpAgent; diff --git a/src/RequestExecutor/RequestExecutorOptions.ts b/src/RequestExecutor/RequestExecutorOptions.ts index 28c0832d..c14f9e23 100644 --- a/src/RequestExecutor/RequestExecutorOptions.ts +++ b/src/RequestExecutor/RequestExecutorOptions.ts @@ -8,6 +8,7 @@ export interface RequestExecutorOptions { whitelistMimes?: string[]; maxContentLength?: number; reuseConnection?: boolean; + proxyDomains?: string[]; } export const RequestExecutorOptions: unique symbol = Symbol( diff --git a/src/Scan/RestScans.ts b/src/Scan/RestScans.ts index 08a13d20..99e43fc2 100644 --- a/src/Scan/RestScans.ts +++ b/src/Scan/RestScans.ts @@ -22,6 +22,7 @@ export interface RestScansOptions { timeout?: number; insecure?: boolean; proxyURL?: string; + proxyDomains?: string[]; } export const RestScansOptions: unique symbol = Symbol('RestScansOptions');