From 087ab6e4e7fa4c3f6439bfce0df29ca214770edc Mon Sep 17 00:00:00 2001 From: blark Date: Tue, 4 Jun 2019 17:48:19 +0000 Subject: [PATCH] v0.3.0 --- CHANGELOG | 9 +- README.md | 8 +- aiodnsbrute/cli.py | 318 ++++++++++++++++++++++++++++---------- aiodnsbrute/logger.py | 31 ++++ aiodnsbrute/resolvers.txt | 6 +- setup.py | 4 +- 6 files changed, 288 insertions(+), 88 deletions(-) create mode 100644 aiodnsbrute/logger.py diff --git a/CHANGELOG b/CHANGELOG index f61698f..748623d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,7 @@ v0.1.0 - - first version v0.2.0 - - switched to f-strings - removed direct dns request feature - added ability to read dns resolvers from file (or stdin with `-r -`) @@ -11,3 +9,10 @@ v0.2.0 - added ability to send output to stdout with `-f -` (suppresses normal output) - added wildcard response detection (user configurable) - added domain test to see if parent domain actually exists (user configurable) + +v0.3.0 +- added gethostbyname lookup method which can identify when CNAMEs have been resolved by c-ares +- reformatted source code with [black](https://github.com/python/black) +- moved logger into its own class using attribute name instead of parameter for log types +- fixed CSV output (None list problem when using .get) +- added cname and aliases to output when -o is specified diff --git a/README.md b/README.md index 4d35396..c9f14d2 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Get help: $ aiodnsbrute --help - Usage: aiodnsbrute [OPTIONS] DOMAIN + Usage: cli.py [OPTIONS] DOMAIN aiodnsbrute is a command line tool for brute forcing domain names utilizing Python's asyncio module. @@ -51,6 +51,8 @@ Get help: automatically appended when not using -f). -f, --outfile FILENAME Output filename. Use '-f -' to send file output to stdout overriding normal output. + --query / --gethostbyname DNS lookup type to use query (default) should + be faster, but won't return CNAME information. --wildcard / --no-wildcard Wildcard detection, enabled by default --verify / --no-verify Verify domain name is sane before beginning, enabled by default @@ -83,6 +85,10 @@ Wildcard detection enabled by default (--no-wildcard turns it off): 100%|██████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:05<00:00, 140.18records/s] [*] Completed, 1 subdomains found +**NEW** use gethostbyname (detects CNAMEs which can be handy for potential subdomain takeover detection) + + $ aiodnsbrute --gethostbyname domain.com + Supply a list of resolvers from file (ignoring blank lines and starting with #), specify `-r -` to read list from stdin. $ aiodnsbrute -r resolvers.txt domain.com diff --git a/aiodnsbrute/cli.py b/aiodnsbrute/cli.py index 15d3bcb..c0045a5 100644 --- a/aiodnsbrute/cli.py +++ b/aiodnsbrute/cli.py @@ -9,12 +9,19 @@ import socket import sys from tqdm import tqdm +from logger import ConsoleLogger + class aioDNSBrute(object): - """Description goes here eventually...""" + """aiodnsbrute implements fast domain name brute forcing using Python's asyncio module.""" def __init__(self, verbosity=0, max_tasks=512): - self.verbosity = verbosity + """Constructor. + + Args: + verbosity: set output verbosity: 0 (default) is none, 3 is debug + max_tasks: the maximum number of tasks asyncio will queue (default 512) + """ self.tasks = [] self.errors = [] self.fqdn = [] @@ -24,160 +31,311 @@ def __init__(self, verbosity=0, max_tasks=512): self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True) self.sem = asyncio.BoundedSemaphore(max_tasks) self.max_tasks = max_tasks + self.verbosity = verbosity + self.new_logger = ConsoleLogger(verbosity) - def logger(self, msg, msg_type='info', level=1): - """A quick and dirty msfconsole style stdout logger.""" - if level <= self.verbosity: - style = {'info': ('[*]', 'blue'), 'pos': ('[+]', 'green'), 'err': ('[-]', 'red'), - 'warn': ('[!]', 'yellow'), 'dbg': ('[D]', 'cyan')} - if msg_type is not 0: - decorator = click.style(f'{style[msg_type][0]}', fg=style[msg_type][1], bold=True) - else: - decorator = '' - m = f'{decorator} {msg}' - tqdm.write(m) + async def _dns_lookup(self, name): + """Performs a DNS request using aiodns, self.lookup_type is set by the run function. + A query for A record returns which does not return metadata about + when a CNAME was resolved (just host and ttl attributes) however it should be faster. + The returned by gethostbyname contains name, aliases, and addresses, if + name is different in response we can surmise that the original domain was a CNAME entry. + + Args: + name: the domain name to resolve - async def _dns_lookup(self, name, _type='A'): - """Performs a DNS request using aiodns, returns an asyncio future.""" - response = await self.resolver.query(name, _type) - return response + Returns: + object: if query, if gethostbyname + """ + if self.lookup_type == "query": + return await self.resolver.query(name, "A") + elif self.lookup_type == "gethostbyname": + return await self.resolver.gethostbyname(name, socket.AF_INET) def _dns_result_callback(self, name, future): - """Handles the result passed by the _dns_lookup function.""" + """Handles the pycares object passed by the _dns_lookup function. We expect an errror to + be present in the returned object because most lookups will be for names that don't exist. + c-ares errors are passed through directly, error types can be identified in ares_strerror.c + + Args: + name: original lookup name (because the query_result object doesn't contain it) + future: the completed future (pycares dns result) + """ # Record processed we can now release the lock self.sem.release() # Handle known exceptions, barf on other ones if future.exception() is not None: try: - err_num = future.exception().args[0] + err_number = future.exception().args[0] err_text = future.exception().args[1] except IndexError: - self.logger(f'Couldn\'t parse exception: {future.exception()}', 'err') - if (err_num == 4): # This is domain name not found, ignore it. + self.new_logger.error(f"Couldn't parse exception: {future.exception()}") + # handle the DNS errors we expect to receive, show user unexpected errors + if err_number == 4: + # This is domain name not found, ignore it pass - elif (err_num == 12): # Timeout from DNS server - self.logger(f'Timeout for {name}', 'warn', 2) - elif (err_num == 1): # Server answered with no data + elif err_number == 12: + # Timeout from DNS server + self.new_logger.warn(f"Timeout for {name}") + elif err_number == 1: + # Server answered with no data pass else: - self.logger(f'{name} generated an unexpected exception: {future.exception()}', 'err') - #self.errors.append({'hostname': name, 'error': err_text}) - # Output result + self.new_logger.error( + f"{name} generated an unexpected exception: {future.exception()}" + ) + # for debugging/troubleshoooting keep a list of errors + # self.errors.append({'hostname': name, 'error': err_text}) + + # parse and output and store results. else: - ip = ', '.join([ip.host for ip in future.result()]) + if self.lookup_type == "query": + ip = ", ".join([ip.host for ip in future.result()]) + cname = False + row = f"{name:<30}\t{ip}" + elif self.lookup_type == "gethostbyname": + r = future.result() + ip = ", ".join([ip for ip in r.addresses]) + if name == r.name: + cname = False + n = f"""{name:<30}\t{f"{'':<35}" if self.verbosity >= 2 else ""}""" + else: + cname = True + # format the name based on verbosity - this is kluge + short_cname = f"{r.name[:28]}.." if len(r.name) > 30 else r.name + n = f'{name}{"**" if self.verbosity <= 1 else ""}' + n = f'{n:<30}\t{f"CNAME {short_cname:<30}" if self.verbosity >= 2 else ""}' + row = f"{n:<30}\t{ip}" + # store the result if ip not in self.ignore_hosts: - self.logger(f'{name:<30}\t{ip}', 'pos') - self.fqdn.append({'domain': name, 'ip': [ip]}) - self.logger(future.result(), 'dbg', 3) + self.new_logger.success(row) + dns_lookup_result = {"domain": name, "ip": [ip]} + if self.lookup_type == "gethostbyname" and cname: + dns_lookup_result["cname"] = r.name + dns_lookup_result["aliases"] = r.aliases + self.fqdn.append(dns_lookup_result) + self.new_logger.debug(future.result()) self.tasks.remove(future) if self.verbosity >= 1: self.pbar.update() - async def _process_dns_wordlist(self, wordlist, domain): - """Takes a list of words and adds them to the task list as space is available""" + async def _queue_lookups(self, wordlist, domain): + """Takes a list of words and adds them to the async loop also passing the original + lookup domain name; then attaches the processing callback to deal with the result. + + Args: + wordlist: a list of names to perform lookups for + domain: the base domain to perform brute force against + """ for word in wordlist: # Wait on the semaphore before adding more tasks await self.sem.acquire() - host = f'{word.strip()}.{domain}' + host = f"{word.strip()}.{domain}" task = asyncio.ensure_future(self._dns_lookup(host)) task.add_done_callback(functools.partial(self._dns_result_callback, host)) self.tasks.append(task) await asyncio.gather(*self.tasks, return_exceptions=True) - def run(self, wordlist, domain, resolvers=None, wildcard=True, verify=True): - self.logger(f'Brute forcing {domain} with a maximum of {self.max_tasks} concurrent tasks...') + def run( + self, wordlist, domain, resolvers=None, wildcard=True, verify=True, query=True + ): + """ + Sets up the bruteforce job, does domain verification, sets resolvers, checks for wildcard + response to lookups, and sets the query type to be used. After all this, open the wordlist + file and start the brute force - with ^C handling to cleanup nicely. + + Args: + wordlist: a string containing a path to a filename to be used as a wordlist + domain: the base domain name to be used for lookups + resolvers: a list of DNS resolvers to be used (default None, uses system resolvers) + wildcard: bool, do wildcard dns detection (default true) + verify: bool, check if domain exists (default true) + query: bool, use query to do lookups (default true), false means gethostbyname is used. + + Returns: + dict containing result of lookups + """ + self.new_logger.info( + f"Brute forcing {domain} with a maximum of {self.max_tasks} concurrent tasks..." + ) if verify: - self.logger(f'Using local resolver to verify {domain} exists.') + self.new_logger.info(f"Using local resolver to verify {domain} exists.") try: socket.gethostbyname(domain) except socket.gaierror as err: - self.logger(f'Couldn\'t resolve {domain}, use the --no-verify switch to ignore this error.', 'err') - raise SystemExit(self.logger(f'Error from host lookup: {err}', 'err')) + self.new_logger.error( + f"Couldn't resolve {domain}, use the --no-verify switch to ignore this error." + ) + raise SystemExit( + self.new_logger.error(f"Error from host lookup: {err}") + ) else: - self.logger('Skipping domain verification. YOLO!', 'warn') + self.new_logger.warn("Skipping domain verification. YOLO!") if resolvers: self.resolver.nameservers = resolvers - self.logger(f'Using recursive DNS with the following servers: {self.resolver.nameservers}') + self.new_logger.info( + f"Using recursive DNS with the following servers: {self.resolver.nameservers}" + ) + + if query: + self.new_logger.info( + "Using pycares `query` function to perform lookups, CNAMEs cannot be identified" + ) + self.lookup_type = "query" + else: + self.new_logger.info( + "Using pycares `gethostbyname` function to perform lookups, CNAME data will be appended to results (** denotes CNAME, show actual name with -vv)" + ) + self.lookup_type = "gethostbyname" + if wildcard: # 63 chars is the max allowed segment length, there is practically no chance that it will be a legit record - random_sld = lambda: f'{"".join(random.choice(string.ascii_lowercase + string.digits) for i in range(63))}' + random_sld = ( + lambda: f'{"".join(random.choice(string.ascii_lowercase + string.digits) for i in range(63))}' + ) try: - wc_check = self.loop.run_until_complete(self._dns_lookup(f'{random_sld()}.{domain}')) + wc_check = self.loop.run_until_complete( + self._dns_lookup(f"{random_sld()}.{domain}") + ) except aiodns.error.DNSError as err: # we expect that the record will not exist and error 4 will be thrown - self.logger(f'No wildcard response was detected for this domain.') + self.new_logger.info( + f"No wildcard response was detected for this domain." + ) wc_check = None finally: if wc_check is not None: self.ignore_hosts = [host.host for host in wc_check] - self.logger(f'Wildcard response detected, ignoring answers containing {self.ignore_hosts}', 'warn') + self.new_logger.warn( + f"Wildcard response detected, ignoring answers containing {self.ignore_hosts}" + ) else: - self.logger('Wildcard detection is disabled', 'warn') + self.new_logger.warn("Wildcard detection is disabled") - with open(wordlist, encoding='utf-8', errors='ignore') as words: + with open(wordlist, encoding="utf-8", errors="ignore") as words: w = words.read().splitlines() - self.logger(f'Wordlist loaded, proceeding with {len(w)} DNS requests') + self.new_logger.info(f"Wordlist loaded, proceeding with {len(w)} DNS requests") try: if self.verbosity >= 1: - self.pbar = tqdm(total=len(w), unit="records", maxinterval=0.1, mininterval=0) - self.loop.run_until_complete(self._process_dns_wordlist(w, domain)) + self.pbar = tqdm( + total=len(w), unit="rec", maxinterval=0.1, mininterval=0 + ) + self.loop.run_until_complete(self._queue_lookups(w, domain)) except KeyboardInterrupt: - self.logger("Caught keyboard interrupt, cleaning up...") + self.new_logger.warn("Caught keyboard interrupt, cleaning up...") asyncio.gather(*asyncio.Task.all_tasks()).cancel() self.loop.stop() finally: self.loop.close() if self.verbosity >= 1: self.pbar.close() - self.logger(f'Completed, {len(self.fqdn)} subdomains found') + self.new_logger.info(f"Completed, {len(self.fqdn)} subdomains found") return self.fqdn -## NOTE: Remember to remove recursive stuff @click.command() -@click.option('--wordlist', '-w', help='Wordlist to use for brute force.', - default=f'{os.path.dirname(os.path.realpath(__file__))}/wordlists/bitquark_20160227_subdomains_popular_1000') -@click.option('--max-tasks', '-t', default=512, - help='Maximum number of tasks to run asynchronosly.') -@click.option('--resolver-file', '-r', type=click.File('r'), default=None, help="A text file containing a list of DNS resolvers to use, one per line, comments start with #. Default: use system resolvers") -@click.option('--verbosity', '-v', count=True, default=1, help="Increase output verbosity") -@click.option('--output', '-o', type=click.Choice(['csv', 'json', 'off']), default='off', help="Output results to DOMAIN.csv/json (extension automatically appended when not using -f).") -@click.option('--outfile', '-f', type=click.File('w'), help="Output filename. Use '-f -' to send file output to stdout overriding normal output.") -@click.option('--wildcard/--no-wildcard', default=True, help="Wildcard detection, enabled by default") -@click.option('--verify/--no-verify', default=True, help="Verify domain name is sane before beginning, enabled by default") -@click.version_option('0.2.1') -@click.argument('domain', required=True) +@click.option( + "--wordlist", + "-w", + help="Wordlist to use for brute force.", + default=f"{os.path.dirname(os.path.realpath(__file__))}/wordlists/bitquark_20160227_subdomains_popular_1000", +) +@click.option( + "--max-tasks", + "-t", + default=512, + help="Maximum number of tasks to run asynchronosly.", +) +@click.option( + "--resolver-file", + "-r", + type=click.File("r"), + default=None, + help="A text file containing a list of DNS resolvers to use, one per line, comments start with #. Default: use system resolvers", +) +@click.option( + "--verbosity", "-v", count=True, default=1, help="Increase output verbosity" +) +@click.option( + "--output", + "-o", + type=click.Choice(["csv", "json", "off"]), + default="off", + help="Output results to DOMAIN.csv/json (extension automatically appended when not using -f).", +) +@click.option( + "--outfile", + "-f", + type=click.File("w"), + help="Output filename. Use '-f -' to send file output to stdout overriding normal output.", +) +@click.option( + "--query/--gethostbyname", + default=True, + help="DNS lookup type to use query (default) should be faster, but won't return CNAME information.", +) +@click.option( + "--wildcard/--no-wildcard", + default=True, + help="Wildcard detection, enabled by default", +) +@click.option( + "--verify/--no-verify", + default=True, + help="Verify domain name is sane before beginning, enabled by default", +) +@click.version_option("0.3.0") +@click.argument("domain", required=True) def main(**kwargs): """aiodnsbrute is a command line tool for brute forcing domain names utilizing Python's asyncio module. credit: blark (@markbaseggio) """ - output = kwargs.get('output') - verbosity = kwargs.get('verbosity') - resolvers = kwargs.get('resolver_file') - if output is not 'off': - outfile = kwargs.get('outfile') + output = kwargs.get("output") + verbosity = kwargs.get("verbosity") + resolvers = kwargs.get("resolver_file") + if output is not "off": + outfile = kwargs.get("outfile") # turn off output if we want JSON/CSV to stdout, hacky - if outfile.__class__.__name__ == 'TextIOWrapper': + if outfile.__class__.__name__ == "TextIOWrapper": verbosity = 0 if outfile is None: # wasn't specified on command line - outfile = open(f'{kwargs["domain"]}.{output}', 'w') + outfile = open(f'{kwargs["domain"]}.{output}', "w") if resolvers: lines = resolvers.read().splitlines() - resolvers = [x.strip() for x in lines if (x and not x.startswith('#'))] + resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))] - bf = aioDNSBrute(verbosity=verbosity, max_tasks=kwargs.get('max_tasks')) - results = bf.run(wordlist=kwargs.get('wordlist'), domain=kwargs.get('domain'), resolvers=resolvers, wildcard=kwargs.get('wildcard'), verify=kwargs.get('verify')) + bf = aioDNSBrute(verbosity=verbosity, max_tasks=kwargs.get("max_tasks")) + results = bf.run( + wordlist=kwargs.get("wordlist"), + domain=kwargs.get("domain"), + resolvers=resolvers, + wildcard=kwargs.get("wildcard"), + verify=kwargs.get("verify"), + query=kwargs.get("query"), + ) - if output in ('json'): + if output in ("json"): import json json.dump(results, outfile) - if output in ('csv'): + + if output in ("csv"): import csv writer = csv.writer(outfile) - writer.writerow(['Hostname', 'IPs']) - [writer.writerow([r.get('domain'), r.get('ip')[0]]) for r in results] + writer.writerow(["Hostname", "IPs", "CNAME", "Aliases"]) + [ + writer.writerow( + [ + r.get("domain"), + r.get("ip", [""])[0], + r.get("cname"), + r.get("aliases", [""])[0], + ] + ) + for r in results + ] + -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/aiodnsbrute/logger.py b/aiodnsbrute/logger.py new file mode 100644 index 0000000..e95736a --- /dev/null +++ b/aiodnsbrute/logger.py @@ -0,0 +1,31 @@ +from tqdm import tqdm +from click import style + + +class ConsoleLogger(object): + """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output.""" + + def __init__(self, verbosity): + self.verbosity = verbosity + self.msg_type = { + "info": ("[*]", "blue", 1), + "success": ("[+]", "green", 1), + "error": ("[-]", "red", 1), + "warn": ("[!]", "yellow", 1), + "debug": ("[D]", "cyan", 3), + } + + def __getattr__(self, attr): + try: + decorator = style( + f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True + ) + msg_verbosity = self.msg_type[attr][2] + except KeyError: + decorator = "" + msg_verbosity = 1 + finally: + if self.verbosity >= msg_verbosity: + return lambda msg: tqdm.write(f"{decorator}{msg}") + else: + return lambda msg: None diff --git a/aiodnsbrute/resolvers.txt b/aiodnsbrute/resolvers.txt index 3efd1b5..27dce68 100644 --- a/aiodnsbrute/resolvers.txt +++ b/aiodnsbrute/resolvers.txt @@ -6,6 +6,6 @@ 1.1.1.1 1.0.0.1 -# google -8.8.4.4 -8.8.8.8 +# google - they seem to rate limit, avoid +#8.8.4.4 +#8.8.8.8 diff --git a/setup.py b/setup.py index 664eeb6..11c385f 100644 --- a/setup.py +++ b/setup.py @@ -7,9 +7,9 @@ setup( name='aiodnsbrute', - version='0.2.1', + version='0.3.0', url='https://github.com/blark/aiodnsbrute', - download_url='https://github.com/blark/aiodnsbrute/archive/v0.2.1.tar.gz', + download_url='https://github.com/blark/aiodnsbrute/archive/v0.3.0.tar.gz', license='BSD', author='Mark Baseggio', author_email='mark@basegg.io',