diff --git a/README.md b/README.md index 8a4ca9e..3653f54 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ DESCRIPTION python3 oneforall.py --target example.com --valid None run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port small run - python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --fmt csv run python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --takeover False run @@ -179,7 +179,7 @@ DESCRIPTION Note: 参数alive可选值True，False分别表示导出存活，全部子域结果 参数port可选值有'default', 'small', 'large', 详见config.py配置 - 参数format可选格式有 'csv','json' + 参数fmt可选格式有 'csv','json' 参数path默认None使用OneForAll结果目录生成路径 ARGUMENTS @@ -199,7 +199,7 @@ FLAGS 请求验证子域的端口范围(默认只探测80端口) --valid=VALID 只导出存活的子域结果(默认False) - --format=FORMAT + --fmt=FMT 结果保存格式(默认csv) --path=PATH 结果保存路径(默认None) @@ -269,7 +269,6 @@ FLAGS ## ⌛后续计划 - [ ] 各模块持续优化和完善 -- [x] 子域监控（标记每次新发现的子域） - [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域） - [ ] 操作强大交互人性的前端界面实现 diff --git a/brute.py b/brute.py index fbcaa4e..4269982 100644 --- a/brute.py +++ b/brute.py @@ -384,7 +384,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): info = dict() cnames = list() ips = list() - public = list() times = list() ttls = list() is_valid_flags = list() @@ -401,7 +400,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): cnames.append(cname) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) - public.append(utils.ip_is_public(ip)) num = ip_times.get(ip) times.append(num) isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname) @@ -416,7 +414,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): info['ttl'] = ttls info['cname'] = cnames info['ip'] = ips - info['public'] = public info['times'] = times info['resolver'] = resolver infos[qname] = info @@ -559,7 +556,7 @@ class Brute(Module): brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run Note: - --format csv/json (result format) + --fmt csv/json (result fmt) --path Result path (default None, automatically generated) @@ -578,13 +575,13 @@ class Brute(Module): :param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode) :param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py) :param bool export: Export the results (default True) - :param str format: Result format (default csv) + :param str fmt: Result format (default csv) :param str path: Result directory (default None) """ def __init__(self, target=None, targets=None, process=None, concurrent=None, word=False, wordlist=None, recursive=False, depth=None, nextlist=None, fuzz=False, place=None, rule=None, fuzzlist=None, export=True, - alive=True, format='csv', path=None): + alive=True, fmt='csv', path=None): Module.__init__(self) self.module = 'Brute' self.source = 'Brute' @@ -603,7 +600,7 @@ class Brute(Module): self.fuzzlist = fuzzlist or settings.fuzz_list self.export = export self.alive = alive - self.format = format + self.fmt = fmt self.path = path self.bulk = False # 是否是批量爆破场景 self.domains = list() # 待爆破的所有域名集合 @@ -761,7 +758,7 @@ class Brute(Module): logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}') if not self.path: - name = f'{self.domain}_brute_result.{self.format}' + name = f'{self.domain}_brute_result.{self.fmt}' self.path = settings.result_save_dir.joinpath(name) # 数据库导出 if self.export: @@ -770,7 +767,7 @@ class Brute(Module): alive=self.alive, limit='resolve', path=self.path, - format=self.format) + fmt=self.fmt) if __name__ == '__main__': diff --git a/common/database.py b/common/database.py index 983a6ca..25244b7 100644 --- a/common/database.py +++ b/common/database.py @@ -57,7 +57,6 @@ class Database(object): f'alive int,' f'request int,' f'resolve int,' - f'new int,' f'url text,' f'subdomain text,' f'port int,' @@ -86,6 +85,20 @@ class Database(object): f'elapse float,' f'find int)') + def insert_table(self, table_name, result): + table_name = table_name.replace('.', '_') + self.conn.query( + f'insert into "{table_name}" ' + f'(id, alive, resolve, request, url, subdomain, port, level,' + f'cname, ip, public, cdn, status, reason, title, banner, header,' + f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,' + f'module, source, elapse, find) ' + f'values (:id, :alive, :resolve, :request, :url,' + f':subdomain, :port, :level, :cname, :ip, :public, :cdn,' + f':status, :reason, :title, :banner, :header, :history, :response,' + f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,' + f':source, :elapse, :find)', **result) + def save_db(self, table_name, results, module_name=None): """ Save the results of each module in the database @@ -101,11 +114,11 @@ class Database(object): try: self.conn.bulk_query( f'insert into "{table_name}" ' - f'(id, alive, resolve, request, new, url, subdomain, port, level,' + f'(id, alive, resolve, request, url, subdomain, port, level,' f'cname, ip, public, cdn, status, reason, title, banner, header,' f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,' f'module, source, elapse, find) ' - f'values (:id, :alive, :resolve, :request, :new, :url,' + f'values (:id, :alive, :resolve, :request, :url,' f':subdomain, :port, :level, :cname, :ip, :public, :cdn,' f':status, :reason, :title, :banner, :header, :history, :response,' f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,' @@ -178,7 +191,7 @@ class Database(object): def deduplicate_subdomain(self, table_name): """ - Deduplicates of subdomains in the table + Deduplicate subdomains in the table :param str table_name: table name """ @@ -199,17 +212,6 @@ class Database(object): self.query(f'delete from "{table_name}" where ' f'subdomain is null or resolve == 0') - def deal_table(self, deal_table_name, backup_table_name): - """ - Process the table when the collection task is complete - - :param str deal_table_name: Pending table name - :param str backup_table_name: Table name for backup - """ - self.copy_table(deal_table_name, backup_table_name) - self.remove_invalid(deal_table_name) - self.deduplicate_subdomain(deal_table_name) - def get_data(self, table_name): """ Get all the data in the table @@ -218,7 +220,7 @@ class Database(object): """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'Get all the data from {table_name} table') - return self.query(f'select * from "{table_name}"') + return self.query(f'select * from {table_name}') def export_data(self, table_name, alive, limit): """ @@ -229,18 +231,42 @@ class Database(object): :param str limit: limit value """ table_name = table_name.replace('.', '_') - query = f'select id, new, alive, request, resolve, url, subdomain, level,' \ - f'cname, ip, public, cdn, port, status, reason, title, banner,' \ - f'cidr, asn, org, addr, isp, source from "{table_name}"' + sql = f'select id, alive, request, resolve, url, subdomain, level,' \ + f'cname, ip, public, cdn, port, status, reason, title, banner,' \ + f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain' if alive and limit: if limit in ['resolve', 'request']: where = f' where {limit} = 1' - query += where + sql += where elif alive: where = f' where alive = 1' - query += where + sql += where logger.log('TRACE', f'Get the data from {table_name} table') - return self.query(query) + return self.query(sql) + + def count_alive(self, table_name): + table_name = table_name.replace('.', '_') + sql = f'select count() from {table_name} where alive = 1' + return self.query(sql) + + def get_resp_by_url(self, table_name, url): + table_name = table_name.replace('.', '_') + sql = f'select response from {table_name} where url = "{url}"' + logger.log('TRACE', f'Get response data from {url}') + return self.query(sql).scalar() + + def get_data_by_fields(self, table_name, fields): + table_name = table_name.replace('.', '_') + field_str = ', '.join(fields) + sql = f"select {field_str} from {table_name}" + logger.log('TRACE', f'Get specified field data {fields} from {table_name} table') + return self.query(sql) + + def update_data_by_url(self, table_name, info, url): + table_name = table_name.replace('.', '_') + field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items())) + sql = f'update {table_name} set {field_str} where url = "{url}"' + return self.query(sql) def close(self): """ diff --git a/common/module.py b/common/module.py index 29dac45..194757d 100644 --- a/common/module.py +++ b/common/module.py @@ -275,7 +275,6 @@ class Module(object): 'alive': None, 'request': None, 'resolve': None, - 'new': None, 'url': None, 'subdomain': None, 'port': None, @@ -315,25 +314,22 @@ class Module(object): ip = info.get('ip') times = info.get('times') ttl = info.get('ttl') - public = info.get('public') if isinstance(cname, list): cname = ','.join(cname) ip = ','.join(ip) times = ','.join([str(num) for num in times]) ttl = ','.join([str(num) for num in ttl]) - public = ','.join([str(num) for num in public]) result = {'id': None, 'alive': info.get('alive'), 'request': info.get('request'), 'resolve': info.get('resolve'), - 'new': None, 'url': url, 'subdomain': subdomain, 'port': 80, 'level': level, 'cname': cname, 'ip': ip, - 'public': public, + 'public': info.get('public'), 'cdn': info.get('cdn'), 'status': None, 'reason': info.get('reason'), diff --git a/common/request.py b/common/request.py index c6fe98c..8346884 100644 --- a/common/request.py +++ b/common/request.py @@ -1,7 +1,6 @@ import json from threading import Thread from queue import Queue -from operator import attrgetter import tqdm import requests @@ -9,6 +8,7 @@ from bs4 import BeautifulSoup from common import utils from config.log import logger +from common.database import Database from config import settings @@ -127,7 +127,7 @@ def get_progress_bar(total): return bar -def get(url, resp_list, session): +def get_resp(url, session): timeout = settings.request_timeout_second redirect = settings.request_allow_redirect proxy = utils.get_proxy() @@ -136,13 +136,14 @@ def get(url, resp_list, session): except Exception as e: logger.log('DEBUG', e.args) resp = e - resp_list.append((url, resp)) + return resp -def request(urls_queue, resp_list, session): +def request(urls_queue, resp_queue, session): while not urls_queue.empty(): - url = urls_queue.get() - get(url, resp_list, session) + index, url = urls_queue.get() + resp = get_resp(url, session) + resp_queue.put((index, resp)) urls_queue.task_done() @@ -168,31 +169,6 @@ def get_session(): return session -def bulk_request(urls): - logger.log('INFOR', 'Requesting urls in bulk') - resp_list = list() - urls_queue = Queue() - for url in urls: - urls_queue.put(url) - total = len(urls) - session = get_session() - thread_count = req_thread_count() - bar = get_progress_bar(total) - - progress_thread = Thread(target=progress, name='ProgressThread', - args=(bar, total, urls_queue), daemon=True) - progress_thread.start() - - for i in range(thread_count): - request_thread = Thread(target=request, name=f'RequestThread-{i}', - args=(urls_queue, resp_list, session), daemon=True) - request_thread.start() - - urls_queue.join() - - return resp_list - - def gen_new_info(info, resp): if isinstance(resp, Exception): info['reason'] = str(resp.args) @@ -220,13 +196,54 @@ def gen_new_info(info, resp): return info -def gen_new_data(data, resp_list): - new_data = list() - for url, resp in resp_list: - for info in data: - if info.get('url') == url: - new_data.append(gen_new_info(info, resp)) - return new_data +def save(name, total, req_data, resp_queue): + db = Database() + db.create_table(name) + i = 0 + while True: + if not resp_queue.empty(): + i += 1 + index, resp = resp_queue.get() + old_info = req_data[index] + new_info = gen_new_info(old_info, resp) + db.insert_table(name, new_info) + resp_queue.task_done() + if i >= total: + break + db.close() + + +def bulk_request(domain, req_data, ret=False): + logger.log('INFOR', 'Requesting urls in bulk') + resp_queue = Queue() + urls_queue = Queue() + total = len(req_data) + for index, info in enumerate(req_data): + try: + url = info.get('url') + except Exception: + pass + urls_queue.put((index, url)) + session = get_session() + thread_count = req_thread_count() + bar = get_progress_bar(total) + + progress_thread = Thread(target=progress, name='ProgressThread', + args=(bar, total, urls_queue), daemon=True) + progress_thread.start() + + for i in range(thread_count): + request_thread = Thread(target=request, name=f'RequestThread-{i}', + args=(urls_queue, resp_queue, session), daemon=True) + request_thread.start() + if ret: + urls_queue.join() + return resp_queue + save_thread = Thread(target=save, name=f'SaveThread', + args=(domain, total, req_data, resp_queue), daemon=True) + save_thread.start() + urls_queue.join() + save_thread.join() def run_request(domain, data, port): @@ -242,20 +259,6 @@ def run_request(domain, data, port): data = utils.set_id_none(data) ports = get_port_seq(port) req_data, req_urls = gen_req_data(data, ports) - resp_list = bulk_request(req_urls) - new_data = gen_new_data(req_data, resp_list) - count = utils.count_alive(new_data) + bulk_request(domain, req_data) + count = utils.count_alive(domain) logger.log('INFOR', f'Found that {domain} has {count} alive subdomains') - sorted_data = utils.sort_by_subdomain(new_data) - return sorted_data - - -def save_db(name, data): - """ - Save request results to database - - :param str name: table name - :param list data: data to be saved - """ - logger.log('INFOR', f'Saving requested results') - utils.save_db(name, data, 'request') diff --git a/common/resolve.py b/common/resolve.py index 3345796..ff52553 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -4,12 +4,6 @@ import json from config.log import logger from config import settings from common import utils -from common.ipasn import IPAsnInfo -from common.ipreg import IpRegData - - -ip_asn = IPAsnInfo() -ip_reg = IpRegData() def filter_subdomain(data): @@ -79,13 +73,7 @@ def gen_infos(data, qname, info, infos): flag = False cname = list() ips = list() - public = list() ttl = list() - cidr = list() - asn = list() - org = list() - addr = list() - isp = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': @@ -94,25 +82,11 @@ def gen_infos(data, qname, info, infos): ip = answer.get('data') ips.append(ip) ttl.append(str(answer.get('ttl'))) - public.append(str(utils.ip_is_public(ip))) - asn_info = ip_asn.find(ip) - cidr.append(asn_info.get('cidr')) - asn.append(asn_info.get('asn')) - org.append(asn_info.get('org')) - ip_info = ip_reg.query(ip) - addr.append(ip_info.get('addr')) - isp.append(ip_info.get('isp')) info['resolve'] = 1 info['reason'] = 'OK' info['cname'] = ','.join(cname) info['ip'] = ','.join(ips) - info['public'] = ','.join(public) info['ttl'] = ','.join(ttl) - info['cidr'] = ','.join(cidr) - info['asn'] = ','.join(asn) - info['org'] = ','.join(org) - info['addr'] = ','.join(addr) - info['isp'] = ','.join(isp) infos[qname] = info if not flag: info['alive'] = 0 diff --git a/common/utils.py b/common/utils.py index 74c03c4..1f12a1f 100644 --- a/common/utils.py +++ b/common/utils.py @@ -176,16 +176,16 @@ def check_dir(dir_path): dir_path.mkdir(parents=True, exist_ok=True) -def check_path(path, name, format): +def check_path(path, name, fmt): """ 检查结果输出目录路径 :param path: 保存路径 :param name: 导出名字 - :param format: 保存格式 + :param fmt: 保存格式 :return: 保存路径 """ - filename = f'{name}.{format}' + filename = f'{name}.{fmt}' default_path = settings.result_save_dir.joinpath(filename) if isinstance(path, str): path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠 @@ -204,19 +204,18 @@ def check_path(path, name, format): return path -def check_format(format, count): +def check_format(fmt): """ 检查导出格式 - :param format: 传入的导出格式 - :param count: 数量 + :param fmt: 传入的导出格式 :return: 导出格式 """ formats = ['csv', 'json', ] - if format in formats: - return format + if fmt in formats: + return fmt else: - logger.log('ALERT', f'Does not support {format} format') + logger.log('ALERT', f'Does not support {fmt} format') logger.log('ALERT', 'So use csv format by default') return 'csv' @@ -333,8 +332,8 @@ def remove_invalid_string(string): return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string) -def export_all_results(path, name, format, datas): - path = check_path(path, name, format) +def export_all_results(path, name, fmt, datas): + path = check_path(path, name, fmt) logger.log('ALERT', f'The subdomain result for all main domains: {path}') row_list = list() for row in datas: @@ -346,7 +345,7 @@ def export_all_results(path, name, format, datas): values = row.values() row_list.append(Record(keys, values)) rows = RecordCollection(iter(row_list)) - content = rows.export(format) + content = rows.export(fmt) save_data(path, content) @@ -366,19 +365,19 @@ def export_all_subdomains(alive, path, name, datas): save_data(path, data) -def export_all(alive, format, path, datas): +def export_all(alive, fmt, path, datas): """ 将所有结果数据导出 :param bool alive: 只导出存活子域结果 - :param str format: 导出文件格式 + :param str fmt: 导出文件格式 :param str path: 导出文件路径 :param list datas: 待导出的结果数据 """ - format = check_format(format, len(datas)) + fmt = check_format(fmt) timestamp = get_timestring() name = f'all_subdomain_result_{timestamp}' - export_all_results(path, name, format, datas) + export_all_results(path, name, fmt, datas) export_all_subdomains(alive, path, name, datas) @@ -430,10 +429,18 @@ def python_version(): return sys.version -def count_alive(data): +def calc_alive(data): return len(list(filter(lambda item: item.get('alive') == 1, data))) +def count_alive(name): + db = Database() + result = db.count_alive(name) + count = result.scalar() + db.close() + return count + + def get_subdomains(data): return set(map(lambda item: item.get('subdomain'), data)) @@ -833,3 +840,23 @@ def looks_like_ip(maybe_ip): return True except socket.error: return False + + +def deal_data(domain): + db = Database() + db.remove_invalid(domain) + db.deduplicate_subdomain(domain) + db.close() + + +def get_data(domain): + db = Database() + data = db.get_data(domain).as_dict() + db.close() + return data + + +def clear_data(domain): + db = Database() + db.drop_table(domain) + db.close() diff --git a/config/default.py b/config/default.py index 902ea7c..11b859f 100644 --- a/config/default.py +++ b/config/default.py @@ -24,7 +24,7 @@ enable_dns_resolve = True # 使用DNS解析子域(默认True) enable_http_request = True # 使用HTTP请求子域(默认True) enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True) enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True) -enable_cdn_check = True # 开启cdn检查模块(默认True) +enable_enrich_module = True # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息 enable_banner_identify = True # 开启WEB指纹识别模块(默认True) enable_takeover_check = False # 开启子域接管风险检查(默认False) # 参数可选值有'small', 'medium', 'large' diff --git a/data/cdn_asn_list.json b/data/cdn_asn_list.json index b6e1260..ad6ff1a 100644 --- a/data/cdn_asn_list.json +++ b/data/cdn_asn_list.json @@ -1,134 +1,134 @@ [ - "10576", - "10762", - "11748", - "131099", - "132601", - "133496", - "134409", - "135295", - "136764", - "137187", - "13777", - "13890", - "14103", - "14520", - "17132", - "199251", - "200013", - "200325", - "200856", - "201263", - "202294", - "203075", - "203139", - "204248", - "204286", - "204545", - "206227", - "206734", - "206848", - "206986", - "207158", - "208559", - "209403", - "21030", - "21257", - "23327", - "23393", - "23637", - "23794", - "24997", - "26492", - "268843", - "28709", - "29264", - "30282", - "30637", - "328126", - "36408", - "38107", - "397192", - "40366", - "43303", - "44907", - "46071", - "46177", - "47542", - "49287", - "49689", - "51286", - "55082", - "55254", - "56636", - "57363", - "58127", - "59730", - "59776", - "60068", - "60626", - "60922", - "61107", - "61159", - "62026", - "62229", - "63062", - "64232", - "8868", - "9053", - "55770", - "49846", - "49249", - "48163", - "45700", - "43639", - "39836", - "393560", - "393234", - "36183", - "35994", - "35993", - "35204", - "34850", - "34164", - "33905", - "32787", - "31377", - "31110", - "31109", - "31108", - "31107", - "30675", - "24319", - "23903", - "23455", - "23454", - "22207", - "21399", - "21357", - "21342", - "20940", - "20189", - "18717", - "18680", - "17334", - "16702", - "16625", - "12222", - "209101", - "201585", - "135429", - "395747", - "394536", - "209242", - "203898", - "202623", - "14789", - "133877", - "13335", - "132892", - "21859", - "6185", - "47823", - "4134" -] \ No newline at end of file + "AS10576", + "AS10762", + "AS11748", + "AS131099", + "AS132601", + "AS133496", + "AS134409", + "AS135295", + "AS136764", + "AS137187", + "AS13777", + "AS13890", + "AS14103", + "AS14520", + "AS17132", + "AS199251", + "AS200013", + "AS200325", + "AS200856", + "AS201263", + "AS202294", + "AS203075", + "AS203139", + "AS204248", + "AS204286", + "AS204545", + "AS206227", + "AS206734", + "AS206848", + "AS206986", + "AS207158", + "AS208559", + "AS209403", + "AS21030", + "AS21257", + "AS23327", + "AS23393", + "AS23637", + "AS23794", + "AS24997", + "AS26492", + "AS268843", + "AS28709", + "AS29264", + "AS30282", + "AS30637", + "AS328126", + "AS36408", + "AS38107", + "AS397192", + "AS40366", + "AS43303", + "AS44907", + "AS46071", + "AS46177", + "AS47542", + "AS49287", + "AS49689", + "AS51286", + "AS55082", + "AS55254", + "AS56636", + "AS57363", + "AS58127", + "AS59730", + "AS59776", + "AS60068", + "AS60626", + "AS60922", + "AS61107", + "AS61159", + "AS62026", + "AS62229", + "AS63062", + "AS64232", + "AS8868", + "AS9053", + "AS55770", + "AS49846", + "AS49249", + "AS48163", + "AS45700", + "AS43639", + "AS39836", + "AS393560", + "AS393234", + "AS36183", + "AS35994", + "AS35993", + "AS35204", + "AS34850", + "AS34164", + "AS33905", + "AS32787", + "AS31377", + "AS31110", + "AS31109", + "AS31108", + "AS31107", + "AS30675", + "AS24319", + "AS23903", + "AS23455", + "AS23454", + "AS22207", + "AS21399", + "AS21357", + "AS21342", + "AS20940", + "AS20189", + "AS18717", + "AS18680", + "AS17334", + "AS16702", + "AS16625", + "AS12222", + "AS209101", + "AS201585", + "AS135429", + "AS395747", + "AS394536", + "AS209242", + "AS203898", + "AS202623", + "AS14789", + "AS133877", + "AS13335", + "AS132892", + "AS21859", + "AS6185", + "AS47823", + "AS4134" +] diff --git a/dbexport.py b/dbexport.py index 7892a81..62e073a 100644 --- a/dbexport.py +++ b/dbexport.py @@ -15,17 +15,17 @@ from common.database import Database from config.log import logger -def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False): +def export(target, type='target', db=None, alive=False, limit=None, path=None, fmt='csv', show=False): """ OneForAll export from database module Example: - python3 dbexport.py --target name --format csv --dir= ./result.csv + python3 dbexport.py --target name --fmt csv --dir= ./result.csv python3 dbexport.py --db result.db --target name --show False python3 dbexport.py --target table_name --tb True --show False Note: - --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --fmt csv/json (result format) --path Result directory (default directory is ./results) :param str target: Table to be exported @@ -33,7 +33,7 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f :param str db: Database path to be exported (default ./results/result.sqlite3) :param bool alive: Only export the results of alive subdomains (default False) :param str limit: Export limit (default None) - :param str format: Result format (default csv) + :param str fmt: Result format (default csv) :param str path: Result directory (default None) :param bool show: Displays the exported data in terminal (default False) """ @@ -48,29 +48,29 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f rows = database.export_data(table_name, alive, limit) if rows is None: continue - data = export_data(format, path, rows, show, table_name, target) + data = export_data(fmt, path, rows, show, table_name, target) datas.extend(data) database.close() if len(domains) > 1: - utils.export_all(alive, format, path, datas) + utils.export_all(alive, fmt, path, datas) elif type == 'table': database = Database(db) rows = database.export_data(target, alive, limit) - data = export_data(format, path, rows, show, target, target) + data, _, _ = export_data(fmt, path, rows, show, target, target) database.close() return data -def export_data(format, path, rows, show, table_name, target): - format = utils.check_format(format, len(rows)) - path = utils.check_path(path, target, format) +def export_data(fmt, path, rows, show, table_name, target): + fmt = utils.check_format(fmt) + path = utils.check_path(path, target, fmt) if show: print(rows.dataset) - data = rows.export(format) + data = rows.export(fmt) utils.save_data(path, data) logger.log('ALERT', f'The subdomain result for {table_name}: {path}') data = rows.as_dict() - return data, format, path + return data, fmt, path def domain_to_table(table): diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 71ea40b..af35b6a 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -164,7 +164,7 @@ DESCRIPTION python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port medium run - python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --fmt csv run python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --takeover False run @@ -173,7 +173,7 @@ DESCRIPTION Note: --alive True/False Only export alive subdomains or not (default False) --port default/small/large See details in ./config/setting.py(default port 80) - --format csv/json (result format) + --fmt csv/json (result format) --path Result directory (default directory is ./results) ARGUMENTS @@ -193,7 +193,7 @@ FLAGS The port range request to the subdomains (default port 80) --alive=ALIVE Only export alive subdomains (default False) - --format=FORMAT + --fmt=FMT Result format (default csv) --path=PATH Result directory (default None) @@ -270,7 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c ## ⌛Follow-up plan - [ ] Continuous optimize and improve of each module -- [x] Subdomain monitoring (mark newly discovered subdomain) - [x] Subdomain collection crawler (collect subdomains from static files such as JS) - [ ] Implementation of front-end interface for powerful interaction diff --git a/modules/altdns.py b/modules/altdns.py index 0cc7a92..62b222b 100644 --- a/modules/altdns.py +++ b/modules/altdns.py @@ -193,9 +193,7 @@ class Altdns(Module): self.end = time.time() self.elapse = round(self.end - self.start, 1) self.gen_result() - temp_data = resolve.run_resolve(self.domain, self.results) - fina_data = request.run_request(self.domain, temp_data, port) - data = data + fina_data + resolved_data = resolve.run_resolve(self.domain, self.results) + request.run_request(self.domain, resolved_data, port) logger.log('INFOR', f'Saving altdns results') utils.save_db(self.domain, data, 'altdns') - return data diff --git a/modules/enrich.py b/modules/enrich.py new file mode 100644 index 0000000..e4a801b --- /dev/null +++ b/modules/enrich.py @@ -0,0 +1,72 @@ +from modules import iscdn +from common import utils +from common.database import Database +from common.ipasn import IPAsnInfo +from common.ipreg import IpRegData + + +def get_ips(info): + ip = info.get('ip') + if not ip: + return None + ips = ip.split(',') + return ips + + +def enrich_info(data): + ip_asn = IPAsnInfo() + ip_reg = IpRegData() + for index, info in enumerate(data): + ips = get_ips(info) + if not ips: + continue + public = list() + cidr = list() + asn = list() + org = list() + addr = list() + isp = list() + for ip in ips: + public.append(str(utils.ip_is_public(ip))) + asn_info = ip_asn.find(ip) + cidr.append(asn_info.get('cidr')) + asn.append(asn_info.get('asn')) + org.append(asn_info.get('org')) + ip_info = ip_reg.query(ip) + addr.append(ip_info.get('addr')) + isp.append(ip_info.get('isp')) + data[index]['public'] = ','.join(public) + data[index]['cidr'] = ','.join(cidr) + data[index]['asn'] = ','.join(asn) + data[index]['org'] = ','.join(org) + data[index]['addr'] = ','.join(addr) + data[index]['isp'] = ','.join(isp) + return data + + +class Enrich(object): + def __init__(self, domain): + self.domain = domain + + def get_data(self): + db = Database() + fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header', + 'cidr', 'asn', 'org', 'addr', 'isp'] + results = db.get_data_by_fields(self.domain, fields) + return results.as_dict() + + def save_db(self, data): + db = Database() + for info in data: + url = info.pop('url') + info.pop('cname') + info.pop('ip') + info.pop('header') + db.update_data_by_url(self.domain, info, url) + db.close() + + def run(self): + data = self.get_data() + data = enrich_info(data) + data = iscdn.do_check(data) + self.save_db(data) diff --git a/modules/finder.py b/modules/finder.py index 304d7b6..abc86c4 100644 --- a/modules/finder.py +++ b/modules/finder.py @@ -7,6 +7,7 @@ from common import utils from common import resolve from common import request from common.module import Module +from common.database import Database from config import settings from config.log import logger @@ -25,16 +26,11 @@ class Finder(Module): new_subdomains = found_subdomains - existing_subdomains if not len(new_subdomains): self.finish() # 未发现新的子域就直接返回 - return data self.subdomains = new_subdomains self.finish() self.gen_result() - temp_data = resolve.run_resolve(domain, self.results) - fina_data = request.run_request(domain, temp_data, port) - data = data + fina_data - logger.log('INFOR', f'Saving finder results') - utils.save_db(domain, data, 'finder') - return data + resolved_data = resolve.run_resolve(domain, self.results) + request.run_request(domain, resolved_data, port) file_path = settings.data_storage_dir.joinpath('common_js_library.json') @@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html): return js_urls +def convert_to_dict(url_list): + url_dict = [] + for url in url_list: + url_dict.append({'url': url}) + return url_dict + def find_subdomains(domain, data): subdomains = set() js_urls = set() + db = Database() for infos in data: jump_history = infos.get('history') req_url = infos.get('url') subdomains.update(find_in_history(domain, req_url, jump_history)) - rsp_html = infos.get('response') + rsp_html = db.get_resp_by_url(domain, req_url) if not rsp_html: logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}') continue subdomains.update(find_in_resp(domain, req_url, rsp_html)) js_urls.update(find_js_urls(domain, req_url, rsp_html)) - resp_data = request.bulk_request(js_urls) - for _, resp in resp_data: + req_data = convert_to_dict(js_urls) + resp_data = request.bulk_request(domain, req_data, ret=True) + while not resp_data.empty(): + _, resp = resp_data.get() if not isinstance(resp, Response): continue text = utils.decode_resp_text(resp) diff --git a/modules/iscdn.py b/modules/iscdn.py index 10920ef..276b43f 100644 --- a/modules/iscdn.py +++ b/modules/iscdn.py @@ -3,6 +3,7 @@ import ipaddress from config import settings from common import utils +from common.database import Database from config.log import logger data_dir = settings.data_storage_dir @@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json')) def check_cname_keyword(cname): + if not cname: + return False names = cname.lower().split(',') for name in names: for keyword in cdn_cname_keyword.keys(): @@ -26,14 +29,22 @@ def check_cname_keyword(cname): def check_header_key(header): - header = set(map(lambda x: x.lower(), header.keys())) - for key in cdn_header_key: - if key in header: - return True + if isinstance(header, str): + header = json.loads(header) + if isinstance(header, dict): + header = set(map(lambda x: x.lower(), header.keys())) + for key in cdn_header_key: + if key in header: + return True + else: + return False -def check_cdn_cidr(content): - ips = set(content.split(',')) +def check_cdn_cidr(ips): + if isinstance(ips, str): + ips = set(ips.split(',')) + else: + return False for ip in ips: try: ip = ipaddress.ip_address(ip) @@ -46,39 +57,30 @@ def check_cdn_cidr(content): def check_cdn_asn(asn): - if str(asn) in cdn_asn_list: - return True + if isinstance(asn, str): + if asn in cdn_asn_list: + return True + return False -def check_cdn(data): - logger.log('DEBUG', f'Start cdn check module') +def do_check(data): + logger.log('DEBUG', f'Checking cdn') for index, item in enumerate(data): cname = item.get('cname') - if cname: - if check_cname_keyword(cname): - data[index]['cdn'] = 1 - continue + if check_cname_keyword(cname): + data[index]['cdn'] = 1 + continue header = item.get('header') - if header: - header = json.loads(header) - if check_header_key(header): - data[index]['cdn'] = 1 - continue + if check_header_key(header): + data[index]['cdn'] = 1 + continue ip = item.get('ip') - if ip: - if check_cdn_cidr(ip): - data[index]['cdn'] = 1 - continue + if check_cdn_cidr(ip): + data[index]['cdn'] = 1 + continue asn = item.get('asn') - if asn: - asn = asn[2:] # 去除AS - if check_cdn_asn(asn): - data[index]['cdn'] = 1 - continue + if check_cdn_asn(asn): + data[index]['cdn'] = 1 + continue data[index]['cdn'] = 0 return data - - -def save_db(name, data): - logger.log('DEBUG', f'Saving cdn check results') - utils.save_db(name, data, 'cdn') diff --git a/oneforall.py b/oneforall.py index cb84655..13cbbd1 100644 --- a/oneforall.py +++ b/oneforall.py @@ -15,12 +15,11 @@ from datetime import datetime import dbexport from brute import Brute from common import utils, resolve, request -from common.database import Database from modules.collect import Collect from modules.srv import BruteSRV from modules.finder import Finder from modules.altdns import Altdns -from modules import iscdn +from modules.enrich import Enrich from config import settings from config.log import logger from takeover import Takeover @@ -60,7 +59,7 @@ class OneForAll(object): python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --brute False run python3 oneforall.py --target example.com --port medium run - python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --fmt csv run python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --takeover False run @@ -68,22 +67,22 @@ class OneForAll(object): Note: --port small/medium/large See details in ./config/setting.py(default small) - --format csv/json (result format) + --fmt csv/json (result format) --path Result path (default None, automatically generated) :param str target: One domain (target or targets must be provided) :param str targets: File path of one domain per line - :param bool brute: Use brute module (default True) - :param bool dns: Use DNS resolution (default True) - :param bool req: HTTP request subdomains (default True) + :param bool brute: Use brute module (default True) + :param bool dns: Use DNS resolution (default True) + :param bool req: HTTP request subdomains (default True) :param str port: The port range to request (default small port is 80,443) - :param bool alive: Only export alive subdomains (default False) - :param str format: Result format (default csv) + :param bool alive: Only export alive subdomains (default False) + :param str fmt: Result format (default csv) :param str path: Result path (default None, automatically generated) - :param bool takeover: Scan subdomain takeover (default False) + :param bool takeover: Scan subdomain takeover (default False) """ def __init__(self, target=None, targets=None, brute=None, dns=None, req=None, - port=None, alive=None, format=None, path=None, takeover=None): + port=None, alive=None, fmt=None, path=None, takeover=None): self.target = target self.targets = targets self.brute = brute @@ -91,17 +90,13 @@ class OneForAll(object): self.req = req self.port = port self.alive = alive - self.format = format + self.fmt = fmt self.path = path self.takeover = takeover self.domain = str() # The domain currently being collected self.domains = set() # All domains that are to be collected self.data = list() # The subdomain results of the current domain self.datas = list() # All subdomain results of the domain - self.old_table = str() # The table name of the last result - self.new_table = str() # The table name of the current result - self.origin_table = str() # The table name of the origin result - self.resolve_table = str() # The table name of the resolute result def config_param(self): """ @@ -119,8 +114,8 @@ class OneForAll(object): self.port = settings.http_request_port if self.alive is None: self.alive = bool(settings.result_export_alive) - if self.format is None: - self.format = settings.result_save_format + if self.fmt is None: + self.fmt = settings.result_save_format if self.path is None: self.path = settings.result_save_path @@ -134,47 +129,13 @@ class OneForAll(object): def export(self, table): """ - Export data from the database and do some follow-up processing + Export data from the database :param table: table name :return: export data :rtype: list """ - db = Database() - data = dbexport.export(table, type='table', alive=self.alive, format=self.format) - db.drop_table(self.new_table) - db.rename_table(self.domain, self.new_table) - db.close() - return data - - def deal_db(self): - """ - Process the data when the collection task is completed - """ - db = Database() - db.deal_table(self.domain, self.origin_table) - db.close() - - def mark(self): - """ - Mark the new discovered subdomain - - :return: marked data - :rtype: list - """ - db = Database() - old_data = list() - now_data = db.get_data(self.domain).as_dict() - # Database pre-processing when it is not the first time to collect this subdomain - if db.exist_table(self.new_table): - # If there is the last collection result table, delete it first - db.drop_table(self.old_table) - # Rename the new table to the old table - db.rename_table(self.new_table, self.old_table) - old_data = db.get_data(self.old_table).as_dict() - db.close() - marked_data = utils.mark_subdomain(old_data, now_data) - return marked_data + return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt) def main(self): """ @@ -183,11 +144,6 @@ class OneForAll(object): :return: subdomain results :rtype: list """ - self.old_table = self.domain + '_old_result' - self.new_table = self.domain + '_now_result' - self.origin_table = self.domain + '_origin_result' - self.resolve_table = self.domain + '_resolve_result' - collect = Collect(self.domain) collect.run() @@ -202,49 +158,44 @@ class OneForAll(object): brute.quite = True brute.run() - # Database processing - self.deal_db() - # Mark the new discovered subdomain - self.data = self.mark() - + utils.deal_data(self.domain) # Export results without resolve if not self.dns: return self.export(self.domain) + self.data = utils.get_data(self.domain) + # Resolve subdomains + utils.clear_data(self.domain) self.data = resolve.run_resolve(self.domain, self.data) # Save resolve results - resolve.save_db(self.resolve_table, self.data) + resolve.save_db(self.domain, self.data) # Export results without HTTP request if not self.req: - return self.export(self.resolve_table) + return self.export(self.domain) # HTTP request - self.data = request.run_request(self.domain, self.data, self.port) - # Save HTTP request result - request.save_db(self.domain, self.data) + utils.clear_data(self.domain) + request.run_request(self.domain, self.data, self.port) # Finder module if settings.enable_finder_module: finder = Finder() - self.data = finder.run(self.domain, self.data, self.port) + finder.run(self.domain, self.data, self.port) # altdns module if settings.enable_altdns_module: - finder = Altdns(self.domain) - self.data = finder.run(self.data, self.port) + altdns = Altdns(self.domain) + altdns.run(self.data, self.port) - # check cdn module - if settings.enable_cdn_check: - self.data = iscdn.check_cdn(self.data) - iscdn.save_db(self.domain, self.data) - - # Add the final result list to the total data list - self.datas.extend(self.data) + # Information enrichment module + if settings.enable_enrich_module: + enrich = Enrich(self.domain) + enrich.run() # Export - self.export(self.domain) + self.datas.extend(self.export(self.domain)) # Scan subdomain takeover if self.takeover: @@ -277,7 +228,7 @@ class OneForAll(object): for domain in self.domains: self.domain = utils.get_main_domain(domain) self.main() - utils.export_all(self.alive, self.format, self.path, self.datas) + utils.export_all(self.alive, self.fmt, self.path, self.datas) else: logger.log('FATAL', 'Failed to obtain domain') logger.log('INFOR', 'Finished OneForAll') diff --git a/takeover.py b/takeover.py index 721b2c4..094ed63 100644 --- a/takeover.py +++ b/takeover.py @@ -45,21 +45,21 @@ class Takeover(Module): OneForAll subdomain takeover module Example: - python3 takeover.py --target www.example.com --format csv run + python3 takeover.py --target www.example.com --fmt csv run python3 takeover.py --targets ./subdomains.txt --thread 10 run Note: - --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --fmt txt/csv/json (result format) --path Result directory (default directory is ./results) :param str target: One domain (target or targets must be provided) :param str targets: File path of one domain per line :param int thread: threads number (default 20) - :param str format: Result format (default csv) + :param str fmt: Result format (default csv) :param str path: Result directory (default None) """ - def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'): + def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'): Module.__init__(self) self.subdomains = set() self.module = 'Check' @@ -68,7 +68,7 @@ class Takeover(Module): self.targets = targets self.thread = thread self.path = path - self.format = format + self.fmt = fmt self.fingerprints = None self.subdomainq = Queue() self.cnames = list() @@ -76,10 +76,10 @@ class Takeover(Module): def save(self): logger.log('DEBUG', 'Saving results') - if self.format == 'txt': + if self.fmt == 'txt': data = str(self.results) else: - data = self.results.export(self.format) + data = self.results.export(self.fmt) utils.save_data(self.path, data) def compare(self, subdomain, cname, responses): @@ -131,10 +131,10 @@ class Takeover(Module): self.subdomains = self.targets else: self.subdomains = utils.get_domains(self.target, self.targets) - self.format = utils.check_format(self.format, len(self.subdomains)) + self.fmt = utils.check_format(self.fmt) timestamp = utils.get_timestamp() name = f'takeover_check_result_{timestamp}' - self.path = utils.check_path(self.path, name, self.format) + self.path = utils.check_path(self.path, name, self.fmt) if self.subdomains: logger.log('INFOR', f'Checking subdomain takeover') self.fingerprints = get_fingerprint()