= self.limit_num: # 搜索条数限制
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ if not (self.id and self.key):
+ logger.log('ERROR', f'{self.source}模块API配置错误')
+ logger.log('ALERT', f'不执行{self.source}模块')
+ return
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = BingAPI(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/duckduckgo.py b/oneforall/modules/search/duckduckgo.py
new file mode 100644
index 0000000..0db2fe6
--- /dev/null
+++ b/oneforall/modules/search/duckduckgo.py
@@ -0,0 +1,93 @@
+# coding=utf-8
+import re
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class DuckDuckGO(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'DuckDuckGoSearch'
+ self.addr = 'https://duckduckgo.com/html/'
+ self.header = self.get_header()
+ self.delay = 2
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ query = 'site:' + domain + filtered_subdomain
+ data = {'q': query, 'kl': 'us-en', 'v': 'l'}
+ while True:
+ time.sleep(self.delay)
+ self.proxy = self.get_proxy(self.source)
+ resp = self.post(self.addr, data)
+ if not resp:
+ return
+ subdomain_find = self.match(domain, resp.text)
+ if not subdomain_find:
+ break
+ if not full_search:
+ if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+ break
+ self.subdomains = self.subdomains.union(subdomain_find)
+ try:
+ s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
+ dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
+ except Exception as e:
+ logger.error(e)
+ break
+ data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ # return # 暂时还有点问题
+ search = DuckDuckGO(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/exalead.py b/oneforall/modules/search/exalead.py
new file mode 100644
index 0000000..5e9f0cd
--- /dev/null
+++ b/oneforall/modules/search/exalead.py
@@ -0,0 +1,93 @@
+# coding=utf-8
+import time
+import queue
+import random
+from common.search import Search
+from config import logger
+
+
+class Exalead(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = "ExaleadSearch"
+ self.addr = "http://www.exalead.com/search/web/results/"
+ self.per_page_num = 30
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ self.page_num = 0
+ while True:
+ self.delay = random.randint(1, 5)
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ query = 'site:' + domain + filtered_subdomain
+ params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
+ resp = self.get(url=self.addr, params=params)
+ if not resp:
+ return
+ subdomain_find = self.match(domain, resp.text)
+ if not subdomain_find:
+ break
+ if not full_search:
+ if subdomain_find.issubset(self.subdomains):
+ break
+ self.subdomains = self.subdomains.union(subdomain_find)
+ self.page_num += self.per_page_num
+ if self.page_num > 1999:
+ break
+ if 'title="Go to the next page"' not in resp.text:
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ statement = statement.replace('-site', 'and -site')
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = Exalead(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/fofa.py b/oneforall/modules/search/fofa.py
new file mode 100644
index 0000000..a1ff341
--- /dev/null
+++ b/oneforall/modules/search/fofa.py
@@ -0,0 +1,71 @@
+# coding=utf-8
+import time
+import queue
+import json
+import base64
+import config
+from common.search import Search
+from config import logger
+
+
+class FoFa(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'FoFaSearch'
+ self.addr = 'https://fofa.so/api/v1/search/all'
+ self.delay = 1
+ self.email = config.fofa_api_email
+ self.key = config.fofa_api_key
+
+ def search(self):
+ """
+ 发送搜索请求并做子域匹配
+ """
+ self.page_num = 1
+ query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
+ while True:
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
+ resp = self.get(self.addr, query)
+ if not resp:
+ return
+ subdomain_find = self.match(self.domain, resp.text)
+ self.subdomains = self.subdomains.union(subdomain_find)
+ self.page_num += 1
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search()
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = FoFa(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/google.py b/oneforall/modules/search/google.py
new file mode 100644
index 0000000..f6aab5e
--- /dev/null
+++ b/oneforall/modules/search/google.py
@@ -0,0 +1,101 @@
+# coding=utf-8
+import time
+import queue
+import random
+from common.search import Search
+from config import logger
+
+
+class Google(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'GoogleSearch'
+ self.init = 'https://www.google.com/'
+ self.addr = 'https://www.google.com/search'
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ page_num = 1
+ per_page_num = 50
+ self.header = self.get_header()
+ self.header.update({'User-Agent': 'Googlebot',
+ 'Referer': 'https://www.google.com'})
+ self.proxy = self.get_proxy(self.source)
+ resp = self.get(self.init)
+ if not resp:
+ return
+ self.cookie = resp.cookies
+ while True:
+ self.delay = random.randint(1, 5)
+ time.sleep(self.delay)
+ self.proxy = self.get_proxy(self.source)
+ word = 'site:' + domain + filtered_subdomain
+ payload = {'q': word, 'start': page_num, 'num': per_page_num,
+ 'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
+ resp = self.get(url=self.addr, params=payload)
+ if not resp:
+ return
+ subdomain_find = self.match(domain, resp.text)
+ if not subdomain_find:
+ break
+ if not full_search:
+ if subdomain_find.issubset(self.subdomains):
+ break
+ self.subdomains = self.subdomains.union(subdomain_find)
+ page_num += per_page_num
+ if 'start='+str(page_num) not in resp.text:
+ break
+ if '302 Moved' in resp.text:
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = Google(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/google_api.py b/oneforall/modules/search/google_api.py
new file mode 100644
index 0000000..5cb4c20
--- /dev/null
+++ b/oneforall/modules/search/google_api.py
@@ -0,0 +1,97 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.search import Search
+from config import logger
+
+
+class GoogleAPI(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'GoogleAPISearch'
+ self.addr = 'https://www.googleapis.com/customsearch/v1'
+ self.delay = 1
+ self.key = config.google_api_key
+ self.cx = config.google_api_cx
+ self.per_page_num = 10 # 每次只能请求10个结果
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ self.page_num = 1
+ while True:
+ word = 'site:' + domain + filtered_subdomain
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
+ 'start': self.page_num, 'num': self.per_page_num}
+ resp = self.get(self.addr, params)
+ if not resp:
+ return
+ subdomain_find = self.match(domain, str(resp.json()))
+ if not subdomain_find:
+ break
+ if not full_search:
+ if subdomain_find.issubset(self.subdomains):
+ break
+ self.subdomains = self.subdomains.union(subdomain_find)
+ self.page_num += self.per_page_num
+ if self.page_num > 100: # 免费的API只能查询前100条结果
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ if not (self.cx and self.key):
+ logger.log('ERROR', f'{self.source}模块API配置错误')
+ logger.log('ALERT', f'不执行{self.source}模块')
+ return
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = GoogleAPI(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/shodan_api.py b/oneforall/modules/search/shodan_api.py
new file mode 100644
index 0000000..24fc4e2
--- /dev/null
+++ b/oneforall/modules/search/shodan_api.py
@@ -0,0 +1,73 @@
+# coding=utf-8
+import time
+import queue
+import queue
+import config
+# from shodan import Shodan
+from common.search import Search
+from config import logger
+
+
+class ShodanAPI(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = self.register(domain)
+ self.module = 'Search'
+ self.source = 'ShodanSearch'
+ self.addr = 'https://api.shodan.io/shodan/host/search'
+ self.key = config.shodan_api_key
+
+ def search(self):
+ """
+ 发送搜索请求并做子域匹配
+ """
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ query = 'hostname:.' + self.domain
+ page = 1
+ while True:
+ params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
+ resp = self.get(self.addr, params)
+ if not resp:
+ return
+ subdomain_find = self.match(self.domain, resp.text)
+ if subdomain_find:
+ self.subdomains = self.subdomains.union(subdomain_find)
+ page += 1
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ if not self.key:
+ logger.log('ERROR', f'{self.source}模块API配置错误')
+ logger.log('ALERT', f'不执行{self.source}模块')
+ return
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search()
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = ShodanAPI(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ results = queue.Queue()
+ do('qq.com', results)
diff --git a/oneforall/modules/search/so.py b/oneforall/modules/search/so.py
new file mode 100644
index 0000000..13ac86a
--- /dev/null
+++ b/oneforall/modules/search/so.py
@@ -0,0 +1,91 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class So(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'SoSearch'
+ self.addr = 'http://www.so.com/s'
+ self.limit_num = 640 # 限制搜索条数
+ self.per_page_num = 10 # 默认每页显示10页
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ page_num = 1
+ while True:
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ word = 'site:' + domain + filtered_subdomain
+ payload = {'q': word, 'pn': page_num}
+ resp = self.get(url=self.addr, params=payload)
+ if not resp:
+ return
+ subdomain_find = self.match(domain, resp.text)
+ if not subdomain_find:
+ break
+ if not full_search:
+ if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+ break
+ self.subdomains = self.subdomains.union(subdomain_find)
+ page_num += 1
+ if '
= self.limit_num: # 搜索条数限制
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = So(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/sogou.py b/oneforall/modules/search/sogou.py
new file mode 100644
index 0000000..6cde4d3
--- /dev/null
+++ b/oneforall/modules/search/sogou.py
@@ -0,0 +1,89 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Sogou(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'SogouSearch'
+ self.addr = 'https://www.sogou.com/web'
+ self.limit_num = 1000 # 限制搜索条数
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ self.page_num = 1
+ while True:
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ word = 'site:' + domain + filtered_subdomain
+ payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
+ resp = self.get(self.addr, payload)
+ if not resp:
+ return
+ subdomain_find = self.match(domain, resp.text)
+ if not subdomain_find:
+ break
+ if not full_search:
+ if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+ break
+ self.subdomains = self.subdomains.union(subdomain_find)
+ self.page_num += 1
+ if '= self.limit_num: # 搜索条数限制
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = Sogou(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/yahoo.py b/oneforall/modules/search/yahoo.py
new file mode 100644
index 0000000..9d78c5d
--- /dev/null
+++ b/oneforall/modules/search/yahoo.py
@@ -0,0 +1,97 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Yahoo(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'YahooSearch'
+ self.init = 'https://hk.search.yahoo.com/'
+ self.addr = 'https://hk.search.yahoo.com/search'
+ self.limit_num = 1000 # 限制搜索条数
+ self.delay = 5
+ self.per_page_num = 40
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ self.page_num = 0
+ resp = self.get(self.init)
+ if not resp:
+ return
+ self.cookie = resp.cookies # 获取cookie Yahoo在搜索时需要带上cookie
+ while True:
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ query = 'site:' + domain + filtered_subdomain
+ params = {'q': query, 'b': self.page_num, 'n': self.per_page_num}
+ resp = self.get(self.addr, params)
+ if not resp:
+ return
+ subdomains_find = self.match(domain, resp.text)
+ if not subdomains_find: # 搜索没有发现子域名则停止搜索
+ break
+ if not full_search:
+ if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+ break
+ self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
+ if '>Next' not in resp.text: # 搜索页面没有出现下一页时停止搜索
+ break
+ self.page_num += self.per_page_num
+ if self.page_num >= self.limit_num: # 搜索条数限制
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = Yahoo(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/yandex.py b/oneforall/modules/search/yandex.py
new file mode 100644
index 0000000..3c109f4
--- /dev/null
+++ b/oneforall/modules/search/yandex.py
@@ -0,0 +1,93 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Yandex(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'YandexSearch'
+ self.init = 'https://yandex.com/'
+ self.addr = 'https://yandex.com/search'
+ self.limit_num = 1000 # 限制搜索条数
+ self.delay = 5
+
+ def search(self, domain, filtered_subdomain='', full_search=False):
+ """
+ 发送搜索请求并做子域匹配
+
+ :param str domain: 域名
+ :param str filtered_subdomain: 过滤的子域
+ :param bool full_search: 全量搜索
+ """
+ self.page_num = 0 # 二次搜索重新置0
+ self.cookie = self.get(self.init).cookies # 获取cookie bing在搜索时需要带上cookie
+ while True:
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ query = 'site:' + domain + filtered_subdomain
+ params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
+ resp = self.get(self.addr, params)
+ if not resp:
+ return
+ subdomains_find = self.match(domain, resp.text)
+ if not subdomains_find: # 搜索没有发现子域名则停止搜索
+ break
+ if not full_search:
+ if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+ break
+ self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
+ if '>next' not in resp.text: # 搜索页面没有出现下一页时停止搜索
+ break
+ self.page_num += 1
+ if self.page_num >= self.limit_num: # 搜索条数限制
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search(self.domain, full_search=True)
+
+ # 排除同一子域搜索结果过多的子域以发现新的子域
+ for statement in self.filter(self.domain, self.subdomains):
+ self.search(self.domain, filtered_subdomain=statement)
+
+ # 递归搜索下一层的子域
+ if self.recursive_search:
+ for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+ for subdomain in self.subdomains:
+ if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
+ self.search(subdomain)
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = Yandex(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/modules/search/zoomeye_api.py b/oneforall/modules/search/zoomeye_api.py
new file mode 100644
index 0000000..2d8a01a
--- /dev/null
+++ b/oneforall/modules/search/zoomeye_api.py
@@ -0,0 +1,97 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.search import Search
+from config import logger
+
+
+class ZoomEyeAPI(Search):
+ def __init__(self, domain):
+ Search.__init__(self)
+ self.domain = domain
+ self.module = 'Search'
+ self.source = 'ZoomEyeAPISearch'
+ self.addr = 'https://api.zoomeye.org/web/search'
+ self.delay = 2
+ self.user = config.zoomeye_api_username
+ self.pwd = config.zoomeye_api_password
+
+ def login(self):
+ """
+ 登陆获取查询taken
+ :return:
+ """
+ url = 'https://api.zoomeye.org/user/login'
+ data = {'username': self.user, 'password': self.pwd}
+ resp = self.post(url=url, json=data)
+ if not resp:
+ logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
+ return
+ resp_json = resp.json()
+ if resp.status_code == 200:
+ # print('登陆成功')
+ return resp_json.get('access_token')
+ else:
+ logger.log('ALERT', resp_json.get('message'))
+ exit(1)
+
+ def search(self):
+ """
+ 发送搜索请求并做子域匹配
+ """
+ page_num = 1
+ access_token = self.login()
+ while True:
+ time.sleep(self.delay)
+ self.header = self.get_header()
+ self.proxy = self.get_proxy(self.source)
+ self.header.update({'Authorization': 'JWT ' + access_token})
+ params = {'query': 'hostname:' + self.domain, 'page': page_num}
+ resp = self.get(self.addr, params)
+ if not resp:
+ return
+ subdomain_find = self.match(self.domain, resp.text)
+ self.subdomains = self.subdomains.union(subdomain_find)
+ page_num += 1
+ if page_num > 500:
+ break
+ if resp.status_code == 403:
+ break
+
+ def run(self, rx_queue):
+ """
+ 类执行入口
+ """
+ if not (self.user and self.pwd):
+ logger.log('ERROR', f'{self.source}模块API配置错误')
+ logger.log('ALERT', f'不执行{self.source}模块')
+ return
+ logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+ start = time.time()
+ self.search()
+ end = time.time()
+ self.elapsed = round(end - start, 1)
+ self.save_json()
+ self.gen_result()
+ self.save_db()
+ rx_queue.put(self.results)
+ logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue): # 统一入口名字 方便多线程调用
+ """
+ 类统一调用入口
+
+ :param str domain: 域名
+ :param rx_queue: 结果集队列
+ """
+ search = ZoomEyeAPI(domain)
+ search.run(rx_queue)
+ logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+ logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+ result_queue = queue.Queue()
+ do('owasp.org', result_queue)
diff --git a/oneforall/oneforall.py b/oneforall/oneforall.py
new file mode 100644
index 0000000..562f32e
--- /dev/null
+++ b/oneforall/oneforall.py
@@ -0,0 +1,98 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+"""
+OneForAll是一款强大的子域收集神器
+"""
+
+import asyncio
+import fire
+import config
+import dbexport
+from config import logger
+from collect import Collect
+from aiobrute import AIOBrute
+from common import utils, database, resolve, request
+
+__author__ = 'Jing Ling, Black Star'
+__contact__ = 'admin@hackfun.org'
+__copyright__ = 'Copyright (c) 2019, Jing Ling. All rights reserved.'
+__license__ = 'GNU General Public License v3.0'
+__version__ = '0.0.1'
+
+
+class OneForAll(object):
+ """
+ OneForAll是一款强大的子域收集神器
+
+ Version: 0.0.1
+ Project: https://github.com/shmilylty/OneForAll/
+
+ :param str target: 单个域名或者每行一个域名的文件路径
+ :param bool brute: 是否使用爆破模块（默认禁用）
+ :param str port: HTTP请求验证的端口范围（默认medium）
+ :param int valid: 导出子域的有效性（默认1）
+ :param str format: 导出格式（默认xlsx）
+ :param str path: 导出路径(默认None)
+ :param bool output: 是否将导出数据输出到终端（默认False）
+
+ Note:
+ 参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+ 参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+ 参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+ 参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+
+ Example:
+ python oneforall.py --target example.com run
+ python oneforall.py --target example.com --brute True --port medium valid 1 run
+ python oneforall.py --target ./domains.txt --format csv --path= ./result.csv --output True run
+ """
+ def __init__(self, target, brute=False, port='medium', valid=1, path=None, format='xlsx', output=False):
+ self.target = target
+ self.port = port
+ self.domains = set()
+ self.domain = ''
+ self.datas = list()
+ self.brute = brute or config.enable_brute_module
+ self.valid = valid
+ self.path = path
+ self.format = format
+ self.output = output
+
+ def run(self):
+ logger.log('INFOR', f'开始运行OneForAll')
+ self.domains = utils.get_domains(self.target)
+ if self.domains:
+ for self.domain in self.domains:
+ collect = Collect(self.domain, export=False)
+ collect.run()
+ if self.brute:
+ # 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
+ brute = AIOBrute(self.domain)
+ brute.run()
+ table_name = self.domain.replace('.', '_')
+ db_conn = database.connect_db()
+ self.datas = database.get_data(db_conn, table_name).as_dict()
+ loop = asyncio.get_event_loop()
+ asyncio.set_event_loop(loop)
+ self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
+ self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
+ loop.run_until_complete(asyncio.sleep(0.25)) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+ loop.close()
+ database.clear_table(db_conn, table_name)
+ database.save_db(db_conn, table_name, self.datas)
+ # database.deduplicate_subdomain(db_conn, table_name)
+ # database.remove_invalid(db_conn, table_name)
+ # 数据库导出
+ if not self.path:
+ self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
+ dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
+ db_conn.close()
+ else:
+ logger.log('FATAL', f'获取域名失败')
+ logger.log('INFOR', f'结束运行OneForAll')
+
+
+if __name__ == '__main__':
+ fire.Fire(OneForAll)
+ # OneForAll('owasp.org').run()
diff --git a/oneforall/requirements.txt b/oneforall/requirements.txt
new file mode 100644
index 0000000..6e761fa
Binary files /dev/null and b/oneforall/requirements.txt differ
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..50d8f68
Binary files /dev/null and b/requirements.txt differ