river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

api配置分离

Browse Source
This commit is contained in:
Jing Ling
2020-02-10 20:11:15 +08:00
parent 3f8aa8b74d
commit 38c4de6896
21 changed files with 196 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
oneforall/api.py
+74
View File
@@ -0,0 +1,74 @@
# 模块API配置
# Censys可以免费注册获取API：https://censys.io/api
censys_api_id = ''
censys_api_secret = ''
# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
binaryedge_api = ''
# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
chinaz_api = ''
# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
# cognitive-services/bing-web-search-api/#web-json
bing_api_id = ''
bing_api_key = ''
# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
securitytrails_api = ''
# https://fofa.so/api
fofa_api_email = '' # fofa用户邮箱
fofa_api_key = '' # fofa用户key
# Google可以免费注册获取API:
# https://developers.google.com/custom-search/v1/overview
# 免费的API只能查询前100条结果
google_api_key = '' # Google API搜索key
google_api_cx = '' # Google API搜索cx
# https://api.passivetotal.org/api/docs/
riskiq_api_username = ''
riskiq_api_key = ''
# Shodan可以免费注册获取API: https://account.shodan.io/register
# 免费的API限速1秒查询1次
shodan_api_key = ''
# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
threatbook_api_key = ''
# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
virustotal_api_key = ''
# https://www.zoomeye.org/doc?channel=api
zoomeye_api_username = ''
zoomeye_api_password = ''
# Spyse可以免费注册获取API: https://spyse.com/
spyse_api_token = ''
# https://www.circl.lu/services/passive-dns/
circl_api_username = ''
circl_api_password = ''
# https://www.dnsdb.info/
dnsdb_api_key = ''
# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
ipv4info_api_key = ''
# https://github.com/360netlab/flint
# passivedns_api_addr默认空使用http://api.passivedns.cn
# passivedns_api_token可为空
passivedns_api_addr = ''
passivedns_api_token = ''
# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
# 用于子域接管
github_api_user = ''
github_api_token = ''
# Github子域收集模块使用
github_email = ''
github_password = ''
oneforall/config.py
+2 -76
View File
@@ -107,7 +107,7 @@ ports = {'default': default_ports, 'small': small_ports, 'large': large_ports}
verify_ssl = False
# aiohttp 支持 HTTP/HTTPS形式的代理
get_proxy = None # proxy="http://user:pass@some.proxy.com"
get_timeout = 200 # http请求探测总超时时间 None或者0则表示不检测超时
get_timeout = 5 * 60 # http请求探测总超时时间 None或者0则表示不检测超时 默认5分钟
get_redirects = True # 允许请求跳转
fake_header = True # 使用伪造请求头
@@ -115,81 +115,7 @@ fake_header = True # 使用伪造请求头
limit_open_conn = 500 # 默认500
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
limit_per_host = 50 # 默认0表示不限制
# 模块API配置
# Censys可以免费注册获取API：https://censys.io/api
censys_api_id = ''
censys_api_secret = ''
# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
binaryedge_api = ''
# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
chinaz_api = ''
# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
# cognitive-services/bing-web-search-api/#web-json
bing_api_id = ''
bing_api_key = ''
# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
securitytrails_api = ''
# https://fofa.so/api
fofa_api_email = '' # fofa用户邮箱
fofa_api_key = '' # fofa用户key
# Google可以免费注册获取API:
# https://developers.google.com/custom-search/v1/overview
# 免费的API只能查询前100条结果
google_api_key = '' # Google API搜索key
google_api_cx = '' # Google API搜索cx
# https://api.passivetotal.org/api/docs/
riskiq_api_username = ''
riskiq_api_key = ''
# Shodan可以免费注册获取API: https://account.shodan.io/register
# 免费的API限速1秒查询1次
shodan_api_key = ''
# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
threatbook_api_key = ''
# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
virustotal_api_key = ''
# https://www.zoomeye.org/doc?channel=api
zoomeye_api_username = ''
zoomeye_api_password = ''
# Spyse可以免费注册获取API: https://spyse.com/
spyse_api_token = ''
# https://www.circl.lu/services/passive-dns/
circl_api_username = ''
circl_api_password = ''
# https://www.dnsdb.info/
dnsdb_api_key = ''
# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
ipv4info_api_key = ''
# https://github.com/360netlab/flint
# passivedns_api_addr默认空使用http://api.passivedns.cn
# passivedns_api_token可为空
passivedns_api_addr = ''
passivedns_api_token = ''
# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
github_api_user = ''
github_api_token = ''
# github子域收集模块使用
github_email = ''
github_password = ''
limit_per_host = 50 # 0表示不限制
subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
oneforall/modules/autotake/github.py
+3 -3
View File
@@ -8,7 +8,7 @@ github自动接管
import json
import base64
import requests
import config
import api
HEADERS = {
"Accept": "application/json, text/javascript, */*; q=0.01",
@@ -21,8 +21,8 @@ def github_takeover(url):
# 读取config配置文件
repo_name = url
print('[*]正在读取配置文件...')
user = config.github_api_user
token = config.github_api_token
user = api.github_api_user
token = api.github_api_token
CHECK_HEADERS = {
"Authorization": 'token ' + token,
"Accept": "application/vnd.github.switcheroo-preview+json"
oneforall/modules/certificates/censys_api.py
+3 -3
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
from config import logger
@@ -10,8 +10,8 @@ class CensysAPI(Query):
self.module = 'Certificate'
self.source = "CensysAPIQuery"
self.addr = 'https://www.censys.io/api/v1/search/certificates'
self.id = config.censys_api_id
self.secret = config.censys_api_secret
self.id = api.censys_api_id
self.secret = api.censys_api_secret
self.delay = 3.0 # Censys 接口查询速率限制 最快2.5秒查1次
def query(self):
oneforall/modules/certificates/spyse_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common import utils
from common.query import Query
@@ -10,7 +10,7 @@ class SpyseAPI(Query):
self.module = 'Certificate'
self.source = 'CertDBAPIQuery'
self.addr = 'https://api.spyse.com/v1/subdomains'
self.token = config.spyse_api_token
self.token = api.spyse_api_token
def query(self):
"""
oneforall/modules/datasets/binaryedge_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -9,7 +9,7 @@ class BinaryEdgeAPI(Query):
self.module = 'Dataset'
self.source = 'BinaryEdgeAPIQuery'
self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
self.api = config.binaryedge_api
self.api = api.binaryedge_api
def query(self):
"""
oneforall/modules/datasets/chinaz_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -9,7 +9,7 @@ class ChinazAPI(Query):
self.module = 'Dataset'
self.source = 'ChinazAPIQuery'
self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
self.api = config.chinaz_api
self.api = api.chinaz_api
def query(self):
"""
oneforall/modules/datasets/circl_api.py
+3 -3
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -9,8 +9,8 @@ class CirclAPI(Query):
self.module = 'Dataset'
self.source = 'CirclAPIQuery'
self.addr = 'https://www.circl.lu/pdns/query/'
self.user = config.circl_api_username
self.pwd = config.circl_api_password
self.user = api.circl_api_username
self.pwd = api.circl_api_password
def query(self):
"""
oneforall/modules/datasets/dnsdb_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common import utils
from common.query import Query
@@ -10,7 +10,7 @@ class DNSdbAPI(Query):
self.module = 'Dataset'
self.source = 'DNSdbAPIQuery'
self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
self.api = config.dnsdb_api_key
self.api = api.dnsdb_api_key
def query(self):
"""
oneforall/modules/datasets/ipv4info.py → oneforall/modules/datasets/ipv4info_api.py
+73 -73
View File
@@ -1,73 +1,73 @@
import config
from common.query import Query
from config import logger
class IPv4InfoAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'IPv4InfoAPIQuery'
self.addr = ' http://ipv4info.com/api_v1/'
self.api = config.ipv4info_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
page = 0
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'type': 'SUBDOMAINS', 'key': self.api,
'value': self.domain, 'page': page}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code != 200:
break # 请求不正常通常网络是有问题，不再继续请求下去
try:
json = resp.json()
except Exception as e:
logger.log('DEBUG', e.args)
break
subdomains = self.match(self.domain, str(json))
if not subdomains:
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
# 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
subdomains = json.get('Subdomains')
if subdomains:
# ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
if len(subdomains) < 300:
break
page += 1
if page >= 50: # ipv4info子域查询接口最多允许查询50页
break
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = IPv4InfoAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
import api
from common.query import Query
from config import logger
class IPv4InfoAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'IPv4InfoAPIQuery'
self.addr = ' http://ipv4info.com/api_v1/'
self.api = api.ipv4info_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
page = 0
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'type': 'SUBDOMAINS', 'key': self.api,
'value': self.domain, 'page': page}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code != 200:
break # 请求不正常通常网络是有问题，不再继续请求下去
try:
json = resp.json()
except Exception as e:
logger.log('DEBUG', e.args)
break
subdomains = self.match(self.domain, str(json))
if not subdomains:
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
# 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
subdomains = json.get('Subdomains')
if subdomains:
# ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
if len(subdomains) < 300:
break
page += 1
if page >= 50: # ipv4info子域查询接口最多允许查询50页
break
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = IPv4InfoAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
oneforall/modules/datasets/passivedns_api.py
+3 -3
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -8,8 +8,8 @@ class PassiveDnsAPI(Query):
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'PassiveDnsQuery'
self.addr = config.passivedns_api_addr or 'http://api.passivedns.cn'
self.token = config.passivedns_api_token
self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
self.token = api.passivedns_api_token
def query(self):
"""
oneforall/modules/datasets/securitytrails_api.py
+3 -3
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -7,9 +7,9 @@ class SecurityTrailsAPI(Query):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'SecurityTrailsQuery'
self.source = 'SecurityTrailsAPIQuery'
self.addr = 'https://api.securitytrails.com/v1/domain/'
self.api = config.securitytrails_api
self.api = api.securitytrails_api
self.delay = 2 # SecurityTrails查询时延至少2秒
def query(self):
oneforall/modules/intelligence/riskiq_api.py
+3 -3
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -9,8 +9,8 @@ class RiskIQ(Query):
self.module = 'Intelligence'
self.source = 'RiskIQAPIQuery'
self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
self.user = config.riskiq_api_username
self.key = config.riskiq_api_key
self.user = api.riskiq_api_username
self.key = api.riskiq_api_key
def query(self):
"""
oneforall/modules/intelligence/threatbook_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -9,7 +9,7 @@ class ThreatBookAPI(Query):
self.module = 'Intelligence'
self.source = 'ThreatBookAPIQuery'
self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
self.key = config.threatbook_api_key
self.key = api.threatbook_api_key
def query(self):
"""
oneforall/modules/intelligence/virustotal_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common.query import Query
@@ -9,7 +9,7 @@ class VirusTotalAPI(Query):
self.module = 'Intelligence'
self.source = 'VirusTotalAPIQuery'
self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
self.key = config.virustotal_api_key
self.key = api.virustotal_api_key
def query(self):
"""
oneforall/modules/search/bing_api.py
+3 -3
View File
@@ -1,5 +1,5 @@
import time
import config
import api
from common.search import Search
@@ -11,8 +11,8 @@ class BingAPI(Search):
self.source = 'BingAPISearch'
self.addr = 'https://api.cognitive.microsoft.com/' \
'bingcustomsearch/v7.0/search'
self.id = config.bing_api_id
self.key = config.bing_api_key
self.id = api.bing_api_id
self.key = api.bing_api_key
self.limit_num = 1000 # 必应同一个搜索关键词限制搜索条数
self.delay = 1 # 必应自定义搜索限制时延1秒
oneforall/modules/search/fofa_api.py
+3 -3
View File
@@ -1,7 +1,7 @@
import base64
import time
import config
import api
from common.search import Search
from config import logger
@@ -14,8 +14,8 @@ class FoFa(Search):
self.source = 'FoFaAPISearch'
self.addr = 'https://fofa.so/api/v1/search/all'
self.delay = 1
self.email = config.fofa_api_email
self.key = config.fofa_api_key
self.email = api.fofa_api_email
self.key = api.fofa_api_key
def search(self):
"""
oneforall/modules/search/github.py
+3 -3
View File
@@ -1,7 +1,7 @@
import re
import time
import requests
import config
import api
from bs4 import BeautifulSoup
from common.search import Search
from config import logger
@@ -18,8 +18,8 @@ class Github(Search):
self.session = requests.Session()
self.login_url = 'https://github.com/login'
self.post_url = 'https://github.com/session'
self.email = config.github_email
self.password = config.github_password
self.email = api.github_email
self.password = api.github_password
def login_github(self):
"""
oneforall/modules/search/google_api.py
+3 -3
View File
@@ -1,5 +1,5 @@
import time
import config
import api
from common.search import Search
@@ -11,8 +11,8 @@ class GoogleAPI(Search):
self.source = 'GoogleAPISearch'
self.addr = 'https://www.googleapis.com/customsearch/v1'
self.delay = 1
self.key = config.google_api_key
self.cx = config.google_api_cx
self.key = api.google_api_key
self.cx = api.google_api_cx
self.per_page_num = 10 # 每次只能请求10个结果
def search(self, domain, filtered_subdomain='', full_search=False):
oneforall/modules/search/shodan_api.py
+2 -2
View File
@@ -1,4 +1,4 @@
import config
import api
from common.search import Search
@@ -9,7 +9,7 @@ class ShodanAPI(Search):
self.module = 'Search'
self.source = 'ShodanAPISearch'
self.addr = 'https://api.shodan.io/shodan/host/search'
self.key = config.shodan_api_key
self.key = api.shodan_api_key
def search(self):
"""
oneforall/modules/search/zoomeye_api.py
+3 -3
View File
@@ -1,5 +1,5 @@
import time
import config
import api
from common.search import Search
from config import logger
@@ -12,8 +12,8 @@ class ZoomEyeAPI(Search):
self.source = 'ZoomEyeAPISearch'
self.addr = 'https://api.zoomeye.org/web/search'
self.delay = 2
self.user = config.zoomeye_api_username
self.pwd = config.zoomeye_api_password
self.user = api.zoomeye_api_username
self.pwd = api.zoomeye_api_password
def login(self):
"""