api配置分离

oneforall/api.py

@@ -0,0 +1,74 @@
+# 模块API配置
+# Censys可以免费注册获取API：https://censys.io/api
+censys_api_id = ''
+censys_api_secret = ''
+
+# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
+# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
+binaryedge_api = ''
+
+# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+chinaz_api = ''
+
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
+# cognitive-services/bing-web-search-api/#web-json
+bing_api_id = ''
+bing_api_key = ''
+
+# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
+securitytrails_api = ''
+
+# https://fofa.so/api
+fofa_api_email = ''  # fofa用户邮箱
+fofa_api_key = ''  # fofa用户key
+
+# Google可以免费注册获取API:
+# https://developers.google.com/custom-search/v1/overview
+# 免费的API只能查询前100条结果
+google_api_key = ''  # Google API搜索key
+google_api_cx = ''  # Google API搜索cx
+
+# https://api.passivetotal.org/api/docs/
+riskiq_api_username = ''
+riskiq_api_key = ''
+
+# Shodan可以免费注册获取API: https://account.shodan.io/register
+# 免费的API限速1秒查询1次
+shodan_api_key = ''
+# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
+threatbook_api_key = ''
+
+# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
+virustotal_api_key = ''
+
+# https://www.zoomeye.org/doc?channel=api
+zoomeye_api_username = ''
+zoomeye_api_password = ''
+
+# Spyse可以免费注册获取API: https://spyse.com/
+spyse_api_token = ''
+
+# https://www.circl.lu/services/passive-dns/
+circl_api_username = ''
+circl_api_password = ''
+
+# https://www.dnsdb.info/
+dnsdb_api_key = ''
+
+# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
+# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
+ipv4info_api_key = ''
+
+# https://github.com/360netlab/flint
+# passivedns_api_addr默认空使用http://api.passivedns.cn
+# passivedns_api_token可为空
+passivedns_api_addr = ''
+passivedns_api_token = ''
+
+# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
+# 用于子域接管
+github_api_user = ''
+github_api_token = ''
+# Github子域收集模块使用
+github_email = ''
+github_password = ''

oneforall/config.py

@@ -107,7 +107,7 @@ ports = {'default': default_ports, 'small': small_ports, 'large': large_ports}
 verify_ssl = False
 # aiohttp 支持 HTTP/HTTPS形式的代理
 get_proxy = None  # proxy="http://user:pass@some.proxy.com"
-get_timeout = 200  # http请求探测总超时时间 None或者0则表示不检测超时
+get_timeout = 5 * 60  # http请求探测总超时时间 None或者0则表示不检测超时 默认5分钟
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
 
@@ -115,81 +115,7 @@ fake_header = True  # 使用伪造请求头
 limit_open_conn = 500  # 默认500
 
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
-limit_per_host = 50  # 默认0表示不限制
+limit_per_host = 50  # 0表示不限制
-
-# 模块API配置
-# Censys可以免费注册获取API：https://censys.io/api
-censys_api_id = ''
-censys_api_secret = ''
-
-# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
-# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
-binaryedge_api = ''
-
-# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
-chinaz_api = ''
-
-# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
-# cognitive-services/bing-web-search-api/#web-json
-bing_api_id = ''
-bing_api_key = ''
-
-# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
-securitytrails_api = ''
-
-# https://fofa.so/api
-fofa_api_email = ''  # fofa用户邮箱
-fofa_api_key = ''  # fofa用户key
-
-# Google可以免费注册获取API:
-# https://developers.google.com/custom-search/v1/overview
-# 免费的API只能查询前100条结果
-google_api_key = ''  # Google API搜索key
-google_api_cx = ''  # Google API搜索cx
-
-# https://api.passivetotal.org/api/docs/
-riskiq_api_username = ''
-riskiq_api_key = ''
-
-# Shodan可以免费注册获取API: https://account.shodan.io/register
-# 免费的API限速1秒查询1次
-shodan_api_key = ''
-# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
-threatbook_api_key = ''
-
-# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
-virustotal_api_key = ''
-
-# https://www.zoomeye.org/doc?channel=api
-zoomeye_api_username = ''
-zoomeye_api_password = ''
-
-# Spyse可以免费注册获取API: https://spyse.com/
-spyse_api_token = ''
-
-# https://www.circl.lu/services/passive-dns/
-circl_api_username = ''
-circl_api_password = ''
-
-# https://www.dnsdb.info/
-dnsdb_api_key = ''
-
-# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
-# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
-ipv4info_api_key = ''
-
-# https://github.com/360netlab/flint
-# passivedns_api_addr默认空使用http://api.passivedns.cn
-# passivedns_api_token可为空
-passivedns_api_addr = ''
-passivedns_api_token = ''
-
-# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
-github_api_user = ''
-github_api_token = ''
-# github子域收集模块使用
-github_email = ''
-github_password = ''
 
 subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
                      'web', 'www', 'job', 'docs', 'news', 'blog', 'data',

oneforall/modules/autotake/github.py

@@ -8,7 +8,7 @@ github自动接管
 import json
 import base64
 import requests
-import config
+import api
 
 HEADERS = {
     "Accept": "application/json, text/javascript, */*; q=0.01",
@@ -21,8 +21,8 @@ def github_takeover(url):
     # 读取config配置文件
     repo_name = url
     print('[*]正在读取配置文件...')
-    user = config.github_api_user
+    user = api.github_api_user
-    token = config.github_api_token
+    token = api.github_api_token
     CHECK_HEADERS = {
         "Authorization": 'token ' + token,
         "Accept": "application/vnd.github.switcheroo-preview+json"

oneforall/modules/certificates/censys_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 from config import logger
 
@@ -10,8 +10,8 @@ class CensysAPI(Query):
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
         self.addr = 'https://www.censys.io/api/v1/search/certificates'
-        self.id = config.censys_api_id
+        self.id = api.censys_api_id
-        self.secret = config.censys_api_secret
+        self.secret = api.censys_api_secret
         self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次
 
     def query(self):

oneforall/modules/certificates/spyse_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common import utils
 from common.query import Query
 
@@ -10,7 +10,7 @@ class SpyseAPI(Query):
         self.module = 'Certificate'
         self.source = 'CertDBAPIQuery'
         self.addr = 'https://api.spyse.com/v1/subdomains'
-        self.token = config.spyse_api_token
+        self.token = api.spyse_api_token
 
     def query(self):
         """

oneforall/modules/datasets/binaryedge_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class BinaryEdgeAPI(Query):
         self.module = 'Dataset'
         self.source = 'BinaryEdgeAPIQuery'
         self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
-        self.api = config.binaryedge_api
+        self.api = api.binaryedge_api
 
     def query(self):
         """

oneforall/modules/datasets/chinaz_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class ChinazAPI(Query):
         self.module = 'Dataset'
         self.source = 'ChinazAPIQuery'
         self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
-        self.api = config.chinaz_api
+        self.api = api.chinaz_api
 
     def query(self):
         """

oneforall/modules/datasets/circl_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -9,8 +9,8 @@ class CirclAPI(Query):
         self.module = 'Dataset'
         self.source = 'CirclAPIQuery'
         self.addr = 'https://www.circl.lu/pdns/query/'
-        self.user = config.circl_api_username
+        self.user = api.circl_api_username
-        self.pwd = config.circl_api_password
+        self.pwd = api.circl_api_password
 
     def query(self):
         """

oneforall/modules/datasets/dnsdb_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common import utils
 from common.query import Query
 
@@ -10,7 +10,7 @@ class DNSdbAPI(Query):
         self.module = 'Dataset'
         self.source = 'DNSdbAPIQuery'
         self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
-        self.api = config.dnsdb_api_key
+        self.api = api.dnsdb_api_key
 
     def query(self):
         """

oneforall/modules/datasets/ipv4info_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 from config import logger
 
@@ -10,7 +10,7 @@ class IPv4InfoAPI(Query):
         self.module = 'Dataset'
         self.source = 'IPv4InfoAPIQuery'
         self.addr = ' http://ipv4info.com/api_v1/'
-        self.api = config.ipv4info_api_key
+        self.api = api.ipv4info_api_key
 
     def query(self):
         """

oneforall/modules/datasets/passivedns_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -8,8 +8,8 @@ class PassiveDnsAPI(Query):
         self.domain = self.register(domain)
         self.module = 'Dataset'
         self.source = 'PassiveDnsQuery'
-        self.addr = config.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
-        self.token = config.passivedns_api_token
+        self.token = api.passivedns_api_token
 
     def query(self):
         """

oneforall/modules/datasets/securitytrails_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -7,9 +7,9 @@ class SecurityTrailsAPI(Query):
         Query.__init__(self)
         self.domain = self.register(domain)
         self.module = 'Dataset'
-        self.source = 'SecurityTrailsQuery'
+        self.source = 'SecurityTrailsAPIQuery'
         self.addr = 'https://api.securitytrails.com/v1/domain/'
-        self.api = config.securitytrails_api
+        self.api = api.securitytrails_api
         self.delay = 2  # SecurityTrails查询时延至少2秒
 
     def query(self):

oneforall/modules/intelligence/riskiq_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -9,8 +9,8 @@ class RiskIQ(Query):
         self.module = 'Intelligence'
         self.source = 'RiskIQAPIQuery'
         self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
-        self.user = config.riskiq_api_username
+        self.user = api.riskiq_api_username
-        self.key = config.riskiq_api_key
+        self.key = api.riskiq_api_key
 
     def query(self):
         """

oneforall/modules/intelligence/threatbook_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class ThreatBookAPI(Query):
         self.module = 'Intelligence'
         self.source = 'ThreatBookAPIQuery'
         self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
-        self.key = config.threatbook_api_key
+        self.key = api.threatbook_api_key
 
     def query(self):
         """

oneforall/modules/intelligence/virustotal_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class VirusTotalAPI(Query):
         self.module = 'Intelligence'
         self.source = 'VirusTotalAPIQuery'
         self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
-        self.key = config.virustotal_api_key
+        self.key = api.virustotal_api_key
 
     def query(self):
         """

oneforall/modules/search/bing_api.py

@@ -1,5 +1,5 @@
 import time
-import config
+import api
 from common.search import Search
 
 
@@ -11,8 +11,8 @@ class BingAPI(Search):
         self.source = 'BingAPISearch'
         self.addr = 'https://api.cognitive.microsoft.com/' \
                     'bingcustomsearch/v7.0/search'
-        self.id = config.bing_api_id
+        self.id = api.bing_api_id
-        self.key = config.bing_api_key
+        self.key = api.bing_api_key
         self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
         self.delay = 1  # 必应自定义搜索限制时延1秒
 

oneforall/modules/search/fofa_api.py

@@ -1,7 +1,7 @@
 import base64
 import time
 
-import config
+import api
 from common.search import Search
 from config import logger
 
@@ -14,8 +14,8 @@ class FoFa(Search):
         self.source = 'FoFaAPISearch'
         self.addr = 'https://fofa.so/api/v1/search/all'
         self.delay = 1
-        self.email = config.fofa_api_email
+        self.email = api.fofa_api_email
-        self.key = config.fofa_api_key
+        self.key = api.fofa_api_key
 
     def search(self):
         """

oneforall/modules/search/github.py

@@ -1,7 +1,7 @@
 import re
 import time
 import requests
-import config
+import api
 from bs4 import BeautifulSoup
 from common.search import Search
 from config import logger
@@ -18,8 +18,8 @@ class Github(Search):
         self.session = requests.Session()
         self.login_url = 'https://github.com/login'
         self.post_url = 'https://github.com/session'
-        self.email = config.github_email
+        self.email = api.github_email
-        self.password = config.github_password
+        self.password = api.github_password
 
     def login_github(self):
         """

oneforall/modules/search/google_api.py

@@ -1,5 +1,5 @@
 import time
-import config
+import api
 from common.search import Search
 
 
@@ -11,8 +11,8 @@ class GoogleAPI(Search):
         self.source = 'GoogleAPISearch'
         self.addr = 'https://www.googleapis.com/customsearch/v1'
         self.delay = 1
-        self.key = config.google_api_key
+        self.key = api.google_api_key
-        self.cx = config.google_api_cx
+        self.cx = api.google_api_cx
         self.per_page_num = 10  # 每次只能请求10个结果
 
     def search(self, domain, filtered_subdomain='', full_search=False):

oneforall/modules/search/shodan_api.py

@@ -1,4 +1,4 @@
-import config
+import api
 from common.search import Search
 
 
@@ -9,7 +9,7 @@ class ShodanAPI(Search):
         self.module = 'Search'
         self.source = 'ShodanAPISearch'
         self.addr = 'https://api.shodan.io/shodan/host/search'
-        self.key = config.shodan_api_key
+        self.key = api.shodan_api_key
 
     def search(self):
         """

oneforall/modules/search/zoomeye_api.py

@@ -1,5 +1,5 @@
 import time
-import config
+import api
 from common.search import Search
 from config import logger
 
@@ -12,8 +12,8 @@ class ZoomEyeAPI(Search):
         self.source = 'ZoomEyeAPISearch'
         self.addr = 'https://api.zoomeye.org/web/search'
         self.delay = 2
-        self.user = config.zoomeye_api_username
+        self.user = api.zoomeye_api_username
-        self.pwd = config.zoomeye_api_password
+        self.pwd = api.zoomeye_api_password
 
     def login(self):
         """