Merge branch 'master' of https://github.com/shmilylty/OneForAll into master

common/utils.py

@@ -606,14 +606,14 @@ def check_version(local):
     try:
         resp = requests.get(url=api, headers=header, proxies=proxy,
                             timeout=timeout, verify=verify)
-        json = resp.json()
-        latest = json['tag_name']
+        resp_json = resp.json()
+        latest = resp_json['tag_name']
     except Exception as e:
         logger.log('ERROR', 'An error occurred while checking the latest version')
         logger.log('ERROR', e.args)
         return
     if latest > local:
-        change = json.get("body")
+        change = resp_json.get("body")
         logger.log('ALERT', f'The current version is {local} '
                             f'but the latest version is {latest}')
         logger.log('ALERT', f'The {latest} version mainly has the following changes')

config/__init__.py

@@ -7,11 +7,11 @@ class Settings(object):
         # 获取全局变量中的配置信息
         for attr in dir(default):
             setattr(self, attr, getattr(default, attr))
-
-        setting = importlib.import_module('config.setting')
-
-        for attr in dir(setting):
-            setattr(self, attr, getattr(setting, attr))
+        setting_modules = ['config.setting', 'config.api']
+        for setting_module in setting_modules:
+            setting = importlib.import_module(setting_module)
+            for attr in dir(setting):
+                setattr(self, attr, getattr(setting, attr))
 
 
 settings = Settings()

config/default.py

@@ -177,3 +177,80 @@ headers = {
     'X-Forwarded-For': '127.0.0.1'
 }
 random_user_agent = True  # 使用随机UA(默认True，开启可以覆盖header的设置)
+
+# 模块API配置
+# Censys可以免费注册获取API：https://censys.io/api
+censys_api_id = ''
+censys_api_secret = ''
+
+# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
+# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
+binaryedge_api = ''
+
+# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+chinaz_api = ''
+
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
+# cognitive-services/bing-web-search-api/#web-json
+bing_api_id = ''
+bing_api_key = ''
+
+# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
+securitytrails_api = ''
+
+# https://fofa.so/api
+fofa_api_email = ''  # fofa用户邮箱
+fofa_api_key = ''  # fofa用户key
+
+# Google可以免费注册获取API:
+# 免费的API只能查询前100条结果
+# https://developers.google.com/custom-search/v1/overview#search_engine_id
+# 创建自定义搜索引擎后需要在响应的控制面板上启用Search the entire web
+google_api_id = ''  # Google API自定义搜索引擎id
+# https://developers.google.com/custom-search/v1/overview#api_key
+google_api_key = ''  # Google API自定义搜索key
+
+# https://api.passivetotal.org/api/docs/
+riskiq_api_username = ''
+riskiq_api_key = ''
+
+# Shodan可以免费注册获取API: https://account.shodan.io/register
+# 免费的API限速1秒查询1次
+shodan_api_key = ''
+# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
+threatbook_api_key = ''
+
+# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
+virustotal_api_key = ''
+
+# https://www.zoomeye.org/doc?channel=api
+zoomeye_api_usermail = ''
+zoomeye_api_password = ''
+
+# Spyse可以免费注册获取API: https://spyse.com/
+spyse_api_token = ''
+
+# https://www.circl.lu/services/passive-dns/
+circl_api_username = ''
+circl_api_password = ''
+
+# https://www.dnsdb.info/
+dnsdb_api_key = ''
+
+# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
+# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
+ipv4info_api_key = ''
+
+# https://github.com/360netlab/flint
+# passivedns_api_addr默认空使用http://api.passivedns.cn
+# passivedns_api_token可为空
+passivedns_api_addr = ''
+passivedns_api_token = ''
+
+# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
+# 用于子域接管和子域收集
+github_api_user = ''
+github_api_token = ''
+
+# obtain Cloudflare API key from https://dash.cloudflare.com/profile/api-tokens
+cloudflare_api_token = ''

config/log.py

@@ -32,6 +32,7 @@ logger.level(name='ERROR', color='<red><bold>', icon='❌️')
 logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
 
 # 如果你想在命令终端静默运行OneForAll，可以将以下一行中的level设置为QUITE
-logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)  # 命令终端日志级别默认为INFOR
-logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
-           encoding='utf-8')  # 日志文件默认为级别为DEBUG
+# 命令终端日志级别默认为INFOR
+logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
+# 日志文件默认为级别为DEBUG
+logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True)

modules/autotake/github.py

@@ -8,7 +8,7 @@ github自动接管
 import json
 import base64
 import requests
-from config import api
+from config import settings
 
 HEADERS = {
     "Accept": "application/json, text/javascript, */*; q=0.01",
@@ -21,8 +21,8 @@ def github_takeover(url):
     # 读取config配置文件
     repo_name = url
     print('[*]正在读取配置文件')
-    user = api.github_api_user
-    token = api.github_api_token
+    user = settings.github_api_user
+    token = settings.github_api_token
     headers = {
         "Authorization": 'token ' + token,
         "Accept": "application/vnd.github.switcheroo-preview+json"

modules/certificates/censys_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 from config.log import logger
 
@@ -10,8 +10,8 @@ class CensysAPI(Query):
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
         self.addr = 'https://www.censys.io/api/v1/search/certificates'
-        self.id = api.censys_api_id
-        self.secret = api.censys_api_secret
+        self.id = settings.censys_api_id
+        self.secret = settings.censys_api_secret
         self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次
 
     def query(self):

modules/datasets/binaryedge_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class BinaryEdgeAPI(Query):
         self.module = 'Dataset'
         self.source = 'BinaryEdgeAPIQuery'
         self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
-        self.api = api.binaryedge_api
+        self.api = settings.binaryedge_api
 
     def query(self):
         """

modules/datasets/chinaz_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class ChinazAPI(Query):
         self.module = 'Dataset'
         self.source = 'ChinazAPIQuery'
         self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
-        self.api = api.chinaz_api
+        self.api = settings.chinaz_api
 
     def query(self):
         """

modules/datasets/circl_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,8 +9,8 @@ class CirclAPI(Query):
         self.module = 'Dataset'
         self.source = 'CirclAPIQuery'
         self.addr = 'https://www.circl.lu/pdns/query/'
-        self.user = api.circl_api_username
-        self.pwd = api.circl_api_password
+        self.user = settings.circl_api_username
+        self.pwd = settings.circl_api_password
 
     def query(self):
         """

modules/datasets/cloudflare_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 from config.log import logger
 from time import sleep
@@ -11,7 +11,7 @@ class CloudFlareAPI(Query):
         self.domain = domain
         self.module = 'Dataset'
         self.source = 'CloudFlareAPIQuery'
-        self.token = api.cloudflare_api_token
+        self.token = settings.cloudflare_api_token
         self.addr = 'https://api.cloudflare.com/client/v4/'
         self.header = self.get_header()
         self.header.update({'Authorization': 'Bearer ' + self.token})

modules/datasets/dnsdb_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class DNSdbAPI(Query):
         self.module = 'Dataset'
         self.source = 'DNSdbAPIQuery'
         self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
-        self.api = api.dnsdb_api_key
+        self.api = settings.dnsdb_api_key
 
     def query(self):
         """

modules/datasets/ipv4info_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 from config.log import logger
 
@@ -10,7 +10,7 @@ class IPv4InfoAPI(Query):
         self.module = 'Dataset'
         self.source = 'IPv4InfoAPIQuery'
         self.addr = ' http://ipv4info.com/api_v1/'
-        self.api = api.ipv4info_api_key
+        self.api = settings.ipv4info_api_key
 
     def query(self):
         """

modules/datasets/passivedns_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -8,8 +8,8 @@ class PassiveDnsAPI(Query):
         self.domain = domain
         self.module = 'Dataset'
         self.source = 'PassiveDnsQuery'
-        self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
-        self.token = api.passivedns_api_token
+        self.addr = settings.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.token = settings.passivedns_api_token
 
     def query(self):
         """

modules/datasets/qianxun.py

@@ -25,9 +25,10 @@ class QianXun(Query):
             url = f'https://www.dnsscan.cn/dns.html?' \
                   f'keywords={self.domain}&page={num}'
             resp = self.post(url, data)
-            self.subdomains = self.collect_subdomains(resp)
-            if resp is None:
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 没有发现子域名则停止查询
                 break
+            self.subdomains = self.subdomains.union(subdomains)
             if '<div id="page" class="pagelist">' not in resp.text:
                 break
             if '<li class="disabled"><span>&raquo;</span></li>' in resp.text:

modules/datasets/securitytrails_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class SecurityTrailsAPI(Query):
         self.module = 'Dataset'
         self.source = 'SecurityTrailsAPIQuery'
         self.addr = 'https://api.securitytrails.com/v1/domain/'
-        self.api = api.securitytrails_api
+        self.api = settings.securitytrails_api
         self.delay = 2  # SecurityTrails查询时延至少2秒
 
     def query(self):

modules/datasets/sitedossier.py

@@ -23,7 +23,7 @@ class SiteDossier(Query):
             url = f'{self.addr}{self.domain}/{self.page_num}'
             resp = self.get(url)
             subdomains = self.match_subdomains(resp)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
+            if not subdomains:  # 没有发现子域名则停止查询
                 break
             self.subdomains = self.subdomains.union(subdomains)
             # 搜索页面没有出现下一页时停止搜索

modules/datasets/spyse_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -8,7 +8,7 @@ class SpyseAPI(Query):
         self.domain = domain
         self.module = 'Dataset'
         self.source = 'SpyseAPIQuery'
-        self.token = api.spyse_api_token
+        self.token = settings.spyse_api_token
 
     def query(self):
         """
@@ -27,7 +27,7 @@ class SpyseAPI(Query):
                 return
             json = resp.json()
             subdomains = self.match_subdomains(str(json))
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
+            if not subdomains:  # 没有发现子域名则停止查询
                 break
             self.subdomains = self.subdomains.union(subdomains)
             offset += limit

modules/datasets/wzpc.py

@@ -32,6 +32,8 @@ class WZPCQuery(Query):
             if not resp:
                 break
             subdomains = self.match_subdomains(resp.text)
+            if not subdomains:  # 没有发现子域名则停止查询
+                break
             self.subdomains = self.subdomains.union(subdomains)
             if not subdomains:
                 break

modules/datasets/ximcx.py

@@ -17,11 +17,7 @@ class Ximcx(Query):
         self.proxy = self.get_proxy(self.source)
         data = {'domain': self.domain}
         resp = self.post(self.addr, data=data)
-        if not resp:
-            return
-        json = resp.json()
-        subdomains = self.match_subdomains(str(json))
-        self.subdomains = self.subdomains.union(subdomains)
+        self.subdomains = self.collect_subdomains(resp)
 
     def run(self):
         """

modules/finder.py

@@ -116,7 +116,11 @@ def filter_name(path, black_name):
 
 
 def filter_url(domain, url, black_name):
-    raw_url = parse.urlparse(url)
+    try:
+        raw_url = parse.urlparse(url)
+    except Exception as e:  # 解析失败则跳过该URL
+        logger.log('DEBUG', url, e.args)
+        return True
     scheme = raw_url.scheme.lower()
     if not scheme:
         return True

modules/intelligence/riskiq_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,8 +9,8 @@ class RiskIQ(Query):
         self.module = 'Intelligence'
         self.source = 'RiskIQAPIQuery'
         self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
-        self.user = api.riskiq_api_username
-        self.key = api.riskiq_api_key
+        self.user = settings.riskiq_api_username
+        self.key = settings.riskiq_api_key
 
     def query(self):
         """

modules/intelligence/threatbook_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class ThreatBookAPI(Query):
         self.module = 'Intelligence'
         self.source = 'ThreatBookAPIQuery'
         self.addr = 'https://api.threatbook.cn/v3/domain/sub_domains'
-        self.key = api.threatbook_api_key
+        self.key = settings.threatbook_api_key
 
     def query(self):
         """

modules/intelligence/virustotal_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.query import Query
 
 
@@ -9,7 +9,7 @@ class VirusTotalAPI(Query):
         self.module = 'Intelligence'
         self.source = 'VirusTotalAPIQuery'
         self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
-        self.key = api.virustotal_api_key
+        self.key = settings.virustotal_api_key
 
     def query(self):
         """

modules/search/bing_api.py

@@ -1,5 +1,5 @@
 import time
-from config import api
+from config import settings
 from common.search import Search
 
 
@@ -10,8 +10,8 @@ class BingAPI(Search):
         self.module = 'Search'
         self.source = 'BingAPISearch'
         self.addr = 'https://api.cognitive.microsoft.com/bing/v7.0/search'
-        self.id = api.bing_api_id
-        self.key = api.bing_api_key
+        self.id = settings.bing_api_id
+        self.key = settings.bing_api_key
         self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
         self.delay = 1  # 必应自定义搜索限制时延1秒
 

modules/search/fofa_api.py

@@ -1,7 +1,7 @@
 import base64
 import time
 
-from config import api
+from config import settings
 from common.search import Search
 
 
@@ -13,8 +13,8 @@ class FoFa(Search):
         self.source = 'FoFaAPISearch'
         self.addr = 'https://fofa.so/api/v1/search/all'
         self.delay = 1
-        self.email = api.fofa_api_email
-        self.key = api.fofa_api_key
+        self.email = settings.fofa_api_email
+        self.key = settings.fofa_api_key
 
     def search(self):
         """

modules/search/github_api.py

@@ -1,5 +1,5 @@
 import requests
-from config import api
+from config import settings
 from common.search import Search
 from config.log import logger
 
@@ -13,7 +13,7 @@ class GithubAPI(Search):
         self.domain = domain
         self.session = requests.Session()
         self.auth_url = 'https://api.github.com'
-        self.token = api.github_api_token
+        self.token = settings.github_api_token
 
     def auth_github(self):
         """

modules/search/google_api.py

@@ -1,5 +1,5 @@
 import time
-from config import api
+from config import settings
 from common.search import Search
 
 
@@ -11,8 +11,8 @@ class GoogleAPI(Search):
         self.source = 'GoogleAPISearch'
         self.addr = 'https://www.googleapis.com/customsearch/v1'
         self.delay = 1
-        self.key = api.google_api_key
-        self.id = api.google_api_id
+        self.key = settings.google_api_key
+        self.id = settings.google_api_id
         self.per_page_num = 10  # 每次只能请求10个结果
 
     def search(self, domain, filtered_subdomain=''):

modules/search/shodan_api.py

@@ -1,4 +1,4 @@
-from config import api
+from config import settings
 from common.search import Search
 
 
@@ -9,7 +9,7 @@ class ShodanAPI(Search):
         self.module = 'Search'
         self.source = 'ShodanAPISearch'
         self.addr = 'https://api.shodan.io/shodan/host/search'
-        self.key = api.shodan_api_key
+        self.key = settings.shodan_api_key
 
     def search(self):
         """

modules/search/zoomeye_api.py

@@ -1,5 +1,5 @@
 import time
-from config import api
+from config import settings
 from common.search import Search
 from config.log import logger
 
@@ -12,8 +12,8 @@ class ZoomEyeAPI(Search):
         self.source = 'ZoomEyeAPISearch'
         self.addr = 'https://api.zoomeye.org/web/search'
         self.delay = 2
-        self.user = api.zoomeye_api_usermail
-        self.pwd = api.zoomeye_api_password
+        self.user = settings.zoomeye_api_usermail
+        self.pwd = settings.zoomeye_api_password
 
     def login(self):
         """