Merge pull request #411 from Sci-Twi/patch-1

exporting utf-8 csv files

.dockerignore

@@ -0,0 +1 @@
+.git/

.github/workflows/docker-image.yml

@@ -0,0 +1,32 @@
+name: Release Docker Image
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+
+  build:
+    name: build and push
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out the repo
+      uses: actions/checkout@v3
+    
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+      
+    - name: Setup Docker buildx
+      uses: docker/setup-buildx-action@v2
+      
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_PASS }}
+    
+    - name: Build the Docker image
+      run: docker buildx build --push --platform linux/amd64,linux/arm64 -t shmilylty/oneforall:latest .

Dockerfile

@@ -1,5 +1,5 @@
-FROM python:3.8-alpine3.10
-MAINTAINER milktea@vmoe.info
+FROM --platform=$TARGETPLATFORM python:3.8-alpine3.10
+LABEL maintainer="milktea@vmoe.info"
 
 RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
 RUN apk update && apk --no-cache add git build-base libffi-dev libxml2-dev libxslt-dev libressl-dev
@@ -10,7 +10,7 @@ RUN git clone https://github.com/blechschmidt/massdns
 WORKDIR /massdns
 RUN make 
 ADD . /OneForAll/
-RUN mv /massdns/bin/massdns /OneForAll/thirdparty/massdns/massdns_linux_x86_64
+RUN mv /massdns/bin/massdns /OneForAll/thirdparty/massdns/massdns_linux_$(uname -m)
 RUN mkdir /OneForAll/results
 WORKDIR /OneForAll/
 

README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
 [![python](https://img.shields.io/badge/python-3.6+-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.4-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.5-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
 
@@ -80,6 +80,8 @@ git stash pop    # 释放本地修改
 ```
 config
 ├── api.py
+├── default.py
+├── __init__.py
 ├── log.py
 └── setting.py
 ```
@@ -227,7 +229,7 @@ FLAGS
   1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certspotter`，`crtsh`，`entrust`，`google`，`spyse_api`）
   2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，利用NSEC记录遍历DNS域`dnssec`，后续会添加NSEC3记录等模块）
   3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
-  4. 利用DNS数据集收集子域（目前有24个模块：`binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`）
+  4. 利用DNS数据集收集子域（目前有24个模块：`bevigil_api`, `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`）
   5. 利用DNS查询收集子域（目前有5个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，以及通过查询域名的DNS记录中的MX,NS,SOA,TXT记录来收集子域）
   6. 利用威胁情报平台数据收集子域（目前有6个模块：`alienvault`, `riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
   7. 利用搜索引擎发现子域（目前有18个模块：`ask`, `baidu`, `bing`, `bing_api`, `duckduckgo`, `exalead`, `fofa_api`, `gitee`, `github`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。

brute.py

@@ -342,7 +342,6 @@ class Brute(Module):
         self.ips_times = dict()  # IP集合出现次数
         self.enable_wildcard = None  # 当前域名是否使用泛解析
         self.quite = False
-        self.in_china = None
 
     def gen_brute_dict(self, domain):
         logger.log('INFOR', f'Generating dictionary for {domain}')
@@ -425,7 +424,7 @@ class Brute(Module):
 
         if self.enable_wildcard:
             wildcard_ips, wildcard_ttl = wildcard.collect_wildcard_record(domain, ns_ip_list)
-        ns_path = utils.get_ns_path(self.in_china, self.enable_wildcard, ns_ip_list)
+        ns_path = utils.get_ns_path(settings.use_china_nameservers, self.enable_wildcard, ns_ip_list)
 
         dict_set = self.gen_brute_dict(domain)
 
@@ -459,8 +458,6 @@ class Brute(Module):
 
     def run(self):
         logger.log('INFOR', f'Start running {self.source} module')
-        if self.in_china is None:
-            _, self.in_china = utils.get_net_env()
         self.domains = utils.get_domains(self.target, self.targets)
         for self.domain in self.domains:
             self.results = list()  # 置空

common/domain.py

@@ -55,6 +55,8 @@ class Domain(object):
 
         :return: registered domain result
         """
+        if not settings.use_tld_extract:
+            return self.string
         result = self.extract()
         if result:
             return result.registered_domain

common/records.py

@@ -230,7 +230,7 @@ class RecordCollection(object):
 
         # Ensure that we don't have more than one row.
         try:
-            self[1]
+            return self[1]
         except IndexError:
             return self.first(default=default, as_dict=as_dict,
                               as_ordereddict=as_ordereddict)

common/search.py

@@ -69,8 +69,9 @@ class Search(Module):
     def recursive_subdomain(self):
         # 递归搜索下一层的子域
         # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+        subdomains = self.subdomains.copy()
         for layer_num in range(1, self.recursive_times):
-            for subdomain in self.subdomains:
+            for subdomain in subdomains:
                 # 进行下一层子域搜索的限制条件
                 count = subdomain.count('.') - self.domain.count('.')
                 if count == layer_num:

common/utils.py

@@ -248,11 +248,11 @@ def save_to_file(path, data):
     :return: 保存成功与否
     """
     try:
-        with open(path, 'w', errors='ignore', newline='') as file:
+        with open(path, 'w', errors='ignore', newline='', encoding='utf-8') as file:
             file.write(data)
             return True
     except TypeError:
-        with open(path, 'wb') as file:
+        with open(path, 'wb', encoding='utf-8') as file:
             file.write(data)
             return True
     except Exception as e:
@@ -330,6 +330,8 @@ def export_all_results(path, name, fmt, datas):
         row_list.append(Record(keys, values))
     rows = RecordCollection(iter(row_list))
     content = rows.export(fmt)
+    if fmt == 'csv':
+        content = '\ufeff' + content
     save_to_file(path, content)
 
 
@@ -495,28 +497,35 @@ def delete_file(*paths):
 @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                 wait=tenacity.wait_fixed(2))
 def check_net():
-    urls = ['http://ip-api.com/json/']
-    url = random.choice(urls)
-    header = {'User_Agent': 'curl'}
-    timeout = settings.request_timeout_second
-    verify = settings.request_ssl_verify
-    logger.log('DEBUG', f'Trying to access {url}')
-    session = requests.Session()
-    session.trust_env = False
-    try:
-        rsp = session.get(url, headers=header, timeout=timeout, verify=verify)
-    except Exception as e:
-        logger.log('ERROR', e.args)
-        logger.log('ALERT', 'Unable to access Internet, retrying...')
-        raise e
-    logger.log('DEBUG', 'Access to Internet OK')
-    country = rsp.json().get('country').lower()
-    if country in ['cn', 'china']:
-        logger.log('DEBUG', f'The computer is located in China')
-        return True, True
-    else:
-        logger.log('DEBUG', f'The computer is not located in China')
-        return True, False
+    times = 0
+    while True:
+        times += 1
+        urls = ['https://www.baidu.com', 'https://www.bing.com',
+                'https://www.cloudflare.com', 'https://www.akamai.com/',
+                'https://www.fastly.com/', 'https://www.amazon.com/']
+        url = random.choice(urls)
+        logger.log('DEBUG', f'Trying to access {url}')
+        header = get_random_header()
+        proxy = get_proxy()
+        timeout = settings.request_timeout_second
+        verify = settings.request_ssl_verify
+        session = requests.Session()
+        session.trust_env = False
+        session = requests.Session()
+        session.trust_env = False
+        try:
+            rsp = session.get(url, headers=header, proxies=proxy,
+                              timeout=timeout, verify=verify)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            logger.log('ALERT', f'Unable to access Internet, retrying for the {times}th time')
+        else:
+            if rsp.status_code == 200:
+                logger.log('DEBUG', 'Access to Internet OK')
+                return True
+        if times >= 3:
+            logger.log('ALERT', 'Access to Internet failed')
+            return False
 
 
 def check_dep():
@@ -538,7 +547,7 @@ def get_net_env():
     except Exception as e:
         logger.log('DEBUG', e.args)
         logger.log('ALERT', 'Please check your network environment.')
-        return False, None
+        return False
     return result
 
 

config/api.py

@@ -7,6 +7,9 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''
 
+# BeVigil API: https://bevigil.com/osint-api
+bevigil_api = ''
+
 # Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''
 
@@ -44,8 +47,7 @@ threatbook_api_key = ''
 virustotal_api_key = ''
 
 # https://www.zoomeye.org/doc?channel=api
-zoomeye_api_usermail = ''
-zoomeye_api_password = ''
+zoomeye_api_key = ''
 
 # Spyse可以免费注册获取API: https://spyse.com/
 spyse_api_token = ''
@@ -74,3 +76,19 @@ github_api_token = ''
 
 # obtain Cloudflare API key from https://dash.cloudflare.com/profile/api-tokens
 cloudflare_api_token = ''
+
+# https://hunter.qianxin.com/home/userInfo
+hunter_api_key = ''
+
+# https://api-docs.fullhunt.io/
+fullhunt_api_key = ''
+
+
+# 登录quake之后可在个人中心获取key https://quake.360.net/quake/#/personal?tab=message
+quake_api_key = ''
+
+#https://www.racent.com/ctlog F2>Network抓包获取Token
+racent_api_token = ''
+
+# https://windvane.lichoin.com/
+windvane_api_token = ''

config/default.py

@@ -18,6 +18,7 @@ result_save_dir = relative_directory.joinpath('results')  # 结果保存目录
 temp_save_dir = result_save_dir.joinpath('temp')
 
 # OneForAll入口参数设置
+enable_check_network = True  # 开启网络环境检查
 enable_check_version = True  # 开启最新版本检查
 enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
@@ -54,6 +55,7 @@ brute_socket_num = 1  # 爆破时每个进程下的socket数量
 brute_resolve_num = 15  # 解析失败时尝试换名称服务器重查次数
 # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
 brute_wordlist_path = None
+use_china_nameservers = True  # 使用中国域名服务器 如果你所在网络不在中国则建议设置False
 # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
 authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
 enable_recursive_brute = False  # 是否使用递归爆破(默认False)
@@ -173,6 +175,9 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''
 
+# BeVigil API: https://bevigil.com/osint-api
+bevigil_api = ''
+
 # Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''
 
@@ -210,8 +215,7 @@ threatbook_api_key = ''
 virustotal_api_key = ''
 
 # https://www.zoomeye.org/doc?channel=api
-zoomeye_api_usermail = ''
-zoomeye_api_password = ''
+zoomeye_api_key = ''
 
 # Spyse可以免费注册获取API: https://spyse.com/
 spyse_api_token = ''
@@ -246,3 +250,6 @@ hunter_api_key = ''
 
 # https://api-docs.fullhunt.io/
 fullhunt_api_key = ''
+
+# https://windvane.lichoin.com/
+windvane_api_token = ''

config/log.py

@@ -23,13 +23,13 @@ logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
               '<blue>{line}</blue> - <level>{message}</level>'
 
 logger.remove()
-logger.level(name='TRACE', color='<cyan><bold>', icon='✏️')
-logger.level(name='DEBUG', color='<blue><bold>', icon='🐞 ')
-logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️')
-logger.level(name='QUITE', no=25, color='<green><bold>', icon='🤫 ')
-logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
-logger.level(name='ERROR', color='<red><bold>', icon='❌️')
-logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
+logger.level(name='TRACE', color='<cyan><bold>')
+logger.level(name='DEBUG', color='<blue><bold>')
+logger.level(name='INFOR', no=20, color='<green><bold>')
+logger.level(name='QUITE', no=25, color='<green><bold>')
+logger.level(name='ALERT', no=30, color='<yellow><bold>')
+logger.level(name='ERROR', color='<red><bold>')
+logger.level(name='FATAL', no=50, color='<RED><bold>')
 
 # 如果你想在命令终端静默运行OneForAll，可以将以下一行中的level设置为QUITE
 # 命令终端日志级别默认为INFOR

config/setting.py

@@ -10,6 +10,7 @@ relative_directory = pathlib.Path(__file__).parent.parent  # OneForAll代码相
 data_storage_dir = relative_directory.joinpath('data')  # 数据存放目录
 
 # OneForAll入口参数设置
+enable_check_network = True  # 开启网络环境检查
 enable_check_version = True  # 开启最新版本检查
 enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
@@ -38,6 +39,7 @@ enable_partial_module = []  # 启用部分收集模块 必须禁用enable_all_mo
 brute_concurrent_num = 2000  # 爆破时并发查询数量(默认2000，最大推荐10000)
 # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
 brute_wordlist_path = None
+use_china_nameservers = True  # 使用中国域名服务器 如果你所在网络不在中国则建议设置False
 enable_recursive_brute = False  # 是否使用递归爆破(默认False)
 brute_recursive_depth = 2  # 递归爆破深度(默认2层)
 # 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
@@ -67,7 +69,7 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                         'YandexSearch', 'CrossDomainXml',
                         'ContentSecurityPolicy']  # 代理自定义的模块
 request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
-                       'https': 'https://127.0.0.1:1080'}]  # 代理池
+                       'https': 'http://127.0.0.1:1080'}]  # 代理池
 # request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
 #                        'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
 
@@ -98,3 +100,9 @@ enable_random_ua = True  # 使用随机UA(默认True，开启可以覆盖request
 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数
+
+# 网络空间测绘引擎设置
+cam_records_maximum_per_domain = 1000   # 对于单个主域名，在测绘引擎中的最多查询多少条记录，防止泛解析和CDN浪费积分，对 fofa, hunter, quake, zoomeye 生效，最低为100
+
+# 是否从输入的数据中使用tldextract提取主域名。若设为 False，OneForAll会直接将输入的域名作为主域名，比如北京分公司的域名 beijing.10086.com 就不会被解析成母公司的域名 10086.com
+use_tld_extract = True

data/altdns_wordlist.txt

@@ -349,4 +349,638 @@ twitch
 vi
 web1
 westeurope
-z
+z
+previous
+new
+new1
+east
+south
+west
+north
+southeast
+northwest
+es
+nt
+tx
+tencent
+ali
+aliyun
+center
+stable
+release
+ga
+rc
+01
+02
+03
+dc
+mq
+oa
+uc
+zk
+apm
+bbs
+biw
+bot
+bus
+cat
+crm
+dc1
+doc
+ec2
+efk
+elk
+gce
+hub
+job
+jpa
+jwt
+lib
+log
+mbs
+mgt
+mvc
+nms
+pan
+pod
+srv
+sso
+svn
+tms
+wms
+www
+amqp
+apis
+app1
+avro
+blog
+boot
+cicd
+flex
+gocd
+gogs
+good
+grid
+guns
+hdfs
+hive
+ldap
+live
+logs
+mqtt
+nifi
+note
+open
+pipe
+pods
+shop
+show
+solr
+tsdb
+user
+wiki
+work
+zuul
+apiv1
+apiv2
+apiv3
+apiv4
+apiv5
+app01
+batch
+beats
+cacti
+camel
+chaos
+drone
+dubbo
+event
+feign
+flink
+flume
+geode
+gitea
+goods
+graph
+group
+habor
+hbase
+html5
+infra
+kafka
+kylin
+label
+maven
+mesos
+micro
+minio
+nacos
+neo4j
+nerve
+nexus
+node1
+oauth
+oozie
+redis
+route
+scala
+spark
+sqoop
+stack
+store
+storm
+webui
+api-v1
+api-v2
+api-v3
+api-v4
+api-v5
+consul
+falcon
+galaxy
+goblin
+gradle
+group1
+hadoop
+harbor
+influx
+kibana
+lcinga
+logapi
+logger
+nagios
+node01
+oauth2
+office
+pgraph
+ribbon
+router
+scribe
+sleuth
+spark1
+splunk
+stream
+syslog
+sysmon
+tracer
+travis
+triton
+tuning
+web-ui
+weblog
+zabbix
+zipkin
+airflow
+akumuli
+ansible
+bigdata
+breaker
+brogmon
+catalog
+circuit
+content
+datadog
+diagram
+eureka1
+fluentd
+gemfire
+grafana
+hystrix
+invoker
+jupyter
+kinesis
+library
+logging
+manager
+meeting
+monitor
+netdata
+netflix
+nodered
+pushapi
+recruit
+restapi
+restful
+rsyslog
+storage
+tracing
+turbine
+webflow
+activemq
+actuator
+cadvisor
+collectd
+contract
+dataflow
+elkstack
+exporter
+filebeat
+graphite
+heapster
+influxdb
+librenms
+logstash
+marathon
+node-red
+opentsdb
+pipeline
+platform
+push-api
+rabbitmq
+schedule
+searcher
+sentinel
+tracking
+atlassian
+cassandra
+community
+discovery
+dockerhub
+dzzoffice
+elk-stack
+hostgroup
+kubernete
+logsearch
+mapreduce
+office365
+openstack
+pagerduty
+serverset
+terraform
+websocket
+zookeeper
+clickhouse
+cloudwatch
+dashboards
+datacenter
+dispatcher
+hostgroup1
+kube-state
+openfalcon
+prometheus
+servicelog
+skywalking
+usercenter
+api-gateway
+data-center
+eureka-zuul
+kube-status
+loganalysis
+open-falcon
+opentracing
+pushgateway
+recruitment
+restful-api
+service-log
+serviceslog
+system-cube
+user-center
+alertmanager
+apis-gateway
+eureka-admin
+grafana-kong
+nacos-config
+push-gateway
+rest-gateway
+services-log
+zuul-gateway
+consul-config
+elasticsearch
+eureka-client
+eureka-server
+node-exporter
+kong-dashboard
+kylin-dashboard
+restful-gateway
+grafana-dashboard
+kylin-system-cube
+grafana-management
+apollo
+apolloconfig
+apolloadmin
+apolloservice
+ws
+lucene
+beam
+struts
+arrow
+hudi
+cloudstack
+pulsar
+commons
+subversion
+asterixdb
+superset
+mxnet
+httpd
+ignite
+sling
+shardingsphere
+axis
+spamassassin
+apisix
+openoffice
+db
+cordova
+qpid
+ofbiz
+tapestry
+impala
+ambari
+cocoon
+carbondata
+geronimo
+dolphinscheduler
+doris
+tvm
+wicket
+ant
+iceberg
+jackrabbit
+myfaces
+pdfbox
+james
+nuttx
+drill
+incubator
+tomee
+ozone
+trafficserver
+accumulo
+kudu
+druid
+pinot
+phoenix
+harmony
+directory
+perl
+thrift
+pig
+felix
+cxf
+echarts
+karaf
+servicemix
+xmlgraphics
+openwhisk
+iotdb
+tuscany
+couchdb
+jclouds
+rocketmq
+trafficcontrol
+tinkerpop
+hc
+calcite
+jmeter
+netbeans
+uima
+poi
+zeppelin
+tez
+isis
+atlas
+mahout
+mynewt
+tika
+mina
+nutch
+deltaspike
+xalan
+portals
+servicecomb
+ranger
+groovy
+jena
+brooklyn
+whimsical
+knox
+forrest
+avalon
+openmeetings
+xerces
+bookkeeper
+bigtop
+apr
+metron
+samza
+openjpa
+tajo
+lenya
+stratos
+cayenne
+airavata
+weex
+shindig
+shenyu
+velocity
+aries
+lens
+fineract
+gobblin
+parquet
+apex
+seatunnel
+syncope
+jakarta
+royale
+usergrid
+manifoldcf
+inlong
+libcloud
+hawq
+gump
+allura
+aurora
+ibatis
+ratis
+helix
+synapse
+kyuubi
+sentry
+archiva
+daffodil
+reef
+trafodion
+sis
+xml
+devlake
+hama
+openwebbeans
+guacamole
+pivot
+lucenenet
+curator
+gora
+olingo
+ode
+jspwiki
+river
+stanbol
+giraph
+streams
+slider
+dlab
+plc4x
+shiro
+juddi
+eventmesh
+roller
+taverna
+oodt
+climate
+bloodhound
+chemistry
+unomi
+yunikorn
+brpc
+orc
+rave
+beehive
+eagle
+yetus
+deltacloud
+clerezza
+datasketches
+systemml
+continuum
+tamaya
+opennlp
+buildstream
+stdcxx
+rya
+santuario
+buildr
+submarine
+linkis
+freemarker
+juneau
+any23
+ariatosca
+lucy
+marmotta
+heron
+datalab
+singa
+whirr
+celix
+vcl
+streampipes
+chukwa
+ctakes
+crunch
+madlib
+metamodel
+xmlbeans
+quickstep
+edgent
+wookie
+griffin
+hop
+uniffle
+esme
+fluo
+nlpcraft
+abdera
+creadur
+predictionio
+wink
+ace
+pagespeed
+kvrocks
+mnemonic
+tcl
+twill
+gearpump
+tubemq
+oltu
+quetz
+livy
+sdap
+johnzon
+systemds
+polygene
+labs
+pegasus
+streampark
+attic
+age
+pekko
+teaclave
+htrace
+ponymail
+tiles
+hivemall
+bahir
+click
+empire-db
+bval
+hivemind
+myriad
+hugegraph
+devicemap
+excalibur
+tuweni
+sedona
+vxquery
+datafu
+joshua
+commonsrdf
+shale
+directmemory
+mrunit
+tephra
+nemo
+senssoft
+flagon
+pirk
+toree
+omid
+distributedlog
+celeborn
+s2graph
+onami
+serf
+etch
+samoa
+milagro
+corinthia
+wayang
+diversity
+batchee
+ripple
+opendal
+gossip
+annotator
+causeway
+sirona
+spot
+baremaps
+mrql
+depot
+crail
+steve
+petri
+amaterasu
+liminal
+iota
+paimon
+kibble
+horn
+hdt
+drat
+openaz
+marvin
+provisionr
+mesatee
+cmda
+bluemarlin
+concerted
+warble
+tac
+composer
+cotton
+kie
+android
+infratest
+zabbix

docs/changes.md

@@ -8,6 +8,9 @@ OneForAll遵守[语义化版本格式](https://semver.org/)。
 # Unreleased
 
 # Released
+## [0.4.5](https://github.com/shmilylty/oneforall/releases/tag/v0.4.5) - 2022-07-10
+- 修复了#254
+
 ## [0.4.4](https://github.com/shmilylty/oneforall/releases/tag/v0.4.4) - 2022-07-03
 - 修复了多个已知问题
 - 添加了多个查询接口

docs/en-us/README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
 [![python](https://img.shields.io/badge/python-3.6+-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.4-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.5-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll is a powerful subdomain integration tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
 
@@ -229,7 +229,7 @@ At present, OneForAll is under development, there must be a lot of problems and
 1. Use 6 certificate modules: `censys_api`, `certspotter`, `crtsh`, `entrust`, `google`, `spyse_api`.
 2. Use 6 baseline testing modules: scan domain transfer vulnerability `axfr`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`, NSEC record `nsec`. NSEC3 record and other modules will be added later.
 3. Use 2 web crawler modules: `archirawl`, `commoncrawl`, which is still being debugged and needs to be added and improved). 
-4. Use 24 DNS datasets modules: `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`.
+4. Use 24 DNS datasets modules: `bevigil`, `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`.
 5. Use 6 DNS queries modules: enumerating SRV records `srv` and collect from `MX`, `NS`, `SOA`, `TXT`, `SPF`. 
 6. Use 6 threat intelligence modules: `alienvault`, `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved.
 7. Use 16 search engines modules: `ask`, `baidu`, `bing`, `bing_api`, `fofa_api`, `gitee`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`, except for special search engines. General search engines support automatic exclusion of search, full search and recursive search. 

docs/field.md

@@ -60,7 +60,7 @@ HTTP响应的状态码
 
 网络连接情况及详情
 
-###title
+### title
 
 网站标题
 
@@ -69,9 +69,11 @@ HTTP响应的状态码
 网站指纹信息
 
 ### history
+
 请求时URL跳转历史
 
 ### response
+
 响应体文本内容
 
 ### times
@@ -116,4 +118,4 @@ ip2region库查询出的网络服务提供商
 
 ### find
 
-当前模块发现的子域个数
+当前模块发现的子域个数

export.py

@@ -60,6 +60,8 @@ def do_export(fmt, path, rows, show, domain, target):
     if show:
         print(rows.dataset)
     data = rows.export(fmt)
+    if fmt == 'csv':
+        data = '\ufeff' + data
     utils.save_to_file(path, data)
     logger.log('ALERT', f'The subdomain result for {domain}: {path}')
     data = rows.as_dict()

modules/certificates/censys_api.py

@@ -9,7 +9,7 @@ class CensysAPI(Query):
         self.domain = domain
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
-        self.addr = 'https://search.censys.io/api/v1/search/certificates'
+        self.addr = 'https://search.censys.io/api/v2/certificates/search'
         self.id = settings.censys_api_id
         self.secret = settings.censys_api_secret
         self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次
@@ -20,26 +20,30 @@ class CensysAPI(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        data = {
-            'query': f'parsed.names: {self.domain}',
-            'page': 1,
-            'fields': ['parsed.subject_dn', 'parsed.names'],
-            'flatten': True}
-        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
+        params = {
+            'q': f'names: {self.domain}',
+            'per_page': 100,
+        }
+        resp = self.get(self.addr, params=params, auth=(self.id, self.secret))
         if not resp:
             return
         json = resp.json()
         status = json.get('status')
-        if status != 'ok':
+        if status != 'OK':
             logger.log('ALERT', f'{self.source} module {status}')
             return
         subdomains = self.match_subdomains(resp.text)
         self.subdomains.update(subdomains)
-        pages = json.get('metadata').get('pages')
-        for page in range(2, pages + 1):
-            data['page'] = page
-            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
-            self.subdomains = self.collect_subdomains(resp)
+        next_cursor = json.get("result").get("links").get("next")
+        while next_cursor:
+            tmp_params = {
+                'q': f'names: {self.domain}',
+                'per_page': 100,
+                "cursor": next_cursor
+            }
+            tmp_resp = self.get(self.addr, params=tmp_params, auth=(self.id, self.secret))
+            self.subdomains = self.collect_subdomains(tmp_resp)
+            next_cursor = tmp_resp.json().get("result").get("links").get("next")
 
     def run(self):
         """

modules/certificates/crtsh.py

@@ -1,4 +1,6 @@
 from common.query import Query
+import json
+import os
 
 
 class Crtsh(Query):
@@ -15,11 +17,44 @@ class Crtsh(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
+        self.timeout = 120
         params = {'q': f'%.{self.domain}', 'output': 'json'}
         resp = self.get(self.addr, params)
         if not resp:
             return
         text = resp.text.replace(r'\n', ' ')
+        """
+        * > altdns
+        """
+        subDomains = set()
+        try:
+            jsonData = json.loads(text)
+        except Exception as e:
+            pass
+        for i in range(len(jsonData)):
+            try:
+                name_value = str(jsonData[i]['name_value'])
+            except Exception as e:
+                pass
+            if '*' in name_value:
+                try:
+                    if 'certificates' in os.path.dirname(os.path.abspath(__file__)):
+                        dictFile = open("../../data/altdns_wordlist.txt", "r", encoding='utf8')
+                    else:
+                        dictFile = open("./data/altdns_wordlist.txt", "r", encoding='utf8')
+                    for line in dictFile.readlines():
+                        altdns = line.strip()
+                        result = name_value.replace('*', altdns)
+                        if self.domain in result:
+                            subDomains.add(result)
+                except Exception as e:
+                    pass
+        if len(subDomains) > 0:
+            for x in subDomains:
+                text = text + ',' + x + ','
+        """
+        * > altdns end
+        """
         subdomains = self.match_subdomains(text)
         self.subdomains.update(subdomains)
 
@@ -46,4 +81,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('163.com')

modules/certificates/myssl.py

@@ -0,0 +1,46 @@
+from common.query import Query
+
+
+class MySSL(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'MySSLQuery'
+        self.addr = 'https://myssl.com/api/v1/discover_sub_domain'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'domain': self.domain}
+        resp = self.get(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+
+    """
+    query = MySSL(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('freebuf.com')

modules/certificates/racent.py

@@ -0,0 +1,49 @@
+from config import settings
+from common.query import Query
+
+
+class Racent(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'RacentQuery'
+        self.addr = 'https://face.racent.com/tool/query_ctlog'
+        self.api = settings.racent_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'token': self.api, 'keyword': self.domain}
+        resp = self.get(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Racent(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

modules/datasets/bevigil.py

@@ -0,0 +1,50 @@
+from config import settings
+from common.query import Query
+
+
+class BeVigilAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'BeVigilOsintApi'
+        self.addr = 'http://osint.bevigil.com/api/{}/subdomains/'
+        self.api = settings.bevigil_api
+    
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+
+        self.header = self.get_header()
+        self.header.update({"X-Access-Token": self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr.format(self.domain)
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = BeVigilAPI(domain)
+    query.run()
+
+if __name__ == '__main__':
+    run('example.com')

modules/datasets/dnsgrep.py

@@ -0,0 +1,44 @@
+from common.query import Query
+
+
+class Dnsgrep(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'DnsgrepQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = 'https://www.dnsgrep.cn/subdomain/' + self.domain
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Dnsgrep(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

modules/datasets/urlscan.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class Urlscan(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'UrlscanQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = 'https://urlscan.io/api/v1/search/'
+        params = {'q': 'domain:' + self.domain}
+        resp = self.get(url, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Urlscan(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('sangfor.com')

modules/datasets/windvane.py

@@ -0,0 +1,92 @@
+from config import settings
+from common.query import Query
+
+
+class Windvane(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = "WindvaneQuery"
+        self.addr = 'https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain'
+        self.api_key = settings.windvane_api_token
+        self.page_size = 1000
+        
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({
+            'Content-Type': 'application/json',
+            'Referer': 'https://windvane.lichoin.com'
+        })
+        
+        if self.api_key:
+            self.header.update({'X-Api-Key': self.api_key})
+            
+        self.proxy = self.get_proxy(self.source)
+        
+        page = 1
+        total_pages = 1
+        all_subdomains = []
+        
+        while page <= total_pages:
+            data = {
+                "domain": self.domain,
+                "page_request": {
+                    "page": page,
+                    "count": self.page_size
+                }
+            }
+            
+            resp = self.post(self.addr, json=data)
+            if not resp:
+                break
+                
+            try:
+                result = resp.json()
+                
+                if result.get('code') != 0:
+                    break
+                
+                data_section = result.get('data', {})
+
+                subdomains = self.match_subdomains(resp)
+                if not subdomains:
+                    break
+                self.subdomains.update(subdomains)
+                
+                page_info = data_section.get('page_response', {})
+                total_pages = int(page_info.get('total_page', 1))
+                
+                page += 1
+                
+            except:
+                break
+        
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Windvane(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('baidu.com')

modules/dnsquery/mx.py

@@ -7,7 +7,7 @@ class QueryMX(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryMX"
-        self.type = 'MX'  # 利用的DNS记录的MX记录收集子域
+        self.qtype = 'MX'  # 利用的DNS记录的MX记录收集子域
 
     def run(self):
         """

modules/finder.py

@@ -179,6 +179,7 @@ def convert_to_dict(url_list):
         url_dict.append({'url': url})
     return url_dict
 
+
 def find_subdomains(domain, data):
     subdomains = set()
     js_urls = set()

modules/search/fofa_api.py

@@ -23,7 +23,7 @@ class FoFa(Search):
         self.page_num = 1
         subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 1000}
+                     'size': min(1000, settings.cam_records_maximum_per_domain)}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 1000:
+            if size < min(1000, settings.cam_records_maximum_per_domain):
                 break
             self.page_num += 1
 

modules/search/hunter_api.py

@@ -22,7 +22,7 @@ class Hunter(Search):
         self.page_num = 1
         subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)

modules/search/quake_api.py

@@ -0,0 +1,72 @@
+import time
+
+from config import settings
+from common.search import Search
+
+
+class Quake(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Quake'
+        self.source = "QuakeAPISearch"
+        self.addr = 'https://quake.360.net/api/v3/search/quake_service'
+        self.delay = 1
+        self.key = settings.quake_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.per_page_num = 100
+        self.page_num = 0
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.header.update({'Content-Type': 'application/json'})
+            self.header.update({'X-QuakeToken': self.key})
+            self.proxy = self.get_proxy(self.source)
+
+            query = {'query': 'domain:"' + self.domain + '"',
+                     'start': self.page_num * self.per_page_num,
+                     'size': self.per_page_num,
+                     'include': ["service.http.host"]}
+            resp = self.post(self.addr, json=query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains.update(subdomains)
+            total = resp_json.get('meta').get('pagination').get('total')
+            self.page_num += 1
+            if self.page_num * self.per_page_num >= int(total):
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Quake(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('nosugartech.com')

modules/search/so.py

@@ -72,4 +72,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('mi.com')

modules/search/wzsearch.py

@@ -0,0 +1,68 @@
+import time
+from common.search import Search
+
+
+class WzSearch(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'WzSearch'
+        self.addr = 'https://www.wuzhuiso.com/s'
+
+    def search(self, domain, filtered_subdomain=''):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        """
+        self.page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'pn': self.page_num, 'src': 'page_www', 'fr': 'none'}
+            resp = self.get(self.addr, params)
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
+                break
+            self.subdomains.update(subdomains)
+            self.page_num += 1
+            if 'next" href' not in resp.text:
+                print(subdomains)
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain)
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = WzSearch(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

modules/search/zoomeye_api.py

@@ -1,7 +1,6 @@
 import time
 from config import settings
 from common.search import Search
-from config.log import logger
 
 
 class ZoomEyeAPI(Search):
@@ -10,59 +9,47 @@ class ZoomEyeAPI(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'ZoomEyeAPISearch'
-        self.addr = 'https://api.zoomeye.org/web/search'
+        self.addr = 'https://api.zoomeye.org/domain/search'
         self.delay = 2
-        self.user = settings.zoomeye_api_usermail
-        self.pwd = settings.zoomeye_api_password
-
-    def login(self):
-        """
-        登陆获取查询taken
-        """
-        url = 'https://api.zoomeye.org/user/login'
-        data = {'username': self.user, 'password': self.pwd}
-        resp = self.post(url=url, json=data)
-        if not resp:
-            logger.log('ALERT', f'{self.source} module login failed')
-            return None
-        data = resp.json()
-        if resp.status_code == 200:
-            logger.log('DEBUG', f'{self.source} module login success')
-            return data.get('access_token')
-        else:
-            logger.log('ALERT', data.get('message'))
-            return None
+        self.key = settings.zoomeye_api_key
 
     def search(self):
         """
         发送搜索请求并做子域匹配
         """
-        page_num = 1
-        access_token = self.login()
-        if not access_token:
-            return
-        while True:
+        self.per_page_num = 30
+        self.page_num = 1
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
+            self.header.update({'API-KEY': self.key})
             self.proxy = self.get_proxy(self.source)
-            self.header.update({'Authorization': 'JWT ' + access_token})
-            params = {'query': 'hostname:' + self.domain, 'page': page_num}
+
+            params = {'q': self.domain,
+                      'page': self.page_num,
+                      'type': 1}
             resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if resp.status_code == 403:
+                break
+            resp_json = resp.json()
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             self.subdomains.update(subdomains)
-            page_num += 1
-            if page_num > 500:
+            total = resp_json.get('total')
+            self.page_num += 1
+            if self.page_num * self.per_page_num >= int(total):
                 break
-            if resp.status_code == 403:
+            if self.page_num > 400:
                 break
 
     def run(self):
         """
         类执行入口
         """
-        if not self.have_api(self.user, self.pwd):
+        if not self.have_api(self.key):
             return
         self.begin()
         self.search()
@@ -83,4 +70,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('mi.com')
+    run('zhipin.com')

modules/wildcard.py

@@ -124,7 +124,6 @@ def detect_wildcard(domain):
     return is_enable
 
 
-@tenacity.retry(stop=tenacity.stop_after_attempt(2))
 def get_wildcard_record(domain, resolver):
     logger.log('INFOR', f"Query {domain} 's wildcard dns record "
                         f"in authoritative name server")
@@ -166,7 +165,6 @@ def collect_wildcard_record(domain, authoritative_ns):
     resolver.cache = None  # 不使用DNS缓存
     ips = set()
     ttl = int()
-    ttls_check = list()
     ips_stat = dict()
     ips_check = list()
     while True:
@@ -178,21 +176,20 @@ def collect_wildcard_record(domain, authoritative_ns):
             logger.log('DEBUG', e.args)
             logger.log('ALERT', f'Multiple query errors,'
                                 f'try to query a new random subdomain')
-            continue
-        # 每5次查询检查结果列表 如果都没结果则结束查询
+            # 查询出错退出循环
+            break
+        # 每5次连续查询后检查结果列表
         ips_check.append(ip)
-        ttls_check.append(ttl)
+        # 如果出现50个以上的泛解析则结束查询
+        if len(ips) >= 50:
+            break
+        # 如果连续5次查询都没结果则结束查询
         if len(ips_check) == 5:
             if not any(ips_check):
                 logger.log('ALERT', 'The query ends because there are '
                                     'no results for 5 consecutive queries.')
                 break
             ips_check = list()
-        if len(ttls_check) == 5 and len(set(ttls_check)) == 5:
-            logger.log('ALERT', 'The query ends because there are '
-                                '5 different TTL results for 5 consecutive queries.')
-            ips, ttl = set(), int()
-            break
         if ip is None:
             continue
         ips.update(ip)
@@ -206,7 +203,7 @@ def collect_wildcard_record(domain, authoritative_ns):
             if times >= 2:
                 addrs.append(addr)
         # 大部分的IP地址出现次数大于2次停止收集泛解析IP记录
-        if len(addrs) / len(ips) >= 0.8:
+        if len(addrs) / len(ips) >= 0.7:
             break
     logger.log('DEBUG', f'Collected the wildcard dns record of {domain}\n{ips}\n{ttl}')
     return ips, ttl

oneforall.py

@@ -32,7 +32,7 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'
 
-version = 'v0.4.4'
+version = 'v0.4.5'
 message = white + '{' + red + version + ' #dev' + white + '}'
 
 oneforall_banner = f"""
@@ -68,7 +68,7 @@ class OneForAll(object):
 
     Note:
         --port   small/medium/large  See details in ./config/setting.py(default small)
-        --fmt csv/json (result format)
+        --fmt    csv/json (result format)
         --path   Result path (default None, automatically generated)
 
     :param str  target:     One domain (target or targets must be provided)
@@ -98,7 +98,6 @@ class OneForAll(object):
         self.domains = set()  # All domains that are to be collected
         self.data = list()  # The subdomain results of the current domain
         self.datas = list()  # All subdomain results of the domain
-        self.in_china = None
         self.access_internet = False
         self.enable_wildcard = False
 
@@ -166,7 +165,6 @@ class OneForAll(object):
             # may cause other network tasks to be error
             brute = Brute(self.domain, word=True, export=False)
             brute.enable_wildcard = self.enable_wildcard
-            brute.in_china = self.in_china
             brute.quite = True
             brute.run()
 
@@ -237,7 +235,7 @@ class OneForAll(object):
         logger.log('DEBUG', 'Python ' + utils.python_version())
         logger.log('DEBUG', 'OneForAll ' + version)
         utils.check_dep()
-        self.access_internet, self.in_china = utils.get_net_env()
+        self.access_internet = utils.get_net_env()
         if self.access_internet and settings.enable_check_version:
             utils.check_version(version)
         logger.log('INFOR', 'Start running OneForAll')

requirements.txt

@@ -4,7 +4,7 @@ certifi==2022.06.15
 chardet==5.0.0
 colorama==0.4.4
 dnspython==2.2.1
-exrex==0.10.5
+exrex==0.11.0
 fire==0.4.0
 future==0.18.2
 idna==3.3
@@ -20,3 +20,4 @@ tqdm==4.64.0
 treelib==1.6.1
 urllib3==1.26.9
 win32-setctime==1.1.0
+setuptools