mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-25 20:37:48 +08:00
Merge branch 'dev' of https://github.com/shmilylty/OneForAll into dev
This commit is contained in:
JrD
Generated
+60
-49
@@ -18,11 +18,11 @@
|
||||
"default": {
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
||||
"sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8",
|
||||
"sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"
|
||||
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
|
||||
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
|
||||
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
|
||||
],
|
||||
"version": "==4.9.1"
|
||||
"version": "==4.9.3"
|
||||
},
|
||||
"bs4": {
|
||||
"hashes": [
|
||||
@@ -47,11 +47,11 @@
|
||||
},
|
||||
"colorama": {
|
||||
"hashes": [
|
||||
"sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
|
||||
"sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
|
||||
"sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
|
||||
"sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
|
||||
],
|
||||
"markers": "sys_platform == 'win32'",
|
||||
"version": "==0.4.3"
|
||||
"version": "==0.4.4"
|
||||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
@@ -79,6 +79,7 @@
|
||||
"hashes": [
|
||||
"sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==0.18.2"
|
||||
},
|
||||
"idna": {
|
||||
@@ -86,6 +87,7 @@
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==2.10"
|
||||
},
|
||||
"loguru": {
|
||||
@@ -118,6 +120,7 @@
|
||||
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==1.15.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
@@ -125,45 +128,52 @@
|
||||
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||
],
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==2.0.1"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
|
||||
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
|
||||
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
|
||||
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
|
||||
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
|
||||
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
|
||||
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
|
||||
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
|
||||
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
|
||||
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
|
||||
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
|
||||
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
|
||||
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
|
||||
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
|
||||
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
|
||||
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
|
||||
"sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6",
|
||||
"sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc",
|
||||
"sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a",
|
||||
"sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d",
|
||||
"sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f",
|
||||
"sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da",
|
||||
"sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb",
|
||||
"sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b",
|
||||
"sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86",
|
||||
"sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a",
|
||||
"sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403",
|
||||
"sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327",
|
||||
"sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2",
|
||||
"sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c",
|
||||
"sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b",
|
||||
"sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d",
|
||||
"sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec",
|
||||
"sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376",
|
||||
"sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629",
|
||||
"sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6",
|
||||
"sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162",
|
||||
"sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980",
|
||||
"sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb",
|
||||
"sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e",
|
||||
"sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b",
|
||||
"sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61",
|
||||
"sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079",
|
||||
"sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf",
|
||||
"sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1",
|
||||
"sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471",
|
||||
"sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77",
|
||||
"sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1",
|
||||
"sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1",
|
||||
"sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710",
|
||||
"sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5",
|
||||
"sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465",
|
||||
"sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142",
|
||||
"sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.3.19"
|
||||
"version": "==1.3.20"
|
||||
},
|
||||
"tenacity": {
|
||||
"hashes": [
|
||||
@@ -181,11 +191,11 @@
|
||||
},
|
||||
"tqdm": {
|
||||
"hashes": [
|
||||
"sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5",
|
||||
"sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b"
|
||||
"sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad",
|
||||
"sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.49.0"
|
||||
"version": "==4.51.0"
|
||||
},
|
||||
"treelib": {
|
||||
"hashes": [
|
||||
@@ -196,18 +206,19 @@
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
|
||||
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
|
||||
"sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
|
||||
"sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
|
||||
],
|
||||
"version": "==1.25.10"
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
|
||||
"version": "==1.25.11"
|
||||
},
|
||||
"win32-setctime": {
|
||||
"hashes": [
|
||||
"sha256:02b4c5959ca0b195f45c98115826c6e8a630b7cf648e724feaab1a5aa6250640",
|
||||
"sha256:47aa7c43548c1fc0a4f026d1944b748b37036df116c7c4cf908e82638d854313"
|
||||
"sha256:4e88556c32fdf47f64165a2180ba4552f8bb32c1103a2fafd05723a0bd42bd4b",
|
||||
"sha256:dc925662de0a6eb987f0b01f599c01a8236cb8c62831c22d9cada09ad958243e"
|
||||
],
|
||||
"markers": "sys_platform == 'win32'",
|
||||
"version": "==1.0.2"
|
||||
"version": "==1.0.3"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
||||
@@ -97,8 +97,6 @@ docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/conf
|
||||
<details>
|
||||
<summary><b>✨使用演示</b></summary>
|
||||
|
||||
如果你的主机不在中国，请把 [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) 中`brute_nameservers_path`选项的`cn_nameservers.txt`修改为`nameservers.txt`。
|
||||
|
||||
如果你是通过pip3安装的依赖则使用以下命令运行示例：
|
||||
```bash
|
||||
python3 oneforall.py --target example.com run
|
||||
@@ -170,7 +168,7 @@ DESCRIPTION
|
||||
python3 oneforall.py --target example.com --valid None run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --port small run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -179,7 +177,7 @@ DESCRIPTION
|
||||
Note:
|
||||
参数alive可选值True，False分别表示导出存活，全部子域结果
|
||||
参数port可选值有'default', 'small', 'large', 详见config.py配置
|
||||
参数format可选格式有 'csv','json'
|
||||
参数fmt可选格式有 'csv','json'
|
||||
参数path默认None使用OneForAll结果目录生成路径
|
||||
|
||||
ARGUMENTS
|
||||
@@ -199,7 +197,7 @@ FLAGS
|
||||
请求验证子域的端口范围(默认只探测80端口)
|
||||
--valid=VALID
|
||||
只导出存活的子域结果(默认False)
|
||||
--format=FORMAT
|
||||
--fmt=FMT
|
||||
结果保存格式(默认csv)
|
||||
--path=PATH
|
||||
结果保存路径(默认None)
|
||||
@@ -269,7 +267,6 @@ FLAGS
|
||||
## ⌛后续计划
|
||||
|
||||
- [ ] 各模块持续优化和完善
|
||||
- [x] 子域监控（标记每次新发现的子域）
|
||||
- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
|
||||
- [ ] 操作强大交互人性的前端界面实现
|
||||
|
||||
|
||||
@@ -349,18 +349,6 @@ def collect_wildcard_record(domain, authoritative_ns):
|
||||
return ips, ttl
|
||||
|
||||
|
||||
def get_nameservers_path(enable_wildcard, ns_ip_list):
|
||||
path = settings.brute_nameservers_path
|
||||
if not enable_wildcard:
|
||||
return path
|
||||
if not ns_ip_list:
|
||||
return path
|
||||
path = settings.authoritative_dns_path
|
||||
ns_data = '\n'.join(ns_ip_list)
|
||||
utils.save_data(path, ns_data)
|
||||
return path
|
||||
|
||||
|
||||
def check_dict():
|
||||
if not settings.enable_check_dict:
|
||||
return
|
||||
@@ -384,7 +372,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
|
||||
info = dict()
|
||||
cnames = list()
|
||||
ips = list()
|
||||
public = list()
|
||||
times = list()
|
||||
ttls = list()
|
||||
is_valid_flags = list()
|
||||
@@ -401,7 +388,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
|
||||
cnames.append(cname) # 去除最右边的`.`点号
|
||||
ip = answer.get('data')
|
||||
ips.append(ip)
|
||||
public.append(utils.ip_is_public(ip))
|
||||
num = ip_times.get(ip)
|
||||
times.append(num)
|
||||
isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname)
|
||||
@@ -416,7 +402,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
|
||||
info['ttl'] = ttls
|
||||
info['cname'] = cnames
|
||||
info['ip'] = ips
|
||||
info['public'] = public
|
||||
info['times'] = times
|
||||
info['resolver'] = resolver
|
||||
infos[qname] = info
|
||||
@@ -559,7 +544,7 @@ class Brute(Module):
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||
|
||||
Note:
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result fmt)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
|
||||
@@ -578,13 +563,13 @@ class Brute(Module):
|
||||
:param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode)
|
||||
:param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py)
|
||||
:param bool export: Export the results (default True)
|
||||
:param str format: Result format (default csv)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
"""
|
||||
def __init__(self, target=None, targets=None, process=None, concurrent=None,
|
||||
word=False, wordlist=None, recursive=False, depth=None, nextlist=None,
|
||||
fuzz=False, place=None, rule=None, fuzzlist=None, export=True,
|
||||
alive=True, format='csv', path=None):
|
||||
alive=True, fmt='csv', path=None):
|
||||
Module.__init__(self)
|
||||
self.module = 'Brute'
|
||||
self.source = 'Brute'
|
||||
@@ -603,15 +588,15 @@ class Brute(Module):
|
||||
self.fuzzlist = fuzzlist or settings.fuzz_list
|
||||
self.export = export
|
||||
self.alive = alive
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.path = path
|
||||
self.bulk = False # 是否是批量爆破场景
|
||||
self.domains = list() # 待爆破的所有域名集合
|
||||
self.domain = str() # 当前正在进行爆破的域名
|
||||
self.ips_times = dict() # IP集合出现次数
|
||||
self.enable_wildcard = False # 当前域名是否使用泛解析
|
||||
self.check_env = True
|
||||
self.quite = False
|
||||
self.in_china = None
|
||||
|
||||
def gen_brute_dict(self, domain):
|
||||
logger.log('INFOR', f'Generating dictionary for {domain}')
|
||||
@@ -668,6 +653,13 @@ class Brute(Module):
|
||||
logger.log('FATAL', f'Incorrect domain for fuzz')
|
||||
exit(1)
|
||||
|
||||
def init_dict_path(self):
|
||||
data_dir = settings.data_storage_dir
|
||||
if self.wordlist is None:
|
||||
self.wordlist = settings.brute_wordlist_path or data_dir.joinpath('subnames.txt')
|
||||
if self.recursive_nextlist is None:
|
||||
self.recursive_nextlist = settings.recursive_nextlist_path or data_dir.joinpath('subnames_next.txt')
|
||||
|
||||
def main(self, domain):
|
||||
start = time.time()
|
||||
logger.log('INFOR', f'Blasting {domain} ')
|
||||
@@ -687,7 +679,7 @@ class Brute(Module):
|
||||
if self.enable_wildcard:
|
||||
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
|
||||
ns_ip_list)
|
||||
ns_path = get_nameservers_path(self.enable_wildcard, ns_ip_list)
|
||||
ns_path = utils.get_ns_path(self.in_china, self.enable_wildcard, ns_ip_list)
|
||||
|
||||
dict_set = self.gen_brute_dict(domain)
|
||||
|
||||
@@ -730,12 +722,13 @@ class Brute(Module):
|
||||
|
||||
def run(self):
|
||||
logger.log('INFOR', f'Start running {self.source} module')
|
||||
if self.check_env:
|
||||
utils.check_env()
|
||||
if self.in_china is None:
|
||||
_, self.in_china = utils.get_net_env()
|
||||
self.domains = utils.get_domains(self.target, self.targets)
|
||||
for self.domain in self.domains:
|
||||
self.results = list() # 置空
|
||||
all_subdomains = list()
|
||||
self.init_dict_path()
|
||||
self.check_brute_params()
|
||||
if self.recursive_brute:
|
||||
logger.log('INFOR', f'Start recursively brute the 1 layer subdomain'
|
||||
@@ -761,7 +754,7 @@ class Brute(Module):
|
||||
|
||||
logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}')
|
||||
if not self.path:
|
||||
name = f'{self.domain}_brute_result.{self.format}'
|
||||
name = f'{self.domain}_brute_result.{self.fmt}'
|
||||
self.path = settings.result_save_dir.joinpath(name)
|
||||
# 数据库导出
|
||||
if self.export:
|
||||
@@ -770,7 +763,7 @@ class Brute(Module):
|
||||
alive=self.alive,
|
||||
limit='resolve',
|
||||
path=self.path,
|
||||
format=self.format)
|
||||
fmt=self.fmt)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
+48
-22
@@ -57,7 +57,6 @@ class Database(object):
|
||||
f'alive int,'
|
||||
f'request int,'
|
||||
f'resolve int,'
|
||||
f'new int,'
|
||||
f'url text,'
|
||||
f'subdomain text,'
|
||||
f'port int,'
|
||||
@@ -86,6 +85,20 @@ class Database(object):
|
||||
f'elapse float,'
|
||||
f'find int)')
|
||||
|
||||
def insert_table(self, table_name, result):
|
||||
table_name = table_name.replace('.', '_')
|
||||
self.conn.query(
|
||||
f'insert into "{table_name}" '
|
||||
f'(id, alive, resolve, request, url, subdomain, port, level,'
|
||||
f'cname, ip, public, cdn, status, reason, title, banner, header,'
|
||||
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
|
||||
f'module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :url,'
|
||||
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
|
||||
f':status, :reason, :title, :banner, :header, :history, :response,'
|
||||
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
|
||||
f':source, :elapse, :find)', **result)
|
||||
|
||||
def save_db(self, table_name, results, module_name=None):
|
||||
"""
|
||||
Save the results of each module in the database
|
||||
@@ -101,11 +114,11 @@ class Database(object):
|
||||
try:
|
||||
self.conn.bulk_query(
|
||||
f'insert into "{table_name}" '
|
||||
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
|
||||
f'(id, alive, resolve, request, url, subdomain, port, level,'
|
||||
f'cname, ip, public, cdn, status, reason, title, banner, header,'
|
||||
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
|
||||
f'module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :new, :url,'
|
||||
f'values (:id, :alive, :resolve, :request, :url,'
|
||||
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
|
||||
f':status, :reason, :title, :banner, :header, :history, :response,'
|
||||
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
|
||||
@@ -178,7 +191,7 @@ class Database(object):
|
||||
|
||||
def deduplicate_subdomain(self, table_name):
|
||||
"""
|
||||
Deduplicates of subdomains in the table
|
||||
Deduplicate subdomains in the table
|
||||
|
||||
:param str table_name: table name
|
||||
"""
|
||||
@@ -199,17 +212,6 @@ class Database(object):
|
||||
self.query(f'delete from "{table_name}" where '
|
||||
f'subdomain is null or resolve == 0')
|
||||
|
||||
def deal_table(self, deal_table_name, backup_table_name):
|
||||
"""
|
||||
Process the table when the collection task is complete
|
||||
|
||||
:param str deal_table_name: Pending table name
|
||||
:param str backup_table_name: Table name for backup
|
||||
"""
|
||||
self.copy_table(deal_table_name, backup_table_name)
|
||||
self.remove_invalid(deal_table_name)
|
||||
self.deduplicate_subdomain(deal_table_name)
|
||||
|
||||
def get_data(self, table_name):
|
||||
"""
|
||||
Get all the data in the table
|
||||
@@ -218,7 +220,7 @@ class Database(object):
|
||||
"""
|
||||
table_name = table_name.replace('.', '_')
|
||||
logger.log('TRACE', f'Get all the data from {table_name} table')
|
||||
return self.query(f'select * from "{table_name}"')
|
||||
return self.query(f'select * from {table_name}')
|
||||
|
||||
def export_data(self, table_name, alive, limit):
|
||||
"""
|
||||
@@ -229,18 +231,42 @@ class Database(object):
|
||||
:param str limit: limit value
|
||||
"""
|
||||
table_name = table_name.replace('.', '_')
|
||||
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
|
||||
f'cname, ip, public, cdn, port, status, reason, title, banner,' \
|
||||
f'cidr, asn, org, addr, isp, source from "{table_name}"'
|
||||
sql = f'select id, alive, request, resolve, url, subdomain, level,' \
|
||||
f'cname, ip, public, cdn, port, status, reason, title, banner,' \
|
||||
f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain'
|
||||
if alive and limit:
|
||||
if limit in ['resolve', 'request']:
|
||||
where = f' where {limit} = 1'
|
||||
query += where
|
||||
sql += where
|
||||
elif alive:
|
||||
where = f' where alive = 1'
|
||||
query += where
|
||||
sql += where
|
||||
logger.log('TRACE', f'Get the data from {table_name} table')
|
||||
return self.query(query)
|
||||
return self.query(sql)
|
||||
|
||||
def count_alive(self, table_name):
|
||||
table_name = table_name.replace('.', '_')
|
||||
sql = f'select count() from {table_name} where alive = 1'
|
||||
return self.query(sql)
|
||||
|
||||
def get_resp_by_url(self, table_name, url):
|
||||
table_name = table_name.replace('.', '_')
|
||||
sql = f'select response from {table_name} where url = "{url}"'
|
||||
logger.log('TRACE', f'Get response data from {url}')
|
||||
return self.query(sql).scalar()
|
||||
|
||||
def get_data_by_fields(self, table_name, fields):
|
||||
table_name = table_name.replace('.', '_')
|
||||
field_str = ', '.join(fields)
|
||||
sql = f"select {field_str} from {table_name}"
|
||||
logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
|
||||
return self.query(sql)
|
||||
|
||||
def update_data_by_url(self, table_name, info, url):
|
||||
table_name = table_name.replace('.', '_')
|
||||
field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
|
||||
sql = f'update {table_name} set {field_str} where url = "{url}"'
|
||||
return self.query(sql)
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
|
||||
+1
-5
@@ -275,7 +275,6 @@ class Module(object):
|
||||
'alive': None,
|
||||
'request': None,
|
||||
'resolve': None,
|
||||
'new': None,
|
||||
'url': None,
|
||||
'subdomain': None,
|
||||
'port': None,
|
||||
@@ -315,25 +314,22 @@ class Module(object):
|
||||
ip = info.get('ip')
|
||||
times = info.get('times')
|
||||
ttl = info.get('ttl')
|
||||
public = info.get('public')
|
||||
if isinstance(cname, list):
|
||||
cname = ','.join(cname)
|
||||
ip = ','.join(ip)
|
||||
times = ','.join([str(num) for num in times])
|
||||
ttl = ','.join([str(num) for num in ttl])
|
||||
public = ','.join([str(num) for num in public])
|
||||
result = {'id': None,
|
||||
'alive': info.get('alive'),
|
||||
'request': info.get('request'),
|
||||
'resolve': info.get('resolve'),
|
||||
'new': None,
|
||||
'url': url,
|
||||
'subdomain': subdomain,
|
||||
'port': 80,
|
||||
'level': level,
|
||||
'cname': cname,
|
||||
'ip': ip,
|
||||
'public': public,
|
||||
'public': info.get('public'),
|
||||
'cdn': info.get('cdn'),
|
||||
'status': None,
|
||||
'reason': info.get('reason'),
|
||||
|
||||
+57
-54
@@ -1,7 +1,6 @@
|
||||
import json
|
||||
from threading import Thread
|
||||
from queue import Queue
|
||||
from operator import attrgetter
|
||||
|
||||
import tqdm
|
||||
import requests
|
||||
@@ -9,6 +8,7 @@ from bs4 import BeautifulSoup
|
||||
|
||||
from common import utils
|
||||
from config.log import logger
|
||||
from common.database import Database
|
||||
from config import settings
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ def get_progress_bar(total):
|
||||
return bar
|
||||
|
||||
|
||||
def get(url, resp_list, session):
|
||||
def get_resp(url, session):
|
||||
timeout = settings.request_timeout_second
|
||||
redirect = settings.request_allow_redirect
|
||||
proxy = utils.get_proxy()
|
||||
@@ -136,13 +136,14 @@ def get(url, resp_list, session):
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
resp = e
|
||||
resp_list.append((url, resp))
|
||||
return resp
|
||||
|
||||
|
||||
def request(urls_queue, resp_list, session):
|
||||
def request(urls_queue, resp_queue, session):
|
||||
while not urls_queue.empty():
|
||||
url = urls_queue.get()
|
||||
get(url, resp_list, session)
|
||||
index, url = urls_queue.get()
|
||||
resp = get_resp(url, session)
|
||||
resp_queue.put((index, resp))
|
||||
urls_queue.task_done()
|
||||
|
||||
|
||||
@@ -168,31 +169,6 @@ def get_session():
|
||||
return session
|
||||
|
||||
|
||||
def bulk_request(urls):
|
||||
logger.log('INFOR', 'Requesting urls in bulk')
|
||||
resp_list = list()
|
||||
urls_queue = Queue()
|
||||
for url in urls:
|
||||
urls_queue.put(url)
|
||||
total = len(urls)
|
||||
session = get_session()
|
||||
thread_count = req_thread_count()
|
||||
bar = get_progress_bar(total)
|
||||
|
||||
progress_thread = Thread(target=progress, name='ProgressThread',
|
||||
args=(bar, total, urls_queue), daemon=True)
|
||||
progress_thread.start()
|
||||
|
||||
for i in range(thread_count):
|
||||
request_thread = Thread(target=request, name=f'RequestThread-{i}',
|
||||
args=(urls_queue, resp_list, session), daemon=True)
|
||||
request_thread.start()
|
||||
|
||||
urls_queue.join()
|
||||
|
||||
return resp_list
|
||||
|
||||
|
||||
def gen_new_info(info, resp):
|
||||
if isinstance(resp, Exception):
|
||||
info['reason'] = str(resp.args)
|
||||
@@ -220,13 +196,54 @@ def gen_new_info(info, resp):
|
||||
return info
|
||||
|
||||
|
||||
def gen_new_data(data, resp_list):
|
||||
new_data = list()
|
||||
for url, resp in resp_list:
|
||||
for info in data:
|
||||
if info.get('url') == url:
|
||||
new_data.append(gen_new_info(info, resp))
|
||||
return new_data
|
||||
def save(name, total, req_data, resp_queue):
|
||||
db = Database()
|
||||
db.create_table(name)
|
||||
i = 0
|
||||
while True:
|
||||
if not resp_queue.empty():
|
||||
i += 1
|
||||
index, resp = resp_queue.get()
|
||||
old_info = req_data[index]
|
||||
new_info = gen_new_info(old_info, resp)
|
||||
db.insert_table(name, new_info)
|
||||
resp_queue.task_done()
|
||||
if i >= total:
|
||||
break
|
||||
db.close()
|
||||
|
||||
|
||||
def bulk_request(domain, req_data, ret=False):
|
||||
logger.log('INFOR', 'Requesting urls in bulk')
|
||||
resp_queue = Queue()
|
||||
urls_queue = Queue()
|
||||
total = len(req_data)
|
||||
for index, info in enumerate(req_data):
|
||||
try:
|
||||
url = info.get('url')
|
||||
except Exception:
|
||||
pass
|
||||
urls_queue.put((index, url))
|
||||
session = get_session()
|
||||
thread_count = req_thread_count()
|
||||
bar = get_progress_bar(total)
|
||||
|
||||
progress_thread = Thread(target=progress, name='ProgressThread',
|
||||
args=(bar, total, urls_queue), daemon=True)
|
||||
progress_thread.start()
|
||||
|
||||
for i in range(thread_count):
|
||||
request_thread = Thread(target=request, name=f'RequestThread-{i}',
|
||||
args=(urls_queue, resp_queue, session), daemon=True)
|
||||
request_thread.start()
|
||||
if ret:
|
||||
urls_queue.join()
|
||||
return resp_queue
|
||||
save_thread = Thread(target=save, name=f'SaveThread',
|
||||
args=(domain, total, req_data, resp_queue), daemon=True)
|
||||
save_thread.start()
|
||||
urls_queue.join()
|
||||
save_thread.join()
|
||||
|
||||
|
||||
def run_request(domain, data, port):
|
||||
@@ -242,20 +259,6 @@ def run_request(domain, data, port):
|
||||
data = utils.set_id_none(data)
|
||||
ports = get_port_seq(port)
|
||||
req_data, req_urls = gen_req_data(data, ports)
|
||||
resp_list = bulk_request(req_urls)
|
||||
new_data = gen_new_data(req_data, resp_list)
|
||||
count = utils.count_alive(new_data)
|
||||
bulk_request(domain, req_data)
|
||||
count = utils.count_alive(domain)
|
||||
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
|
||||
sorted_data = utils.sort_by_subdomain(new_data)
|
||||
return sorted_data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
"""
|
||||
Save request results to database
|
||||
|
||||
:param str name: table name
|
||||
:param list data: data to be saved
|
||||
"""
|
||||
logger.log('INFOR', f'Saving requested results')
|
||||
utils.save_db(name, data, 'request')
|
||||
|
||||
+1
-28
@@ -4,12 +4,6 @@ import json
|
||||
from config.log import logger
|
||||
from config import settings
|
||||
from common import utils
|
||||
from common.ipasn import IPAsnInfo
|
||||
from common.ipreg import IpRegData
|
||||
|
||||
|
||||
ip_asn = IPAsnInfo()
|
||||
ip_reg = IpRegData()
|
||||
|
||||
|
||||
def filter_subdomain(data):
|
||||
@@ -79,13 +73,7 @@ def gen_infos(data, qname, info, infos):
|
||||
flag = False
|
||||
cname = list()
|
||||
ips = list()
|
||||
public = list()
|
||||
ttl = list()
|
||||
cidr = list()
|
||||
asn = list()
|
||||
org = list()
|
||||
addr = list()
|
||||
isp = list()
|
||||
answers = data.get('answers')
|
||||
for answer in answers:
|
||||
if answer.get('type') == 'A':
|
||||
@@ -94,25 +82,11 @@ def gen_infos(data, qname, info, infos):
|
||||
ip = answer.get('data')
|
||||
ips.append(ip)
|
||||
ttl.append(str(answer.get('ttl')))
|
||||
public.append(str(utils.ip_is_public(ip)))
|
||||
asn_info = ip_asn.find(ip)
|
||||
cidr.append(asn_info.get('cidr'))
|
||||
asn.append(asn_info.get('asn'))
|
||||
org.append(asn_info.get('org'))
|
||||
ip_info = ip_reg.query(ip)
|
||||
addr.append(ip_info.get('addr'))
|
||||
isp.append(ip_info.get('isp'))
|
||||
info['resolve'] = 1
|
||||
info['reason'] = 'OK'
|
||||
info['cname'] = ','.join(cname)
|
||||
info['ip'] = ','.join(ips)
|
||||
info['public'] = ','.join(public)
|
||||
info['ttl'] = ','.join(ttl)
|
||||
info['cidr'] = ','.join(cidr)
|
||||
info['asn'] = ','.join(asn)
|
||||
info['org'] = ','.join(org)
|
||||
info['addr'] = ','.join(addr)
|
||||
info['isp'] = ','.join(isp)
|
||||
infos[qname] = info
|
||||
if not flag:
|
||||
info['alive'] = 0
|
||||
@@ -181,8 +155,7 @@ def run_resolve(domain, data):
|
||||
output_name = f'resolved_result_{domain}_{timestring}.json'
|
||||
output_path = temp_dir.joinpath(output_name)
|
||||
log_path = result_dir.joinpath('massdns.log')
|
||||
|
||||
ns_path = settings.brute_nameservers_path
|
||||
ns_path = utils.get_ns_path()
|
||||
|
||||
logger.log('INFOR', f'Running massdns to resolve subdomains')
|
||||
utils.call_massdns(massdns_path, save_path, ns_path,
|
||||
|
||||
+80
-115
@@ -176,16 +176,16 @@ def check_dir(dir_path):
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def check_path(path, name, format):
|
||||
def check_path(path, name, fmt):
|
||||
"""
|
||||
检查结果输出目录路径
|
||||
|
||||
:param path: 保存路径
|
||||
:param name: 导出名字
|
||||
:param format: 保存格式
|
||||
:param fmt: 保存格式
|
||||
:return: 保存路径
|
||||
"""
|
||||
filename = f'{name}.{format}'
|
||||
filename = f'{name}.{fmt}'
|
||||
default_path = settings.result_save_dir.joinpath(filename)
|
||||
if isinstance(path, str):
|
||||
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
|
||||
@@ -204,19 +204,18 @@ def check_path(path, name, format):
|
||||
return path
|
||||
|
||||
|
||||
def check_format(format, count):
|
||||
def check_format(fmt):
|
||||
"""
|
||||
检查导出格式
|
||||
|
||||
:param format: 传入的导出格式
|
||||
:param count: 数量
|
||||
:param fmt: 传入的导出格式
|
||||
:return: 导出格式
|
||||
"""
|
||||
formats = ['csv', 'json', ]
|
||||
if format in formats:
|
||||
return format
|
||||
if fmt in formats:
|
||||
return fmt
|
||||
else:
|
||||
logger.log('ALERT', f'Does not support {format} format')
|
||||
logger.log('ALERT', f'Does not support {fmt} format')
|
||||
logger.log('ALERT', 'So use csv format by default')
|
||||
return 'csv'
|
||||
|
||||
@@ -333,8 +332,8 @@ def remove_invalid_string(string):
|
||||
return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
|
||||
|
||||
|
||||
def export_all_results(path, name, format, datas):
|
||||
path = check_path(path, name, format)
|
||||
def export_all_results(path, name, fmt, datas):
|
||||
path = check_path(path, name, fmt)
|
||||
logger.log('ALERT', f'The subdomain result for all main domains: {path}')
|
||||
row_list = list()
|
||||
for row in datas:
|
||||
@@ -346,7 +345,7 @@ def export_all_results(path, name, format, datas):
|
||||
values = row.values()
|
||||
row_list.append(Record(keys, values))
|
||||
rows = RecordCollection(iter(row_list))
|
||||
content = rows.export(format)
|
||||
content = rows.export(fmt)
|
||||
save_data(path, content)
|
||||
|
||||
|
||||
@@ -366,19 +365,19 @@ def export_all_subdomains(alive, path, name, datas):
|
||||
save_data(path, data)
|
||||
|
||||
|
||||
def export_all(alive, format, path, datas):
|
||||
def export_all(alive, fmt, path, datas):
|
||||
"""
|
||||
将所有结果数据导出
|
||||
|
||||
:param bool alive: 只导出存活子域结果
|
||||
:param str format: 导出文件格式
|
||||
:param str fmt: 导出文件格式
|
||||
:param str path: 导出文件路径
|
||||
:param list datas: 待导出的结果数据
|
||||
"""
|
||||
format = check_format(format, len(datas))
|
||||
fmt = check_format(fmt)
|
||||
timestamp = get_timestring()
|
||||
name = f'all_subdomain_result_{timestamp}'
|
||||
export_all_results(path, name, format, datas)
|
||||
export_all_results(path, name, fmt, datas)
|
||||
export_all_subdomains(alive, path, name, datas)
|
||||
|
||||
|
||||
@@ -430,10 +429,18 @@ def python_version():
|
||||
return sys.version
|
||||
|
||||
|
||||
def count_alive(data):
|
||||
def calc_alive(data):
|
||||
return len(list(filter(lambda item: item.get('alive') == 1, data)))
|
||||
|
||||
|
||||
def count_alive(name):
|
||||
db = Database()
|
||||
result = db.count_alive(name)
|
||||
count = result.scalar()
|
||||
db.close()
|
||||
return count
|
||||
|
||||
|
||||
def get_subdomains(data):
|
||||
return set(map(lambda item: item.get('subdomain'), data))
|
||||
|
||||
@@ -511,15 +518,16 @@ def delete_file(*paths):
|
||||
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(3))
|
||||
def check_net():
|
||||
logger.log('INFOR', 'Checking Internet environment')
|
||||
urls = ['http://www.baidu.com', 'http://www.bing.com',
|
||||
'http://www.apple.com', 'http://www.microsoft.com']
|
||||
urls = ['http://ipinfo.io/json', 'http://ipconfig.io/json']
|
||||
url = random.choice(urls)
|
||||
logger.log('INFOR', f'Trying to access {url}')
|
||||
header = {'User_Agent': 'curl'}
|
||||
timeout = settings.request_timeout_second
|
||||
verify = settings.request_ssl_verify
|
||||
logger.log('DEBUG', f'Trying to access {url}')
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
try:
|
||||
rsp = session.get(url, proxies=get_proxy())
|
||||
rsp = session.get(url, headers=header, timeout=timeout, verify=verify)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
logger.log('ALERT', 'Can not access Internet, retrying')
|
||||
@@ -529,10 +537,16 @@ def check_net():
|
||||
f'{rsp.status_code} {rsp.reason}')
|
||||
logger.log('ALERT', 'Can not access Internet normally, retrying')
|
||||
raise tenacity.TryAgain
|
||||
logger.log('INFOR', 'Access to Internet OK')
|
||||
logger.log('DEBUG', 'Access to Internet OK')
|
||||
country = rsp.json().get('country').lower()
|
||||
if country in ['cn', 'china']:
|
||||
logger.log('DEBUG', f'The host in china')
|
||||
return True, True
|
||||
else:
|
||||
return True, False
|
||||
|
||||
|
||||
def check_pre():
|
||||
def check_dep():
|
||||
logger.log('INFOR', 'Checking dependent environment')
|
||||
implementation = platform.python_implementation()
|
||||
version = platform.python_version()
|
||||
@@ -544,15 +558,15 @@ def check_pre():
|
||||
exit(1)
|
||||
|
||||
|
||||
def check_env():
|
||||
logger.log('INFOR', 'Checking the environment')
|
||||
def get_net_env():
|
||||
logger.log('INFOR', 'Checking network environment')
|
||||
try:
|
||||
check_net()
|
||||
result = check_net()
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('FATAL', 'Can not access Internet')
|
||||
exit(1)
|
||||
check_pre()
|
||||
logger.log('ALERT', 'Can not access Internet')
|
||||
return False, None
|
||||
return result
|
||||
|
||||
|
||||
def check_version(local):
|
||||
@@ -570,7 +584,7 @@ def check_version(local):
|
||||
resp_json = resp.json()
|
||||
latest = resp_json['tag_name']
|
||||
except Exception as e:
|
||||
logger.log('ERROR', 'An error occurred while checking the latest version')
|
||||
logger.log('ALERT', 'An error occurred while checking the latest version')
|
||||
logger.log('DEBUG', e.args)
|
||||
return
|
||||
if latest > local:
|
||||
@@ -736,90 +750,6 @@ def sort_by_subdomain(data):
|
||||
return sorted(data, key=lambda item: item.get('subdomain'))
|
||||
|
||||
|
||||
def ping(host, path):
|
||||
param = '-n' if platform.system().lower() == 'windows' else '-c'
|
||||
command = ['ping', param, '5', host]
|
||||
with open(path, "w") as f:
|
||||
return subprocess.call(command, stdout=f, stderr=f)
|
||||
|
||||
|
||||
def ping_avg_time(nameserver):
|
||||
check_dir(settings.temp_save_dir)
|
||||
temp_path = settings.temp_save_dir.joinpath('ping')
|
||||
ping(nameserver, path=temp_path)
|
||||
with open(temp_path, 'r') as f:
|
||||
text = f.read()
|
||||
if '100.0% packet loss' in text or '100% packet loss' in text or '100% 丢失' in text:
|
||||
logger.log('ALERT', f'100.0% packet loss, ping {nameserver} failed.')
|
||||
return None
|
||||
elif platform.system() in ('Darwin', 'Linux'):
|
||||
try:
|
||||
avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0]
|
||||
logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
|
||||
except IndexError:
|
||||
return None
|
||||
return avg_time
|
||||
elif platform.system() == 'Windows':
|
||||
try:
|
||||
avg_time = re.findall(r'(?:Average|平均).+(\d.?)ms', text)[0]
|
||||
logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
|
||||
except IndexError:
|
||||
return None
|
||||
return avg_time
|
||||
else:
|
||||
logger.log('ALERT', f'{text}')
|
||||
return None
|
||||
|
||||
|
||||
def auto_select_nameserver():
|
||||
logger.log('INFOR', f'Ping test start, to select nameservers.')
|
||||
avg_time1 = ping_avg_time('114.114.114.114')
|
||||
avg_time2 = ping_avg_time('8.8.8.8')
|
||||
if avg_time1 and avg_time2:
|
||||
if avg_time1 < avg_time2:
|
||||
change_nameservers_file('cn')
|
||||
logger.log('INFOR', f'Ping test finished, use cn nameservers.')
|
||||
else:
|
||||
change_nameservers_file('common')
|
||||
logger.log('INFOR', f'Ping test finished, use common nameservers.')
|
||||
elif avg_time1 and not avg_time2:
|
||||
change_nameservers_file('cn')
|
||||
logger.log('INFOR', f'Ping test finished, use cn nameservers.')
|
||||
elif not avg_time1 and avg_time1:
|
||||
change_nameservers_file('common')
|
||||
logger.log('INFOR', f'Ping test finished, use common nameservers.')
|
||||
elif not avg_time1 and not avg_time1:
|
||||
change_nameservers_file('default')
|
||||
logger.log('INFOR', f'Ping test finished, use default nameservers.')
|
||||
return
|
||||
|
||||
|
||||
def change_nameservers_file(option):
|
||||
text = ''
|
||||
if option == 'cn':
|
||||
with open(settings.data_storage_dir.joinpath('cn_nameservers.txt'), 'r') as f:
|
||||
text = f.read()
|
||||
elif option == 'common':
|
||||
with open(settings.data_storage_dir.joinpath('common_nameservers.txt'), 'r') as f:
|
||||
text = f.read()
|
||||
elif option == 'default':
|
||||
for n in default_nameserver():
|
||||
text = '\n'.join(n)
|
||||
with open(settings.data_storage_dir.joinpath('nameservers.txt'), 'w') as f:
|
||||
f.write(text)
|
||||
return
|
||||
|
||||
|
||||
def default_nameserver():
|
||||
try:
|
||||
resolver = dns.resolver.Resolver()
|
||||
return resolver.nameservers
|
||||
except dns.resolver.NoResolverConfiguration:
|
||||
logger.log('ERROR', 'Resolver configuration could not be read '
|
||||
'or specified no nameservers.')
|
||||
exit(1)
|
||||
|
||||
|
||||
def looks_like_ip(maybe_ip):
|
||||
"""Does the given str look like an IP address?"""
|
||||
if not maybe_ip[0].isdigit():
|
||||
@@ -833,3 +763,38 @@ def looks_like_ip(maybe_ip):
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
|
||||
|
||||
def deal_data(domain):
|
||||
db = Database()
|
||||
db.remove_invalid(domain)
|
||||
db.deduplicate_subdomain(domain)
|
||||
db.close()
|
||||
|
||||
|
||||
def get_data(domain):
|
||||
db = Database()
|
||||
data = db.get_data(domain).as_dict()
|
||||
db.close()
|
||||
return data
|
||||
|
||||
|
||||
def clear_data(domain):
|
||||
db = Database()
|
||||
db.drop_table(domain)
|
||||
db.close()
|
||||
|
||||
|
||||
def get_ns_path(in_china=None, enable_wildcard=None, ns_ip_list=None):
|
||||
data_dir = settings.data_storage_dir
|
||||
path = data_dir.joinpath('nameservers.txt')
|
||||
if in_china:
|
||||
path = data_dir.joinpath('nameservers_cn.txt')
|
||||
if not enable_wildcard:
|
||||
return path
|
||||
if not ns_ip_list:
|
||||
return path
|
||||
path = settings.authoritative_dns_path
|
||||
ns_data = '\n'.join(ns_ip_list)
|
||||
save_data(path, ns_data)
|
||||
return path
|
||||
|
||||
+5
-8
@@ -24,7 +24,7 @@ enable_dns_resolve = True # 使用DNS解析子域(默认True)
|
||||
enable_http_request = True # 使用HTTP请求子域(默认True)
|
||||
enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
|
||||
enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
|
||||
enable_cdn_check = True # 开启cdn检查模块(默认True)
|
||||
enable_enrich_module = True # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
|
||||
enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
|
||||
enable_takeover_check = False # 开启子域接管风险检查(默认False)
|
||||
# 参数可选值有'small', 'medium', 'large'
|
||||
@@ -56,17 +56,14 @@ brute_process_num = 1 # 默认1
|
||||
brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000)
|
||||
brute_socket_num = 1 # 爆破时每个进程下的socket数量
|
||||
brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
# 爆破所使用的字典路径 默认data/cn_nameservers.txt
|
||||
# 如果你不在中国请改为nameservers.txt
|
||||
brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
|
||||
# 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
|
||||
brute_wordlist_path = None
|
||||
# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
|
||||
authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
|
||||
enable_recursive_brute = False # 是否使用递归爆破(默认False)
|
||||
brute_recursive_depth = 2 # 递归爆破深度(默认2层)
|
||||
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
|
||||
recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt')
|
||||
# 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
|
||||
recursive_nextlist_path = None
|
||||
enable_check_dict = False # 是否开启字典配置检查提示(默认False)
|
||||
delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True)
|
||||
delete_massdns_result = True # 是否删除爆破时massdns输出的解析结果 (默认True)
|
||||
|
||||
+6
-12
@@ -18,8 +18,8 @@ enable_finder_module = True # 开启finder模块,开启会从响应体和JS中
|
||||
enable_cdn_check = True # 开启cdn检查模块(默认True)
|
||||
enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
|
||||
enable_takeover_check = False # 开启子域接管风险检查(默认False)
|
||||
# 参数可选值有'small', 'medium', 'large'
|
||||
http_request_port = 'small' # HTTP请求子域(默认'small'，探测80,443端口)
|
||||
# HTTP请求子域的端口范围 参数可选值有 'small', 'medium', 'large'
|
||||
http_request_port = 'small' # 请求端口范围(默认 'small'，表示请求子域的80,443端口)
|
||||
# 参数可选值True，False分别表示导出存活，全部子域结果
|
||||
result_export_alive = False # 只导出存活的子域结果(默认False)
|
||||
result_save_format = 'csv' # 子域结果保存文件格式(默认csv)
|
||||
@@ -36,18 +36,12 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
|
||||
|
||||
# 爆破模块设置
|
||||
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
# 爆破所使用的DNS服务器路径 默认data/cn_nameservers.txt 如果你不在中国请改为nameservers.txt
|
||||
# DNS resolve server file path default data/nameservers.txt
|
||||
# If your computer's location are not in China, change `cn_nameservers.txt` to `nameservers.txt` plz.
|
||||
brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
|
||||
# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
|
||||
authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
|
||||
# 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
|
||||
brute_wordlist_path = None
|
||||
enable_recursive_brute = False # 是否使用递归爆破(默认False)
|
||||
brute_recursive_depth = 2 # 递归爆破深度(默认2层)
|
||||
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
|
||||
recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt')
|
||||
# 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
|
||||
recursive_nextlist_path = None
|
||||
enable_check_dict = False # 是否开启字典配置检查提示(默认False)
|
||||
delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True)
|
||||
# 是否删除爆破时massdns输出的解析结果 (默认True)
|
||||
|
||||
+133
-133
@@ -1,134 +1,134 @@
|
||||
[
|
||||
"10576",
|
||||
"10762",
|
||||
"11748",
|
||||
"131099",
|
||||
"132601",
|
||||
"133496",
|
||||
"134409",
|
||||
"135295",
|
||||
"136764",
|
||||
"137187",
|
||||
"13777",
|
||||
"13890",
|
||||
"14103",
|
||||
"14520",
|
||||
"17132",
|
||||
"199251",
|
||||
"200013",
|
||||
"200325",
|
||||
"200856",
|
||||
"201263",
|
||||
"202294",
|
||||
"203075",
|
||||
"203139",
|
||||
"204248",
|
||||
"204286",
|
||||
"204545",
|
||||
"206227",
|
||||
"206734",
|
||||
"206848",
|
||||
"206986",
|
||||
"207158",
|
||||
"208559",
|
||||
"209403",
|
||||
"21030",
|
||||
"21257",
|
||||
"23327",
|
||||
"23393",
|
||||
"23637",
|
||||
"23794",
|
||||
"24997",
|
||||
"26492",
|
||||
"268843",
|
||||
"28709",
|
||||
"29264",
|
||||
"30282",
|
||||
"30637",
|
||||
"328126",
|
||||
"36408",
|
||||
"38107",
|
||||
"397192",
|
||||
"40366",
|
||||
"43303",
|
||||
"44907",
|
||||
"46071",
|
||||
"46177",
|
||||
"47542",
|
||||
"49287",
|
||||
"49689",
|
||||
"51286",
|
||||
"55082",
|
||||
"55254",
|
||||
"56636",
|
||||
"57363",
|
||||
"58127",
|
||||
"59730",
|
||||
"59776",
|
||||
"60068",
|
||||
"60626",
|
||||
"60922",
|
||||
"61107",
|
||||
"61159",
|
||||
"62026",
|
||||
"62229",
|
||||
"63062",
|
||||
"64232",
|
||||
"8868",
|
||||
"9053",
|
||||
"55770",
|
||||
"49846",
|
||||
"49249",
|
||||
"48163",
|
||||
"45700",
|
||||
"43639",
|
||||
"39836",
|
||||
"393560",
|
||||
"393234",
|
||||
"36183",
|
||||
"35994",
|
||||
"35993",
|
||||
"35204",
|
||||
"34850",
|
||||
"34164",
|
||||
"33905",
|
||||
"32787",
|
||||
"31377",
|
||||
"31110",
|
||||
"31109",
|
||||
"31108",
|
||||
"31107",
|
||||
"30675",
|
||||
"24319",
|
||||
"23903",
|
||||
"23455",
|
||||
"23454",
|
||||
"22207",
|
||||
"21399",
|
||||
"21357",
|
||||
"21342",
|
||||
"20940",
|
||||
"20189",
|
||||
"18717",
|
||||
"18680",
|
||||
"17334",
|
||||
"16702",
|
||||
"16625",
|
||||
"12222",
|
||||
"209101",
|
||||
"201585",
|
||||
"135429",
|
||||
"395747",
|
||||
"394536",
|
||||
"209242",
|
||||
"203898",
|
||||
"202623",
|
||||
"14789",
|
||||
"133877",
|
||||
"13335",
|
||||
"132892",
|
||||
"21859",
|
||||
"6185",
|
||||
"47823",
|
||||
"4134"
|
||||
]
|
||||
"AS10576",
|
||||
"AS10762",
|
||||
"AS11748",
|
||||
"AS131099",
|
||||
"AS132601",
|
||||
"AS133496",
|
||||
"AS134409",
|
||||
"AS135295",
|
||||
"AS136764",
|
||||
"AS137187",
|
||||
"AS13777",
|
||||
"AS13890",
|
||||
"AS14103",
|
||||
"AS14520",
|
||||
"AS17132",
|
||||
"AS199251",
|
||||
"AS200013",
|
||||
"AS200325",
|
||||
"AS200856",
|
||||
"AS201263",
|
||||
"AS202294",
|
||||
"AS203075",
|
||||
"AS203139",
|
||||
"AS204248",
|
||||
"AS204286",
|
||||
"AS204545",
|
||||
"AS206227",
|
||||
"AS206734",
|
||||
"AS206848",
|
||||
"AS206986",
|
||||
"AS207158",
|
||||
"AS208559",
|
||||
"AS209403",
|
||||
"AS21030",
|
||||
"AS21257",
|
||||
"AS23327",
|
||||
"AS23393",
|
||||
"AS23637",
|
||||
"AS23794",
|
||||
"AS24997",
|
||||
"AS26492",
|
||||
"AS268843",
|
||||
"AS28709",
|
||||
"AS29264",
|
||||
"AS30282",
|
||||
"AS30637",
|
||||
"AS328126",
|
||||
"AS36408",
|
||||
"AS38107",
|
||||
"AS397192",
|
||||
"AS40366",
|
||||
"AS43303",
|
||||
"AS44907",
|
||||
"AS46071",
|
||||
"AS46177",
|
||||
"AS47542",
|
||||
"AS49287",
|
||||
"AS49689",
|
||||
"AS51286",
|
||||
"AS55082",
|
||||
"AS55254",
|
||||
"AS56636",
|
||||
"AS57363",
|
||||
"AS58127",
|
||||
"AS59730",
|
||||
"AS59776",
|
||||
"AS60068",
|
||||
"AS60626",
|
||||
"AS60922",
|
||||
"AS61107",
|
||||
"AS61159",
|
||||
"AS62026",
|
||||
"AS62229",
|
||||
"AS63062",
|
||||
"AS64232",
|
||||
"AS8868",
|
||||
"AS9053",
|
||||
"AS55770",
|
||||
"AS49846",
|
||||
"AS49249",
|
||||
"AS48163",
|
||||
"AS45700",
|
||||
"AS43639",
|
||||
"AS39836",
|
||||
"AS393560",
|
||||
"AS393234",
|
||||
"AS36183",
|
||||
"AS35994",
|
||||
"AS35993",
|
||||
"AS35204",
|
||||
"AS34850",
|
||||
"AS34164",
|
||||
"AS33905",
|
||||
"AS32787",
|
||||
"AS31377",
|
||||
"AS31110",
|
||||
"AS31109",
|
||||
"AS31108",
|
||||
"AS31107",
|
||||
"AS30675",
|
||||
"AS24319",
|
||||
"AS23903",
|
||||
"AS23455",
|
||||
"AS23454",
|
||||
"AS22207",
|
||||
"AS21399",
|
||||
"AS21357",
|
||||
"AS21342",
|
||||
"AS20940",
|
||||
"AS20189",
|
||||
"AS18717",
|
||||
"AS18680",
|
||||
"AS17334",
|
||||
"AS16702",
|
||||
"AS16625",
|
||||
"AS12222",
|
||||
"AS209101",
|
||||
"AS201585",
|
||||
"AS135429",
|
||||
"AS395747",
|
||||
"AS394536",
|
||||
"AS209242",
|
||||
"AS203898",
|
||||
"AS202623",
|
||||
"AS14789",
|
||||
"AS133877",
|
||||
"AS13335",
|
||||
"AS132892",
|
||||
"AS21859",
|
||||
"AS6185",
|
||||
"AS47823",
|
||||
"AS4134"
|
||||
]
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
8.8.8.8
|
||||
8.8.4.4
|
||||
9.9.9.9
|
||||
9.9.9.10
|
||||
149.112.112.112
|
||||
4.2.2.1
|
||||
4.2.2.2
|
||||
4.2.2.3
|
||||
4.2.2.4
|
||||
4.2.2.5
|
||||
4.2.2.6
|
||||
1.1.1.1
|
||||
1.0.0.1
|
||||
1.0.0.2
|
||||
1.0.0.3
|
||||
1.0.0.19
|
||||
208.67.222.222
|
||||
208.67.220.220
|
||||
8.26.56.26
|
||||
8.20.247.20
|
||||
84.200.69.80
|
||||
84.200.70.40
|
||||
185.228.168.9
|
||||
185.228.169.9
|
||||
64.6.64.6
|
||||
64.6.65.6
|
||||
198.101.242.72
|
||||
23.253.163.53
|
||||
176.103.130.130
|
||||
176.103.130.131
|
||||
@@ -0,0 +1,30 @@
|
||||
8.8.8.8
|
||||
8.8.4.4
|
||||
9.9.9.9
|
||||
9.9.9.10
|
||||
149.112.112.112
|
||||
4.2.2.1
|
||||
4.2.2.2
|
||||
4.2.2.3
|
||||
4.2.2.4
|
||||
4.2.2.5
|
||||
4.2.2.6
|
||||
1.1.1.1
|
||||
1.0.0.1
|
||||
1.0.0.2
|
||||
1.0.0.3
|
||||
1.0.0.19
|
||||
208.67.222.222
|
||||
208.67.220.220
|
||||
8.26.56.26
|
||||
8.20.247.20
|
||||
84.200.69.80
|
||||
84.200.70.40
|
||||
185.228.168.9
|
||||
185.228.169.9
|
||||
64.6.64.6
|
||||
64.6.65.6
|
||||
198.101.242.72
|
||||
23.253.163.53
|
||||
176.103.130.130
|
||||
176.103.130.131
|
||||
Binary file not shown.
+12
-12
@@ -15,17 +15,17 @@ from common.database import Database
|
||||
from config.log import logger
|
||||
|
||||
|
||||
def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False):
|
||||
def export(target, type='target', db=None, alive=False, limit=None, path=None, fmt='csv', show=False):
|
||||
"""
|
||||
OneForAll export from database module
|
||||
|
||||
Example:
|
||||
python3 dbexport.py --target name --format csv --dir= ./result.csv
|
||||
python3 dbexport.py --target name --fmt csv --dir= ./result.csv
|
||||
python3 dbexport.py --db result.db --target name --show False
|
||||
python3 dbexport.py --target table_name --tb True --show False
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
:param str target: Table to be exported
|
||||
@@ -33,7 +33,7 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
|
||||
:param str db: Database path to be exported (default ./results/result.sqlite3)
|
||||
:param bool alive: Only export the results of alive subdomains (default False)
|
||||
:param str limit: Export limit (default None)
|
||||
:param str format: Result format (default csv)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
:param bool show: Displays the exported data in terminal (default False)
|
||||
"""
|
||||
@@ -48,29 +48,29 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
|
||||
rows = database.export_data(table_name, alive, limit)
|
||||
if rows is None:
|
||||
continue
|
||||
data = export_data(format, path, rows, show, table_name, target)
|
||||
data = export_data(fmt, path, rows, show, table_name, target)
|
||||
datas.extend(data)
|
||||
database.close()
|
||||
if len(domains) > 1:
|
||||
utils.export_all(alive, format, path, datas)
|
||||
utils.export_all(alive, fmt, path, datas)
|
||||
elif type == 'table':
|
||||
database = Database(db)
|
||||
rows = database.export_data(target, alive, limit)
|
||||
data = export_data(format, path, rows, show, target, target)
|
||||
data, _, _ = export_data(fmt, path, rows, show, target, target)
|
||||
database.close()
|
||||
return data
|
||||
|
||||
|
||||
def export_data(format, path, rows, show, table_name, target):
|
||||
format = utils.check_format(format, len(rows))
|
||||
path = utils.check_path(path, target, format)
|
||||
def export_data(fmt, path, rows, show, table_name, target):
|
||||
fmt = utils.check_format(fmt)
|
||||
path = utils.check_path(path, target, fmt)
|
||||
if show:
|
||||
print(rows.dataset)
|
||||
data = rows.export(format)
|
||||
data = rows.export(fmt)
|
||||
utils.save_data(path, data)
|
||||
logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
|
||||
data = rows.as_dict()
|
||||
return data, format, path
|
||||
return data, fmt, path
|
||||
|
||||
|
||||
def domain_to_table(table):
|
||||
|
||||
@@ -47,11 +47,11 @@ D:.
|
||||
|
|
||||
+---data 存放一些所需数据
|
||||
| authoritative_dns.txt 临时存放开启了泛解析域名的权威DNS名称服务器IP地址
|
||||
| big_subnames.txt 子域爆破超大字典
|
||||
| cn_nameservers.txt 中国主流名称服务器IP地址
|
||||
| subnames_big.7z 子域爆破超大字典
|
||||
| nameservers_cn.txt 中国主流名称服务器IP地址
|
||||
| fingerprints.json 检查子域接管风险的指纹
|
||||
| nameservers.txt 全球主流名称服务器IP地址
|
||||
| next_subnames.txt 下一层子域字典
|
||||
| subnames_next.txt 下一层子域字典
|
||||
| public_suffix_list.dat 顶级域名后缀
|
||||
| srv_prefixes.json 常见SRV记录前缀名
|
||||
| subnames.txt 子域爆破常见字典
|
||||
@@ -175,4 +175,4 @@ D:.
|
||||
cyggcc_s-1.dll
|
||||
cygwin1.dll
|
||||
massdns_windows_i686.exe
|
||||
```
|
||||
```
|
||||
|
||||
@@ -89,8 +89,6 @@ Result will be saved in `~/results`.
|
||||
<details>
|
||||
<summary><b>✨OneForAll usage</b></summary>
|
||||
|
||||
If your computer are not in China, change [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) `brute_nameservers_path` param `cn_nameservers.txt` to `nameservers.txt` plz.
|
||||
|
||||
If you are use pip3, run the following command:
|
||||
|
||||
```bash
|
||||
@@ -164,7 +162,7 @@ DESCRIPTION
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -173,7 +171,7 @@ DESCRIPTION
|
||||
Note:
|
||||
--alive True/False Only export alive subdomains or not (default False)
|
||||
--port default/small/large See details in ./config/setting.py(default port 80)
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
ARGUMENTS
|
||||
@@ -193,7 +191,7 @@ FLAGS
|
||||
The port range request to the subdomains (default port 80)
|
||||
--alive=ALIVE
|
||||
Only export alive subdomains (default False)
|
||||
--format=FORMAT
|
||||
--fmt=FMT
|
||||
Result format (default csv)
|
||||
--path=PATH
|
||||
Result directory (default None)
|
||||
@@ -270,7 +268,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
|
||||
## ⌛Follow-up plan
|
||||
|
||||
- [ ] Continuous optimize and improve of each module
|
||||
- [x] Subdomain monitoring (mark newly discovered subdomain)
|
||||
- [x] Subdomain collection crawler (collect subdomains from static files such as JS)
|
||||
- [ ] Implementation of front-end interface for powerful interaction
|
||||
|
||||
|
||||
+2
-4
@@ -193,9 +193,7 @@ class Altdns(Module):
|
||||
self.end = time.time()
|
||||
self.elapse = round(self.end - self.start, 1)
|
||||
self.gen_result()
|
||||
temp_data = resolve.run_resolve(self.domain, self.results)
|
||||
fina_data = request.run_request(self.domain, temp_data, port)
|
||||
data = data + fina_data
|
||||
resolved_data = resolve.run_resolve(self.domain, self.results)
|
||||
request.run_request(self.domain, resolved_data, port)
|
||||
logger.log('INFOR', f'Saving altdns results')
|
||||
utils.save_db(self.domain, data, 'altdns')
|
||||
return data
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
from config.log import logger
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class PhoneBook(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Dataset'
|
||||
self.source = 'PhoneBookQuery'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.header.update({'Referer': 'https://phonebook.cz/',
|
||||
'Origin': 'https://phonebook.cz'})
|
||||
addr = 'https://public.intelx.io/phonebook/search'
|
||||
key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab'
|
||||
url = f'{addr}?k={key}'
|
||||
data = {"term": self.domain, "maxresults": 10000,
|
||||
"media": 0, "target": 1,
|
||||
"terminate": [], "timeout": 20}
|
||||
resp = self.post(url, json=data)
|
||||
if not resp:
|
||||
return
|
||||
json = resp.json()
|
||||
ids = json.get('id')
|
||||
if not ids:
|
||||
logger.log('ALERT', f'Get PhoneBook id fail')
|
||||
return
|
||||
url = f'{addr}/result?k={key}&id={ids}&limit=10000'
|
||||
resp = self.get(url)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def run(domain):
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = PhoneBook(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run('freebuf.com')
|
||||
@@ -17,7 +17,7 @@ class Robtex(Query):
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
base_addr = 'https://freeapi.robtex.com/pdns/'
|
||||
base_addr = 'https://freeapi.robtex.com/pdns'
|
||||
url = f'{base_addr}/forward/{self.domain}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
from modules import iscdn
|
||||
from common import utils
|
||||
from common.database import Database
|
||||
from common.ipasn import IPAsnInfo
|
||||
from common.ipreg import IpRegData
|
||||
|
||||
|
||||
def get_ips(info):
|
||||
ip = info.get('ip')
|
||||
if not ip:
|
||||
return None
|
||||
ips = ip.split(',')
|
||||
return ips
|
||||
|
||||
|
||||
def enrich_info(data):
|
||||
ip_asn = IPAsnInfo()
|
||||
ip_reg = IpRegData()
|
||||
for index, info in enumerate(data):
|
||||
ips = get_ips(info)
|
||||
if not ips:
|
||||
continue
|
||||
public = list()
|
||||
cidr = list()
|
||||
asn = list()
|
||||
org = list()
|
||||
addr = list()
|
||||
isp = list()
|
||||
for ip in ips:
|
||||
public.append(str(utils.ip_is_public(ip)))
|
||||
asn_info = ip_asn.find(ip)
|
||||
cidr.append(asn_info.get('cidr'))
|
||||
asn.append(asn_info.get('asn'))
|
||||
org.append(asn_info.get('org'))
|
||||
ip_info = ip_reg.query(ip)
|
||||
addr.append(ip_info.get('addr'))
|
||||
isp.append(ip_info.get('isp'))
|
||||
data[index]['public'] = ','.join(public)
|
||||
data[index]['cidr'] = ','.join(cidr)
|
||||
data[index]['asn'] = ','.join(asn)
|
||||
data[index]['org'] = ','.join(org)
|
||||
data[index]['addr'] = ','.join(addr)
|
||||
data[index]['isp'] = ','.join(isp)
|
||||
return data
|
||||
|
||||
|
||||
class Enrich(object):
|
||||
def __init__(self, domain):
|
||||
self.domain = domain
|
||||
|
||||
def get_data(self):
|
||||
db = Database()
|
||||
fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
|
||||
'cidr', 'asn', 'org', 'addr', 'isp']
|
||||
results = db.get_data_by_fields(self.domain, fields)
|
||||
return results.as_dict()
|
||||
|
||||
def save_db(self, data):
|
||||
db = Database()
|
||||
for info in data:
|
||||
url = info.pop('url')
|
||||
info.pop('cname')
|
||||
info.pop('ip')
|
||||
info.pop('header')
|
||||
db.update_data_by_url(self.domain, info, url)
|
||||
db.close()
|
||||
|
||||
def run(self):
|
||||
data = self.get_data()
|
||||
data = enrich_info(data)
|
||||
data = iscdn.do_check(data)
|
||||
self.save_db(data)
|
||||
+15
-10
@@ -7,6 +7,7 @@ from common import utils
|
||||
from common import resolve
|
||||
from common import request
|
||||
from common.module import Module
|
||||
from common.database import Database
|
||||
from config import settings
|
||||
from config.log import logger
|
||||
|
||||
@@ -25,16 +26,11 @@ class Finder(Module):
|
||||
new_subdomains = found_subdomains - existing_subdomains
|
||||
if not len(new_subdomains):
|
||||
self.finish() # 未发现新的子域就直接返回
|
||||
return data
|
||||
self.subdomains = new_subdomains
|
||||
self.finish()
|
||||
self.gen_result()
|
||||
temp_data = resolve.run_resolve(domain, self.results)
|
||||
fina_data = request.run_request(domain, temp_data, port)
|
||||
data = data + fina_data
|
||||
logger.log('INFOR', f'Saving finder results')
|
||||
utils.save_db(domain, data, 'finder')
|
||||
return data
|
||||
resolved_data = resolve.run_resolve(domain, self.results)
|
||||
request.run_request(domain, resolved_data, port)
|
||||
|
||||
|
||||
file_path = settings.data_storage_dir.joinpath('common_js_library.json')
|
||||
@@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html):
|
||||
return js_urls
|
||||
|
||||
|
||||
def convert_to_dict(url_list):
|
||||
url_dict = []
|
||||
for url in url_list:
|
||||
url_dict.append({'url': url})
|
||||
return url_dict
|
||||
|
||||
def find_subdomains(domain, data):
|
||||
subdomains = set()
|
||||
js_urls = set()
|
||||
db = Database()
|
||||
for infos in data:
|
||||
jump_history = infos.get('history')
|
||||
req_url = infos.get('url')
|
||||
subdomains.update(find_in_history(domain, req_url, jump_history))
|
||||
rsp_html = infos.get('response')
|
||||
rsp_html = db.get_resp_by_url(domain, req_url)
|
||||
if not rsp_html:
|
||||
logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}')
|
||||
continue
|
||||
subdomains.update(find_in_resp(domain, req_url, rsp_html))
|
||||
js_urls.update(find_js_urls(domain, req_url, rsp_html))
|
||||
|
||||
resp_data = request.bulk_request(js_urls)
|
||||
for _, resp in resp_data:
|
||||
req_data = convert_to_dict(js_urls)
|
||||
resp_data = request.bulk_request(domain, req_data, ret=True)
|
||||
while not resp_data.empty():
|
||||
_, resp = resp_data.get()
|
||||
if not isinstance(resp, Response):
|
||||
continue
|
||||
text = utils.decode_resp_text(resp)
|
||||
|
||||
+35
-33
@@ -3,6 +3,7 @@ import ipaddress
|
||||
|
||||
from config import settings
|
||||
from common import utils
|
||||
from common.database import Database
|
||||
from config.log import logger
|
||||
|
||||
data_dir = settings.data_storage_dir
|
||||
@@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
|
||||
|
||||
|
||||
def check_cname_keyword(cname):
|
||||
if not cname:
|
||||
return False
|
||||
names = cname.lower().split(',')
|
||||
for name in names:
|
||||
for keyword in cdn_cname_keyword.keys():
|
||||
@@ -26,14 +29,22 @@ def check_cname_keyword(cname):
|
||||
|
||||
|
||||
def check_header_key(header):
|
||||
header = set(map(lambda x: x.lower(), header.keys()))
|
||||
for key in cdn_header_key:
|
||||
if key in header:
|
||||
return True
|
||||
if isinstance(header, str):
|
||||
header = json.loads(header)
|
||||
if isinstance(header, dict):
|
||||
header = set(map(lambda x: x.lower(), header.keys()))
|
||||
for key in cdn_header_key:
|
||||
if key in header:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def check_cdn_cidr(content):
|
||||
ips = set(content.split(','))
|
||||
def check_cdn_cidr(ips):
|
||||
if isinstance(ips, str):
|
||||
ips = set(ips.split(','))
|
||||
else:
|
||||
return False
|
||||
for ip in ips:
|
||||
try:
|
||||
ip = ipaddress.ip_address(ip)
|
||||
@@ -46,39 +57,30 @@ def check_cdn_cidr(content):
|
||||
|
||||
|
||||
def check_cdn_asn(asn):
|
||||
if str(asn) in cdn_asn_list:
|
||||
return True
|
||||
if isinstance(asn, str):
|
||||
if asn in cdn_asn_list:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_cdn(data):
|
||||
logger.log('DEBUG', f'Start cdn check module')
|
||||
def do_check(data):
|
||||
logger.log('DEBUG', f'Checking cdn')
|
||||
for index, item in enumerate(data):
|
||||
cname = item.get('cname')
|
||||
if cname:
|
||||
if check_cname_keyword(cname):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
if check_cname_keyword(cname):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
header = item.get('header')
|
||||
if header:
|
||||
header = json.loads(header)
|
||||
if check_header_key(header):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
if check_header_key(header):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
ip = item.get('ip')
|
||||
if ip:
|
||||
if check_cdn_cidr(ip):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
if check_cdn_cidr(ip):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
asn = item.get('asn')
|
||||
if asn:
|
||||
asn = asn[2:] # 去除AS
|
||||
if check_cdn_asn(asn):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
if check_cdn_asn(asn):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
data[index]['cdn'] = 0
|
||||
return data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
logger.log('DEBUG', f'Saving cdn check results')
|
||||
utils.save_db(name, data, 'cdn')
|
||||
|
||||
@@ -57,7 +57,7 @@ class Baidu(Search):
|
||||
self.subdomains.update(subdomains)
|
||||
self.page_num += self.per_page_num
|
||||
# 搜索页面没有出现下一页时停止搜索
|
||||
if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
|
||||
if f'&pn={self.page_num}&' not in resp.text:
|
||||
break
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
+45
-88
@@ -15,12 +15,11 @@ from datetime import datetime
|
||||
import dbexport
|
||||
from brute import Brute
|
||||
from common import utils, resolve, request
|
||||
from common.database import Database
|
||||
from modules.collect import Collect
|
||||
from modules.srv import BruteSRV
|
||||
from modules.finder import Finder
|
||||
from modules.altdns import Altdns
|
||||
from modules import iscdn
|
||||
from modules.enrich import Enrich
|
||||
from config import settings
|
||||
from config.log import logger
|
||||
from takeover import Takeover
|
||||
@@ -60,7 +59,7 @@ class OneForAll(object):
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute False run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -68,22 +67,22 @@ class OneForAll(object):
|
||||
|
||||
Note:
|
||||
--port small/medium/large See details in ./config/setting.py(default small)
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param bool brute: Use brute module (default True)
|
||||
:param bool dns: Use DNS resolution (default True)
|
||||
:param bool req: HTTP request subdomains (default True)
|
||||
:param bool brute: Use brute module (default True)
|
||||
:param bool dns: Use DNS resolution (default True)
|
||||
:param bool req: HTTP request subdomains (default True)
|
||||
:param str port: The port range to request (default small port is 80,443)
|
||||
:param bool alive: Only export alive subdomains (default False)
|
||||
:param str format: Result format (default csv)
|
||||
:param bool alive: Only export alive subdomains (default False)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result path (default None, automatically generated)
|
||||
:param bool takeover: Scan subdomain takeover (default False)
|
||||
:param bool takeover: Scan subdomain takeover (default False)
|
||||
"""
|
||||
def __init__(self, target=None, targets=None, brute=None, dns=None, req=None,
|
||||
port=None, alive=None, format=None, path=None, takeover=None):
|
||||
port=None, alive=None, fmt=None, path=None, takeover=None):
|
||||
self.target = target
|
||||
self.targets = targets
|
||||
self.brute = brute
|
||||
@@ -91,17 +90,15 @@ class OneForAll(object):
|
||||
self.req = req
|
||||
self.port = port
|
||||
self.alive = alive
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.path = path
|
||||
self.takeover = takeover
|
||||
self.domain = str() # The domain currently being collected
|
||||
self.domains = set() # All domains that are to be collected
|
||||
self.data = list() # The subdomain results of the current domain
|
||||
self.datas = list() # All subdomain results of the domain
|
||||
self.old_table = str() # The table name of the last result
|
||||
self.new_table = str() # The table name of the current result
|
||||
self.origin_table = str() # The table name of the origin result
|
||||
self.resolve_table = str() # The table name of the resolute result
|
||||
self.in_china = None
|
||||
self.access_internet = False
|
||||
|
||||
def config_param(self):
|
||||
"""
|
||||
@@ -119,8 +116,8 @@ class OneForAll(object):
|
||||
self.port = settings.http_request_port
|
||||
if self.alive is None:
|
||||
self.alive = bool(settings.result_export_alive)
|
||||
if self.format is None:
|
||||
self.format = settings.result_save_format
|
||||
if self.fmt is None:
|
||||
self.fmt = settings.result_save_format
|
||||
if self.path is None:
|
||||
self.path = settings.result_save_path
|
||||
|
||||
@@ -134,47 +131,13 @@ class OneForAll(object):
|
||||
|
||||
def export(self, table):
|
||||
"""
|
||||
Export data from the database and do some follow-up processing
|
||||
Export data from the database
|
||||
|
||||
:param table: table name
|
||||
:return: export data
|
||||
:rtype: list
|
||||
"""
|
||||
db = Database()
|
||||
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
|
||||
db.drop_table(self.new_table)
|
||||
db.rename_table(self.domain, self.new_table)
|
||||
db.close()
|
||||
return data
|
||||
|
||||
def deal_db(self):
|
||||
"""
|
||||
Process the data when the collection task is completed
|
||||
"""
|
||||
db = Database()
|
||||
db.deal_table(self.domain, self.origin_table)
|
||||
db.close()
|
||||
|
||||
def mark(self):
|
||||
"""
|
||||
Mark the new discovered subdomain
|
||||
|
||||
:return: marked data
|
||||
:rtype: list
|
||||
"""
|
||||
db = Database()
|
||||
old_data = list()
|
||||
now_data = db.get_data(self.domain).as_dict()
|
||||
# Database pre-processing when it is not the first time to collect this subdomain
|
||||
if db.exist_table(self.new_table):
|
||||
# If there is the last collection result table, delete it first
|
||||
db.drop_table(self.old_table)
|
||||
# Rename the new table to the old table
|
||||
db.rename_table(self.new_table, self.old_table)
|
||||
old_data = db.get_data(self.old_table).as_dict()
|
||||
db.close()
|
||||
marked_data = utils.mark_subdomain(old_data, now_data)
|
||||
return marked_data
|
||||
return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt)
|
||||
|
||||
def main(self):
|
||||
"""
|
||||
@@ -183,13 +146,12 @@ class OneForAll(object):
|
||||
:return: subdomain results
|
||||
:rtype: list
|
||||
"""
|
||||
self.old_table = self.domain + '_old_result'
|
||||
self.new_table = self.domain + '_now_result'
|
||||
self.origin_table = self.domain + '_origin_result'
|
||||
self.resolve_table = self.domain + '_resolve_result'
|
||||
|
||||
collect = Collect(self.domain)
|
||||
collect.run()
|
||||
if not self.access_internet:
|
||||
logger.log('ALERT', 'Because it cannot access the Internet, '
|
||||
'OneForAll will not execute the subdomain collection module!')
|
||||
if self.access_internet:
|
||||
collect = Collect(self.domain)
|
||||
collect.run()
|
||||
|
||||
srv = BruteSRV(self.domain)
|
||||
srv.run()
|
||||
@@ -198,53 +160,48 @@ class OneForAll(object):
|
||||
# Due to there will be a large number of dns resolution requests,
|
||||
# may cause other network tasks to be error
|
||||
brute = Brute(self.domain, word=True, export=False)
|
||||
brute.check_env = False
|
||||
brute.in_china = self.in_china
|
||||
brute.quite = True
|
||||
brute.run()
|
||||
|
||||
# Database processing
|
||||
self.deal_db()
|
||||
# Mark the new discovered subdomain
|
||||
self.data = self.mark()
|
||||
|
||||
utils.deal_data(self.domain)
|
||||
# Export results without resolve
|
||||
if not self.dns:
|
||||
return self.export(self.domain)
|
||||
|
||||
self.data = utils.get_data(self.domain)
|
||||
|
||||
# Resolve subdomains
|
||||
utils.clear_data(self.domain)
|
||||
self.data = resolve.run_resolve(self.domain, self.data)
|
||||
# Save resolve results
|
||||
resolve.save_db(self.resolve_table, self.data)
|
||||
resolve.save_db(self.domain, self.data)
|
||||
|
||||
# Export results without HTTP request
|
||||
if not self.req:
|
||||
return self.export(self.resolve_table)
|
||||
return self.export(self.domain)
|
||||
|
||||
# HTTP request
|
||||
self.data = request.run_request(self.domain, self.data, self.port)
|
||||
# Save HTTP request result
|
||||
request.save_db(self.domain, self.data)
|
||||
utils.clear_data(self.domain)
|
||||
request.run_request(self.domain, self.data, self.port)
|
||||
|
||||
# Finder module
|
||||
if settings.enable_finder_module:
|
||||
finder = Finder()
|
||||
self.data = finder.run(self.domain, self.data, self.port)
|
||||
finder.run(self.domain, self.data, self.port)
|
||||
|
||||
# altdns module
|
||||
if settings.enable_altdns_module:
|
||||
finder = Altdns(self.domain)
|
||||
self.data = finder.run(self.data, self.port)
|
||||
altdns = Altdns(self.domain)
|
||||
altdns.run(self.data, self.port)
|
||||
|
||||
# check cdn module
|
||||
if settings.enable_cdn_check:
|
||||
self.data = iscdn.check_cdn(self.data)
|
||||
iscdn.save_db(self.domain, self.data)
|
||||
|
||||
# Add the final result list to the total data list
|
||||
self.datas.extend(self.data)
|
||||
# Information enrichment module
|
||||
if settings.enable_enrich_module:
|
||||
enrich = Enrich(self.domain)
|
||||
enrich.run()
|
||||
|
||||
# Export
|
||||
self.export(self.domain)
|
||||
self.datas.extend(self.export(self.domain))
|
||||
|
||||
# Scan subdomain takeover
|
||||
if self.takeover:
|
||||
@@ -263,12 +220,12 @@ class OneForAll(object):
|
||||
print(oneforall_banner)
|
||||
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
print(f'[*] Starting OneForAll @ {dt}\n')
|
||||
utils.check_env()
|
||||
utils.auto_select_nameserver()
|
||||
if settings.enable_check_version:
|
||||
utils.check_version(version)
|
||||
logger.log('DEBUG', 'Python ' + utils.python_version())
|
||||
logger.log('DEBUG', 'OneForAll ' + version)
|
||||
utils.check_dep()
|
||||
self.access_internet, self.in_china = utils.get_net_env()
|
||||
if self.access_internet and settings.enable_check_version:
|
||||
utils.check_version(version)
|
||||
logger.log('INFOR', 'Start running OneForAll')
|
||||
self.config_param()
|
||||
self.check_param()
|
||||
@@ -277,7 +234,7 @@ class OneForAll(object):
|
||||
for domain in self.domains:
|
||||
self.domain = utils.get_main_domain(domain)
|
||||
self.main()
|
||||
utils.export_all(self.alive, self.format, self.path, self.datas)
|
||||
utils.export_all(self.alive, self.fmt, self.path, self.datas)
|
||||
else:
|
||||
logger.log('FATAL', 'Failed to obtain domain')
|
||||
logger.log('INFOR', 'Finished OneForAll')
|
||||
|
||||
+10
-10
@@ -1,23 +1,23 @@
|
||||
-i https://mirrors.aliyun.com/pypi/simple/
|
||||
beautifulsoup4==4.9.1
|
||||
beautifulsoup4==4.9.3
|
||||
bs4==0.0.1
|
||||
certifi==2020.6.20
|
||||
chardet==3.0.4
|
||||
colorama==0.4.3 ; sys_platform == 'win32'
|
||||
colorama==0.4.4; sys_platform == 'win32'
|
||||
dnspython==2.0.0
|
||||
exrex==0.10.5
|
||||
fire==0.3.1
|
||||
future==0.18.2
|
||||
idna==2.10
|
||||
future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
loguru==0.5.3
|
||||
pysocks==1.7.1
|
||||
requests==2.24.0
|
||||
six==1.15.0
|
||||
soupsieve==2.0.1
|
||||
sqlalchemy==1.3.19
|
||||
six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
soupsieve==2.0.1; python_version >= '3.0'
|
||||
sqlalchemy==1.3.20
|
||||
tenacity==6.2.0
|
||||
termcolor==1.1.0
|
||||
tqdm==4.49.0
|
||||
tqdm==4.51.0
|
||||
treelib==1.6.1
|
||||
urllib3==1.25.10
|
||||
win32-setctime==1.0.2 ; sys_platform == 'win32'
|
||||
urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
|
||||
win32-setctime==1.0.3; sys_platform == 'win32'
|
||||
|
||||
+9
-9
@@ -45,21 +45,21 @@ class Takeover(Module):
|
||||
OneForAll subdomain takeover module
|
||||
|
||||
Example:
|
||||
python3 takeover.py --target www.example.com --format csv run
|
||||
python3 takeover.py --target www.example.com --fmt csv run
|
||||
python3 takeover.py --targets ./subdomains.txt --thread 10 run
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--fmt txt/csv/json (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param int thread: threads number (default 20)
|
||||
:param str format: Result format (default csv)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
"""
|
||||
|
||||
def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'):
|
||||
def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'):
|
||||
Module.__init__(self)
|
||||
self.subdomains = set()
|
||||
self.module = 'Check'
|
||||
@@ -68,7 +68,7 @@ class Takeover(Module):
|
||||
self.targets = targets
|
||||
self.thread = thread
|
||||
self.path = path
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.fingerprints = None
|
||||
self.subdomainq = Queue()
|
||||
self.cnames = list()
|
||||
@@ -76,10 +76,10 @@ class Takeover(Module):
|
||||
|
||||
def save(self):
|
||||
logger.log('DEBUG', 'Saving results')
|
||||
if self.format == 'txt':
|
||||
if self.fmt == 'txt':
|
||||
data = str(self.results)
|
||||
else:
|
||||
data = self.results.export(self.format)
|
||||
data = self.results.export(self.fmt)
|
||||
utils.save_data(self.path, data)
|
||||
|
||||
def compare(self, subdomain, cname, responses):
|
||||
@@ -131,10 +131,10 @@ class Takeover(Module):
|
||||
self.subdomains = self.targets
|
||||
else:
|
||||
self.subdomains = utils.get_domains(self.target, self.targets)
|
||||
self.format = utils.check_format(self.format, len(self.subdomains))
|
||||
self.fmt = utils.check_format(self.fmt)
|
||||
timestamp = utils.get_timestamp()
|
||||
name = f'takeover_check_result_{timestamp}'
|
||||
self.path = utils.check_path(self.path, name, self.format)
|
||||
self.path = utils.check_path(self.path, name, self.fmt)
|
||||
if self.subdomains:
|
||||
logger.log('INFOR', f'Checking subdomain takeover')
|
||||
self.fingerprints = get_fingerprint()
|
||||
|
||||
Reference in New Issue
Block a user