From 306f9ce6c20b1bf67ab6b3a972eccfb35c11eeae Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 15 Sep 2020 23:21:53 +0800 Subject: [PATCH 01/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/resolve.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/resolve.py b/common/resolve.py index 66ca4b1..0a46634 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -21,9 +21,9 @@ def filter_subdomain(data): """ logger.log('DEBUG', f'Filtering subdomains to be resolved') subdomains = [] - for data in data: - if not data.get('ip'): - subdomain = data.get('subdomain') + for infos in data: + if not infos.get('ip'): + subdomain = infos.get('subdomain') if subdomain: subdomains.append(subdomain) return subdomains From aa34cb87d38ff8233d3894a07e4a2399fa951110 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 15 Sep 2020 23:37:06 +0800 Subject: [PATCH 02/12] typos --- common/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/module.py b/common/module.py index 5f3aaba..e9e5880 100644 --- a/common/module.py +++ b/common/module.py @@ -95,7 +95,7 @@ class Module(object): return resp return None - def get(self, url, params=None, check=True, ignore=False,raise_error=False, **kwargs): + def get(self, url, params=None, check=True, ignore=False, raise_error=False, **kwargs): """ Custom get request @@ -171,7 +171,7 @@ class Module(object): :param str url: request url :param bool check: check response :param kwargs: other params - :return: requests's response object + :return: response object """ session = requests.Session() session.trust_env = False From aa7cec7fbbddd60af3a91116aab9115408e77863 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 15 Sep 2020 23:51:58 +0800 Subject: [PATCH 03/12] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E5=90=8D=E8=A6=86=E7=9B=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/module.py | 2 +- modules/certificates/censys_api.py | 2 +- modules/datasets/binaryedge_api.py | 2 +- modules/datasets/chinaz_api.py | 2 +- modules/datasets/circl_api.py | 2 +- modules/datasets/cloudflare_api.py | 2 +- modules/datasets/dnsdb_api.py | 2 +- modules/datasets/ipv4info_api.py | 2 +- modules/datasets/passivedns_api.py | 2 +- modules/datasets/securitytrails_api.py | 2 +- modules/datasets/spyse_api.py | 2 +- modules/intelligence/riskiq_api.py | 2 +- modules/intelligence/threatbook_api.py | 2 +- modules/intelligence/virustotal_api.py | 2 +- modules/search/bing_api.py | 2 +- modules/search/fofa_api.py | 2 +- modules/search/github_api.py | 2 +- modules/search/google_api.py | 2 +- modules/search/shodan_api.py | 2 +- modules/search/zoomeye_api.py | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/common/module.py b/common/module.py index e9e5880..cc2a8fc 100644 --- a/common/module.py +++ b/common/module.py @@ -33,7 +33,7 @@ class Module(object): self.end = None # 模块结束执行时间 self.elapse = None # 模块执行耗时 - def check(self, *apis): + def have_api(self, *apis): """ Simply check whether the api information configure or not diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index 33d6682..3e13d62 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -45,7 +45,7 @@ class CensysAPI(Query): """ 类执行入口 """ - if not self.check(self.id, self.secret): + if not self.have_api(self.id, self.secret): return self.begin() self.query() diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index 4057105..e5383da 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -26,7 +26,7 @@ class BinaryEdgeAPI(Query): """ 类执行入口 """ - if not self.check(self.api): + if not self.have_api(self.api): return self.begin() self.query() diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index e7a8d37..f8d2f90 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -25,7 +25,7 @@ class ChinazAPI(Query): """ 类执行入口 """ - if not self.check(self.api): + if not self.have_api(self.api): return self.begin() self.query() diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index ddd0bc5..52b4075 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -25,7 +25,7 @@ class CirclAPI(Query): """ 类执行入口 """ - if not self.check(self.user, self.pwd): + if not self.have_api(self.user, self.pwd): return self.begin() self.query() diff --git a/modules/datasets/cloudflare_api.py b/modules/datasets/cloudflare_api.py index 89f95b3..4c7a6f2 100644 --- a/modules/datasets/cloudflare_api.py +++ b/modules/datasets/cloudflare_api.py @@ -109,7 +109,7 @@ class CloudFlareAPI(Query): """ class entrance """ - if not self.check(self.token): + if not self.have_api(self.token): return self.begin() self.query() diff --git a/modules/datasets/dnsdb_api.py b/modules/datasets/dnsdb_api.py index 0a30e64..497818e 100644 --- a/modules/datasets/dnsdb_api.py +++ b/modules/datasets/dnsdb_api.py @@ -26,7 +26,7 @@ class DNSdbAPI(Query): """ 类执行入口 """ - if not self.check(self.api): + if not self.have_api(self.api): return self.begin() self.query() diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index c41d7f3..3674d45 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -49,7 +49,7 @@ class IPv4InfoAPI(Query): """ 类执行入口 """ - if not self.check(self.api): + if not self.have_api(self.api): return self.begin() self.query() diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index cabc202..9da805e 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -27,7 +27,7 @@ class PassiveDnsAPI(Query): 类执行入口 """ if 'api.passivedns.cn' in self.addr: - if not self.check(self.token): + if not self.have_api(self.token): return self.begin() self.query() diff --git a/modules/datasets/securitytrails_api.py b/modules/datasets/securitytrails_api.py index 7a8e34d..13580d7 100644 --- a/modules/datasets/securitytrails_api.py +++ b/modules/datasets/securitytrails_api.py @@ -32,7 +32,7 @@ class SecurityTrailsAPI(Query): """ 类执行入口 """ - if not self.check(self.api): + if not self.have_api(self.api): return self.begin() self.query() diff --git a/modules/datasets/spyse_api.py b/modules/datasets/spyse_api.py index 59651f3..8d70b6a 100644 --- a/modules/datasets/spyse_api.py +++ b/modules/datasets/spyse_api.py @@ -38,7 +38,7 @@ class SpyseAPI(Query): """ 类执行入口 """ - if not self.check(self.token): + if not self.have_api(self.token): return self.begin() self.query() diff --git a/modules/intelligence/riskiq_api.py b/modules/intelligence/riskiq_api.py index 4a4cca0..b388551 100644 --- a/modules/intelligence/riskiq_api.py +++ b/modules/intelligence/riskiq_api.py @@ -33,7 +33,7 @@ class RiskIQ(Query): """ 类执行入口 """ - if not self.check(self.user, self.key): + if not self.have_api(self.user, self.key): return self.begin() self.query() diff --git a/modules/intelligence/threatbook_api.py b/modules/intelligence/threatbook_api.py index d50c5aa..237b4bc 100644 --- a/modules/intelligence/threatbook_api.py +++ b/modules/intelligence/threatbook_api.py @@ -26,7 +26,7 @@ class ThreatBookAPI(Query): """ 类执行入口 """ - if not self.check(self.key): + if not self.have_api(self.key): return self.begin() self.query() diff --git a/modules/intelligence/virustotal_api.py b/modules/intelligence/virustotal_api.py index 19ae9e5..8b3ba12 100644 --- a/modules/intelligence/virustotal_api.py +++ b/modules/intelligence/virustotal_api.py @@ -33,7 +33,7 @@ class VirusTotalAPI(Query): """ 类执行入口 """ - if not self.check(self.key): + if not self.have_api(self.key): return self.begin() self.query() diff --git a/modules/search/bing_api.py b/modules/search/bing_api.py index 9c6f787..54c6b51 100644 --- a/modules/search/bing_api.py +++ b/modules/search/bing_api.py @@ -45,7 +45,7 @@ class BingAPI(Search): """ 类执行入口 """ - if not self.check(self.id, self.key): + if not self.have_api(self.id, self.key): return self.begin() self.search(self.domain) diff --git a/modules/search/fofa_api.py b/modules/search/fofa_api.py index 4ce17fe..d97b7ef 100644 --- a/modules/search/fofa_api.py +++ b/modules/search/fofa_api.py @@ -50,7 +50,7 @@ class FoFa(Search): """ 类执行入口 """ - if not self.check(self.email, self.key): + if not self.have_api(self.email, self.key): return self.begin() self.search() diff --git a/modules/search/github_api.py b/modules/search/github_api.py index c71f2d9..cef1544 100644 --- a/modules/search/github_api.py +++ b/modules/search/github_api.py @@ -83,7 +83,7 @@ class GithubAPI(Search): """ 类执行入口 """ - if not self.check(self.token): + if not self.have_api(self.token): return self.begin() self.search() diff --git a/modules/search/google_api.py b/modules/search/google_api.py index 2027df7..042afaf 100644 --- a/modules/search/google_api.py +++ b/modules/search/google_api.py @@ -44,7 +44,7 @@ class GoogleAPI(Search): """ 类执行入口 """ - if not self.check(self.id, self.key): + if not self.have_api(self.id, self.key): return self.begin() self.search(self.domain) diff --git a/modules/search/shodan_api.py b/modules/search/shodan_api.py index e0ae022..e80cf2a 100644 --- a/modules/search/shodan_api.py +++ b/modules/search/shodan_api.py @@ -33,7 +33,7 @@ class ShodanAPI(Search): """ 类执行入口 """ - if not self.check(self.key): + if not self.have_api(self.key): return self.begin() self.search() diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index b95e10b..d63b93e 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -61,7 +61,7 @@ class ZoomEyeAPI(Search): """ 类执行入口 """ - if not self.check(self.user, self.pwd): + if not self.have_api(self.user, self.pwd): return self.begin() self.search() From 9d639008ea7f6b96b9ef692db5740025095863c4 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 15 Sep 2020 23:57:46 +0800 Subject: [PATCH 04/12] typos --- common/tablib/tablib.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/tablib/tablib.py b/common/tablib/tablib.py index e75e6b1..dd36063 100644 --- a/common/tablib/tablib.py +++ b/common/tablib/tablib.py @@ -143,10 +143,9 @@ class Dataset: if is_valid: return True - else: - if not safety: - raise InvalidDimensions - return False + if not safety: + raise InvalidDimensions + return False def _package(self, dicts=True, ordered=True): """Packages Dataset into lists of dictionaries for transmission.""" @@ -274,7 +273,8 @@ class Dataset: """ Export :class:`Dataset` object to `format`. - :param \\*\\*kwargs: (optional) custom configuration to the format `export_set`. + :param format: export format + :param kwargs: (optional) custom configuration to the format `export_set`. """ fmt = registry.get_format(format) if not hasattr(fmt, 'export_set'): From add9a864b2e88f36ff5f90df76f01bd0d3e65e0d Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:08:32 +0800 Subject: [PATCH 05/12] typos --- modules/datasets/cloudflare_api.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/datasets/cloudflare_api.py b/modules/datasets/cloudflare_api.py index 4c7a6f2..a1b0dc0 100644 --- a/modules/datasets/cloudflare_api.py +++ b/modules/datasets/cloudflare_api.py @@ -42,8 +42,7 @@ class CloudFlareAPI(Query): if zone_id: self.list_dns(zone_id) return - else: - return + return elif zones_resp.json()['success']: zone_id = self.create_zone(account_id) if zone_id: @@ -60,14 +59,12 @@ class CloudFlareAPI(Query): return def create_zone(self, account_id): - data = {"name": self.domain, "account": {"id": account_id}, "jump_start": True, - "type": "full"} - create_zone_resp = self.post( - self.addr + 'zones', json=data, check=False) + data = {"name": self.domain, "account": {"id": account_id}, + "jump_start": True, "type": "full"} + create_zone_resp = self.post(self.addr + 'zones', json=data, check=False) if not create_zone_resp: - logger.log('DEBUG', - f'{create_zone_resp.status_code} {create_zone_resp.text}') - return + logger.log('DEBUG', f'{create_zone_resp.status_code} {create_zone_resp.text}') + return False if create_zone_resp.json()['success']: return create_zone_resp.json()['result']['id'] else: From 899e243a3a5307e119e87ea9ef078e8d671237f4 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:23:34 +0800 Subject: [PATCH 06/12] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/datasets/robtex.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/datasets/robtex.py b/modules/datasets/robtex.py index 9e21bd2..5a93cd3 100644 --- a/modules/datasets/robtex.py +++ b/modules/datasets/robtex.py @@ -10,7 +10,6 @@ class Robtex(Query): self.domain = domain self.module = 'Dataset' self.source = "RobtexQuery" - self.addr = 'https://freeapi.robtex.com/pdns/' def query(self): """ @@ -18,17 +17,18 @@ class Robtex(Query): """ self.header = self.get_header() self.proxy = self.get_proxy(self.source) - url = self.addr + 'forward/' + self.domain + base_addr = 'https://freeapi.robtex.com/pdns/' + url = f'{base_addr}/forward/{self.domain}' resp = self.get(url) if not resp: return text_list = resp.text.splitlines() - text_json = list(map(lambda x: json.loads(x), text_list)) - for record in text_json: + for item in text_list: + record = json.loads(item) if record.get('rrtype') in ['A', 'AAAA']: time.sleep(self.delay) # Robtex有查询频率限制 ip = record.get('rrdata') - url = self.addr + 'reverse/' + ip + url = f'{base_addr}/reverse/{ip}' resp = self.get(url) self.subdomains = self.collect_subdomains(resp) @@ -55,4 +55,4 @@ def run(domain): if __name__ == '__main__': - run('example.com') + run('google.com') From a1631bf6fc04ed706fa13e075eff9705ea2fdbaf Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:28:23 +0800 Subject: [PATCH 07/12] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/search/zoomeye_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index d63b93e..afef3ee 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -23,16 +23,15 @@ class ZoomEyeAPI(Search): data = {'username': self.user, 'password': self.pwd} resp = self.post(url=url, json=data) if not resp: - logger.log('FATAL', f'{self.source} module login failed,' - f' can not get access token') - exit(1) + logger.log('ALERT', f'{self.source} module login failed') + return None data = resp.json() if resp.status_code == 200: logger.log('DEBUG', f'{self.source} module login success') return data.get('access_token') else: logger.log('ALERT', data.get('message')) - exit(1) + return None def search(self): """ @@ -40,6 +39,8 @@ class ZoomEyeAPI(Search): """ page_num = 1 access_token = self.login() + if not access_token: + return while True: time.sleep(self.delay) self.header = self.get_header() From c1ce4cbf3c994ca009e68670ad74a49ace7d0cbd Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:31:16 +0800 Subject: [PATCH 08/12] typos --- modules/search/github_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/search/github_api.py b/modules/search/github_api.py index cef1544..7e9b4f2 100644 --- a/modules/search/github_api.py +++ b/modules/search/github_api.py @@ -33,8 +33,7 @@ class GithubAPI(Search): msg = resp_json.get('message') logger.log('ERROR', msg) return False - else: - return True + return True def search(self): """ @@ -104,4 +103,4 @@ def run(domain): if __name__ == '__main__': - run('exmaple.com') + run('example.com') From aee53f572fe7a52da46c1c0bfab5eef076e34e44 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:33:01 +0800 Subject: [PATCH 09/12] typos --- common/domain.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/common/domain.py b/common/domain.py index 2654e00..49cfab8 100644 --- a/common/domain.py +++ b/common/domain.py @@ -23,8 +23,7 @@ class Domain(object): result = re.search(self.regexp, self.string, re.I) if result: return result.group() - else: - return None + return None def extract(self): """ @@ -43,8 +42,7 @@ class Domain(object): result = self.match() if result: return ext(result) - else: - return None + return None def registered(self): """ @@ -60,5 +58,4 @@ class Domain(object): result = self.extract() if result: return result.registered_domain - else: - return None + return None From 78a1f14c9198b0f510955c54d16c766eabf12a8c Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:40:11 +0800 Subject: [PATCH 10/12] typos --- common/module.py | 3 +-- common/tablib/format.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/common/module.py b/common/module.py index cc2a8fc..b04835b 100644 --- a/common/module.py +++ b/common/module.py @@ -202,8 +202,7 @@ class Module(object): if isinstance(headers, dict): self.header = headers return headers - else: - return self.header + return self.header def get_proxy(self, module): """ diff --git a/common/tablib/format.py b/common/tablib/format.py index 2646bf6..e567dff 100644 --- a/common/tablib/format.py +++ b/common/tablib/format.py @@ -9,7 +9,7 @@ from uuid import UUID from collections import OrderedDict -class Registry(object): +class Registry: _formats = OrderedDict() def register(self, key, format_or_path): @@ -47,7 +47,7 @@ Tablib - JSON Support """ -class JSONFormat(object): +class JSONFormat: title = 'json' extensions = ('json',) @@ -61,7 +61,7 @@ class JSONFormat(object): """ -class CSVFormat(object): +class CSVFormat: title = 'csv' extensions = ('csv',) From 685d9ff5d2e8cd30418983de7a8c8900390e6663 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:48:41 +0800 Subject: [PATCH 11/12] typos --- common/records.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/records.py b/common/records.py index 41cff66..a6455b1 100644 --- a/common/records.py +++ b/common/records.py @@ -10,7 +10,7 @@ from sqlalchemy import create_engine, exc, inspect, text DATABASE_URL = os.environ.get('DATABASE_URL') -def isexception(obj): +def is_exception(obj): """Given an object, return a boolean indicating whether it is an instance or subclass of :py:class:`Exception`. """ @@ -212,7 +212,7 @@ class RecordCollection(object): try: record = self[0] except IndexError: - if isexception(default): + if is_exception(default): raise default return default @@ -276,7 +276,7 @@ class Database(object): def __repr__(self): return ''.format(self.open) - def get_table_names(self, internal=False): + def get_table_names(self): """Returns a list of table names for the connected database.""" # Setup SQLAlchemy for Database inspection. From 30423e7bf54aa1965e8683966b7ffcc9679a08d0 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 16 Sep 2020 00:56:57 +0800 Subject: [PATCH 12/12] typos --- common/utils.py | 4 ++-- modules/banner.py | 3 +-- modules/iscdn.py | 5 ----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/common/utils.py b/common/utils.py index ad6f4b3..8a617e3 100644 --- a/common/utils.py +++ b/common/utils.py @@ -689,11 +689,11 @@ def match_subdomains(domain, html, distinct=True, fuzzy=True): def check_random_subdomain(subdomains): if not subdomains: logger.log('ALERT', f'The generated dictionary is empty') - return False + return for subdomain in subdomains: if subdomain: logger.log('ALERT', f'Please check whether {subdomain} is correct or not') - return True + return def get_url_resp(url): diff --git a/modules/banner.py b/modules/banner.py index 4d0757d..83464bc 100644 --- a/modules/banner.py +++ b/modules/banner.py @@ -43,7 +43,7 @@ class MultiIdentify(Module): pool = Pool(processes_num) rules = self.load_rules() _identify = Identify(rules) - for i in range(processes_num): + for _ in range(processes_num): pool.apply_async(func=_identify.run, args=(task_queue, done_queue)) pool.close() pool.join() @@ -116,7 +116,6 @@ class Identify(object): self.parse(item) banners = [] for name, rule in self.RULES.items(): - # print(name) r = self._check_rule(rule) if r: if 'implies' in rule: diff --git a/modules/iscdn.py b/modules/iscdn.py index 70ea532..10920ef 100644 --- a/modules/iscdn.py +++ b/modules/iscdn.py @@ -17,11 +17,6 @@ cdn_cname_keyword = utils.load_json(data_dir.joinpath('cdn_cname_keywords.json') cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json')) -def check_cname_count(cnames): - if len(cnames) > 1: - return True - - def check_cname_keyword(cname): names = cname.lower().split(',') for name in names: