Merge remote-tracking branch 'origin/master' into master

common/domain.py

@@ -23,7 +23,6 @@ class Domain(object):
         result = re.search(self.regexp, self.string, re.I)
         if result:
             return result.group()
-        else:
         return None
 
     def extract(self):
@@ -43,7 +42,6 @@ class Domain(object):
         result = self.match()
         if result:
             return ext(result)
-        else:
         return None
 
     def registered(self):
@@ -60,5 +58,4 @@ class Domain(object):
         result = self.extract()
         if result:
             return result.registered_domain
-        else:
         return None

common/module.py

@@ -33,7 +33,7 @@ class Module(object):
         self.end = None  # 模块结束执行时间
         self.elapse = None  # 模块执行耗时
 
-    def check(self, *apis):
+    def have_api(self, *apis):
         """
         Simply check whether the api information configure or not
 
@@ -95,7 +95,7 @@ class Module(object):
             return resp
         return None
 
-    def get(self, url, params=None, check=True, ignore=False,raise_error=False, **kwargs):
+    def get(self, url, params=None, check=True, ignore=False, raise_error=False, **kwargs):
         """
         Custom get request
 
@@ -171,7 +171,7 @@ class Module(object):
         :param str  url: request url
         :param bool check: check response
         :param kwargs: other params
-        :return: requests's response object
+        :return: response object
         """
         session = requests.Session()
         session.trust_env = False
@@ -202,7 +202,6 @@ class Module(object):
         if isinstance(headers, dict):
             self.header = headers
             return headers
-        else:
         return self.header
 
     def get_proxy(self, module):

common/records.py

@@ -10,7 +10,7 @@ from sqlalchemy import create_engine, exc, inspect, text
 DATABASE_URL = os.environ.get('DATABASE_URL')
 
 
-def isexception(obj):
+def is_exception(obj):
     """Given an object, return a boolean indicating whether it is an instance
     or subclass of :py:class:`Exception`.
     """
@@ -212,7 +212,7 @@ class RecordCollection(object):
         try:
             record = self[0]
         except IndexError:
-            if isexception(default):
+            if is_exception(default):
                 raise default
             return default
 
@@ -276,7 +276,7 @@ class Database(object):
     def __repr__(self):
         return '<Database open={}>'.format(self.open)
 
-    def get_table_names(self, internal=False):
+    def get_table_names(self):
         """Returns a list of table names for the connected database."""
 
         # Setup SQLAlchemy for Database inspection.

common/resolve.py

@@ -21,9 +21,9 @@ def filter_subdomain(data):
     """
     logger.log('DEBUG', f'Filtering subdomains to be resolved')
     subdomains = []
-    for data in data:
+    for infos in data:
-        if not data.get('ip'):
+        if not infos.get('ip'):
-            subdomain = data.get('subdomain')
+            subdomain = infos.get('subdomain')
             if subdomain:
                 subdomains.append(subdomain)
     return subdomains

common/tablib/format.py

@@ -9,7 +9,7 @@ from uuid import UUID
 from collections import OrderedDict
 
 
-class Registry(object):
+class Registry:
     _formats = OrderedDict()
 
     def register(self, key, format_or_path):
@@ -47,7 +47,7 @@ Tablib - JSON Support
 """
 
 
-class JSONFormat(object):
+class JSONFormat:
     title = 'json'
     extensions = ('json',)
 
@@ -61,7 +61,7 @@ class JSONFormat(object):
 """
 
 
-class CSVFormat(object):
+class CSVFormat:
     title = 'csv'
     extensions = ('csv',)
 

common/tablib/tablib.py

@@ -143,7 +143,6 @@ class Dataset:
 
         if is_valid:
             return True
-        else:
         if not safety:
             raise InvalidDimensions
         return False
@@ -274,7 +273,8 @@ class Dataset:
         """
         Export :class:`Dataset` object to `format`.
 
-        :param \\*\\*kwargs: (optional) custom configuration to the format `export_set`.
+        :param format: export format
+        :param kwargs: (optional) custom configuration to the format `export_set`.
         """
         fmt = registry.get_format(format)
         if not hasattr(fmt, 'export_set'):

common/utils.py

@@ -689,11 +689,11 @@ def match_subdomains(domain, html, distinct=True, fuzzy=True):
 def check_random_subdomain(subdomains):
     if not subdomains:
         logger.log('ALERT', f'The generated dictionary is empty')
-        return False
+        return
     for subdomain in subdomains:
         if subdomain:
             logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
-            return True
+            return
 
 
 def get_url_resp(url):

modules/banner.py

@@ -43,7 +43,7 @@ class MultiIdentify(Module):
         pool = Pool(processes_num)
         rules = self.load_rules()
         _identify = Identify(rules)
-        for i in range(processes_num):
+        for _ in range(processes_num):
             pool.apply_async(func=_identify.run, args=(task_queue, done_queue))
         pool.close()
         pool.join()
@@ -116,7 +116,6 @@ class Identify(object):
             self.parse(item)
             banners = []
             for name, rule in self.RULES.items():
-                # print(name)
                 r = self._check_rule(rule)
                 if r:
                     if 'implies' in rule:

modules/certificates/censys_api.py

@@ -45,7 +45,7 @@ class CensysAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.id, self.secret):
+        if not self.have_api(self.id, self.secret):
             return
         self.begin()
         self.query()

modules/datasets/binaryedge_api.py

@@ -26,7 +26,7 @@ class BinaryEdgeAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.api):
+        if not self.have_api(self.api):
             return
         self.begin()
         self.query()

modules/datasets/chinaz_api.py

@@ -25,7 +25,7 @@ class ChinazAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.api):
+        if not self.have_api(self.api):
             return
         self.begin()
         self.query()

modules/datasets/circl_api.py

@@ -25,7 +25,7 @@ class CirclAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.user, self.pwd):
+        if not self.have_api(self.user, self.pwd):
             return
         self.begin()
         self.query()

modules/datasets/cloudflare_api.py

@@ -42,7 +42,6 @@ class CloudFlareAPI(Query):
                     if zone_id:
                         self.list_dns(zone_id)
                         return
-                    else:
                     return
                 elif zones_resp.json()['success']:
                     zone_id = self.create_zone(account_id)
@@ -60,14 +59,12 @@ class CloudFlareAPI(Query):
         return
 
     def create_zone(self, account_id):
-        data = {"name": self.domain, "account": {"id": account_id}, "jump_start": True,
+        data = {"name": self.domain, "account": {"id": account_id},
-                "type": "full"}
+                "jump_start": True, "type": "full"}
-        create_zone_resp = self.post(
+        create_zone_resp = self.post(self.addr + 'zones', json=data, check=False)
-            self.addr + 'zones', json=data, check=False)
         if not create_zone_resp:
-            logger.log('DEBUG',
+            logger.log('DEBUG', f'{create_zone_resp.status_code} {create_zone_resp.text}')
-                       f'{create_zone_resp.status_code} {create_zone_resp.text}')
+            return False
-            return
         if create_zone_resp.json()['success']:
             return create_zone_resp.json()['result']['id']
         else:
@@ -109,7 +106,7 @@ class CloudFlareAPI(Query):
         """
         class entrance
         """
-        if not self.check(self.token):
+        if not self.have_api(self.token):
             return
         self.begin()
         self.query()

modules/datasets/dnsdb_api.py

@@ -26,7 +26,7 @@ class DNSdbAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.api):
+        if not self.have_api(self.api):
             return
         self.begin()
         self.query()

modules/datasets/ipv4info_api.py

@@ -49,7 +49,7 @@ class IPv4InfoAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.api):
+        if not self.have_api(self.api):
             return
         self.begin()
         self.query()

modules/datasets/passivedns_api.py

@@ -27,7 +27,7 @@ class PassiveDnsAPI(Query):
         类执行入口
         """
         if 'api.passivedns.cn' in self.addr:
-            if not self.check(self.token):
+            if not self.have_api(self.token):
                 return
         self.begin()
         self.query()

modules/datasets/robtex.py

@@ -10,7 +10,6 @@ class Robtex(Query):
         self.domain = domain
         self.module = 'Dataset'
         self.source = "RobtexQuery"
-        self.addr = 'https://freeapi.robtex.com/pdns/'
 
     def query(self):
         """
@@ -18,17 +17,18 @@ class Robtex(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        url = self.addr + 'forward/' + self.domain
+        base_addr = 'https://freeapi.robtex.com/pdns/'
+        url = f'{base_addr}/forward/{self.domain}'
         resp = self.get(url)
         if not resp:
             return
         text_list = resp.text.splitlines()
-        text_json = list(map(lambda x: json.loads(x), text_list))
+        for item in text_list:
-        for record in text_json:
+            record = json.loads(item)
             if record.get('rrtype') in ['A', 'AAAA']:
                 time.sleep(self.delay)  # Robtex有查询频率限制
                 ip = record.get('rrdata')
-                url = self.addr + 'reverse/' + ip
+                url = f'{base_addr}/reverse/{ip}'
                 resp = self.get(url)
                 self.subdomains = self.collect_subdomains(resp)
 
@@ -55,4 +55,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('google.com')

modules/datasets/securitytrails_api.py

@@ -32,7 +32,7 @@ class SecurityTrailsAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.api):
+        if not self.have_api(self.api):
             return
         self.begin()
         self.query()

modules/datasets/spyse_api.py

@@ -38,7 +38,7 @@ class SpyseAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.token):
+        if not self.have_api(self.token):
             return
         self.begin()
         self.query()

modules/intelligence/riskiq_api.py

@@ -33,7 +33,7 @@ class RiskIQ(Query):
         """
         类执行入口
         """
-        if not self.check(self.user, self.key):
+        if not self.have_api(self.user, self.key):
             return
         self.begin()
         self.query()

modules/intelligence/threatbook_api.py

@@ -26,7 +26,7 @@ class ThreatBookAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.key):
+        if not self.have_api(self.key):
             return
         self.begin()
         self.query()

modules/intelligence/virustotal_api.py

@@ -33,7 +33,7 @@ class VirusTotalAPI(Query):
         """
         类执行入口
         """
-        if not self.check(self.key):
+        if not self.have_api(self.key):
             return
         self.begin()
         self.query()

modules/iscdn.py

@@ -17,11 +17,6 @@ cdn_cname_keyword = utils.load_json(data_dir.joinpath('cdn_cname_keywords.json')
 cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
 
 
-def check_cname_count(cnames):
-    if len(cnames) > 1:
-        return True
-
-
 def check_cname_keyword(cname):
     names = cname.lower().split(',')
     for name in names:

modules/search/bing_api.py

@@ -45,7 +45,7 @@ class BingAPI(Search):
         """
         类执行入口
         """
-        if not self.check(self.id, self.key):
+        if not self.have_api(self.id, self.key):
             return
         self.begin()
         self.search(self.domain)

modules/search/fofa_api.py

@@ -50,7 +50,7 @@ class FoFa(Search):
         """
         类执行入口
         """
-        if not self.check(self.email, self.key):
+        if not self.have_api(self.email, self.key):
             return
         self.begin()
         self.search()

modules/search/github_api.py

@@ -33,7 +33,6 @@ class GithubAPI(Search):
             msg = resp_json.get('message')
             logger.log('ERROR', msg)
             return False
-        else:
         return True
 
     def search(self):
@@ -83,7 +82,7 @@ class GithubAPI(Search):
         """
         类执行入口
         """
-        if not self.check(self.token):
+        if not self.have_api(self.token):
             return
         self.begin()
         self.search()
@@ -104,4 +103,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('exmaple.com')
+    run('example.com')

modules/search/google_api.py

@@ -44,7 +44,7 @@ class GoogleAPI(Search):
         """
         类执行入口
         """
-        if not self.check(self.id, self.key):
+        if not self.have_api(self.id, self.key):
             return
         self.begin()
         self.search(self.domain)

modules/search/shodan_api.py

@@ -33,7 +33,7 @@ class ShodanAPI(Search):
         """
         类执行入口
         """
-        if not self.check(self.key):
+        if not self.have_api(self.key):
             return
         self.begin()
         self.search()

modules/search/zoomeye_api.py

@@ -23,16 +23,15 @@ class ZoomEyeAPI(Search):
         data = {'username': self.user, 'password': self.pwd}
         resp = self.post(url=url, json=data)
         if not resp:
-            logger.log('FATAL', f'{self.source} module login failed,'
+            logger.log('ALERT', f'{self.source} module login failed')
-                                f' can not get access token')
+            return None
-            exit(1)
         data = resp.json()
         if resp.status_code == 200:
             logger.log('DEBUG', f'{self.source} module login success')
             return data.get('access_token')
         else:
             logger.log('ALERT', data.get('message'))
-            exit(1)
+            return None
 
     def search(self):
         """
@@ -40,6 +39,8 @@ class ZoomEyeAPI(Search):
         """
         page_num = 1
         access_token = self.login()
+        if not access_token:
+            return
         while True:
             time.sleep(self.delay)
             self.header = self.get_header()
@@ -61,7 +62,7 @@ class ZoomEyeAPI(Search):
         """
         类执行入口
         """
-        if not self.check(self.user, self.pwd):
+        if not self.have_api(self.user, self.pwd):
             return
         self.begin()
         self.search()