Merge branch 'master' of https://github.com/shmilylty/OneForAll into master

common/module.py

@@ -87,7 +87,7 @@ class Module(object):
                                 verify=self.verify,
                                 **kwargs)
         except Exception as e:
-            logger.log('ERROR', e.args)
+            logger.log('ERROR', e.args[0])
             return None
         if not check:
             return resp
@@ -124,9 +124,9 @@ class Module(object):
         except Exception as e:
             if raise_error:
                 if isinstance(e, requests.exceptions.ConnectTimeout):
-                    logger.log(level, e.args)
+                    logger.log(level, e.args[0])
                     raise e
-            logger.log(level, e.args)
+            logger.log(level, e.args[0])
             return None
         if not check:
             return resp
@@ -156,7 +156,7 @@ class Module(object):
                                 verify=self.verify,
                                 **kwargs)
         except Exception as e:
-            logger.log('ERROR', e.args)
+            logger.log('ERROR', e.args[0])
             return None
         if not check:
             return resp
@@ -184,7 +184,7 @@ class Module(object):
                                   verify=self.verify,
                                   **kwargs)
         except Exception as e:
-            logger.log('ERROR', e.args)
+            logger.log('ERROR', e.args[0])
             return None
         if not check:
             return resp

data/cdn_header_keys.json

@@ -2,6 +2,8 @@
     "xcs",
     "via",
     "x-via",
+    "x-cdn",
+    "x-cdn-forward",
     "x-ser",
     "x-cf1",
     "cache",
@@ -11,6 +13,7 @@
     "x-hit-cache",
     "x-cache-status",
     "x-cache-hits",
+    "x-cache-lookup",
     "cc_cache",
     "webcache",
     "chinacache",
@@ -20,14 +23,20 @@
     "x-github-request-id",
     "x-sucuri-id",
     "x-amz-cf-id",
+    "x-airee-node",
     "x-cdn-provider",
     "x-fastly",
+    "x-iinfo",
+    "x-llid",
+    "sozu-id",
+    "x-cf-tsc",
     "x-ws-request-id",
     "fss-cache",
     "powered-by-chinacache",
     "verycdn",
     "yunjiasu",
     "skyparkcdn",
+    "x-beluga-cache-status",
     "x-content-type-options",
     "x-download-options",
     "x-proxy-node",
@@ -36,5 +45,6 @@
     "etag",
     "expires",
     "pragma",
-    "cache-control"
+    "cache-control",
+    "last-modified"
 ]

modules/intelligence/riskiq_api.py

@@ -25,8 +25,8 @@ class RiskIQ(Query):
         if not resp:
             return
         data = resp.json()
-        subnames = data.get('subdomains')
-        subdomain_str = str(set(map(lambda subname: f'{subname}.{self.domain}', subnames)))
+        names = data.get('subdomains')
+        subdomain_str = str(set(map(lambda name: f'{name}.{self.domain}', names)))
         self.subdomains = self.collect_subdomains(subdomain_str)
 
     def run(self):

modules/search/shodan_api.py

@@ -8,7 +8,6 @@ class ShodanAPI(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'ShodanAPISearch'
-        self.addr = 'https://api.shodan.io/shodan/host/search'
         self.key = settings.shodan_api_key
 
     def search(self):
@@ -17,17 +16,14 @@ class ShodanAPI(Search):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        query = 'hostname:.' + self.domain
-        page = 1
-        while True:
-            params = {'key': self.key, 'page': page, 'query': query,
-                      'minify': True, 'facets': {'hostnames'}}
-            resp = self.get(self.addr, params)
-            subdomains = self.match_subdomains(resp)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
-                break
-            self.subdomains.update(subdomains)
-            page += 1
+        url = f'https://api.shodan.io/dns/domain/{self.domain}?key={self.key}'
+        resp = self.get(url)
+        if not resp:
+            return
+        data = resp.json()
+        names = data.get('subdomains')
+        subdomain_str = str(set(map(lambda name: f'{name}.{self.domain}', names)))
+        self.subdomains = self.collect_subdomains(subdomain_str)
 
     def run(self):
         """
@@ -54,4 +50,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('freebuf.com')