river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 12:57:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加超时

Browse Source
This commit is contained in:
shmilylty
2019-08-18 19:29:06 +08:00
parent 39b92b79d9
commit 61075f40ac
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
oneforall/modules/datasets/dnsdb.py
+7 -3
View File
@@ -24,8 +24,9 @@ class DNSdb(Query):
scraper = cloudscraper.create_scraper() scraper = cloudscraper.create_scraper()
scraper.interpreter = 'js2py' scraper.interpreter = 'js2py'
scraper.proxies = self.get_proxy(self.source) scraper.proxies = self.get_proxy(self.source)
scraper.timeout = 10
try: try:
tokens = scraper.get_tokens(self.url, timeout=self.timeout) tokens = scraper.get_tokens(self.url)
except Exception as e: except Exception as e:
logger.log('ERROR', e.args) logger.log('ERROR', e.args)
return None return None
@@ -43,13 +44,16 @@ class DNSdb(Query):
return False return False
self.cookie = tokens[0] self.cookie = tokens[0]
self.header = {'User-Agent': tokens[1]} self.header = {'User-Agent': tokens[1]}
self.timeout = 10
resp = self.get(self.url) resp = self.get(self.url)
if not resp: if not resp:
return return
if 'index' in resp.text: if 'index' in resp.text:
soup = BeautifulSoup(resp.text, features='lxml') soup = BeautifulSoup(resp.text, features='lxml')
urls = set(map(lambda x: self.addr + self.domain + x.text, base = self.addr+self.domain
soup.find_all('a'))) urls = list(map(lambda a: base + '/' + a.get('href'),
soup.find_all('a')))
urls = urls[:-1] # idn域名暂时不考虑
for url in urls: for url in urls:
resp = self.get(url) resp = self.get(url)
if not resp: if not resp: