From c2985352b1c23f8551ee407eda1e412c103695ed Mon Sep 17 00:00:00 2001 From: JrD <421273918@qq.com> Date: Mon, 7 Sep 2020 15:11:06 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Drequests=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=97=B6=E9=BB=98=E8=AE=A4=E4=BD=BF=E7=94=A8=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E4=BB=A3=E7=90=86=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/module.py | 56 ++++++++++++++++++++---------------- common/request.py | 2 +- common/utils.py | 12 ++++++-- modules/search/github_api.py | 1 + 4 files changed, 43 insertions(+), 28 deletions(-) diff --git a/common/module.py b/common/module.py index f851674..6a47a96 100644 --- a/common/module.py +++ b/common/module.py @@ -75,8 +75,10 @@ class Module(object): :param kwargs: other params :return: response object """ + session = requests.Session() + session.trust_env = False try: - resp = requests.head(url, + resp = session.head(url, params=params, cookies=self.cookie, headers=self.header, @@ -104,18 +106,20 @@ class Module(object): :param kwargs: other params :return: response object """ + session = requests.Session() + session.trust_env = False level = 'ERROR' if ignore: level = 'DEBUG' try: - resp = requests.get(url, - params=params, - cookies=self.cookie, - headers=self.header, - proxies=self.proxy, - timeout=self.timeout, - verify=self.verify, - **kwargs) + resp = session.get(url, + params=params, + cookies=self.cookie, + headers=self.header, + proxies=self.proxy, + timeout=self.timeout, + verify=self.verify, + **kwargs) except Exception as e: logger.log(level, e.args) return None @@ -135,15 +139,17 @@ class Module(object): :param kwargs: other params :return: response object """ + session = requests.Session() + session.trust_env = False try: - resp = requests.post(url, - data=data, - cookies=self.cookie, - headers=self.header, - proxies=self.proxy, - timeout=self.timeout, - verify=self.verify, - **kwargs) + resp = session.post(url, + data=data, + cookies=self.cookie, + headers=self.header, + proxies=self.proxy, + timeout=self.timeout, + verify=self.verify, + **kwargs) except Exception as e: logger.log('ERROR', e.args) return None @@ -162,14 +168,16 @@ class Module(object): :param kwargs: other params :return: requests's response object """ + session = requests.Session() + session.trust_env = False try: - resp = requests.delete(url, - cookies=self.cookie, - headers=self.header, - proxies=self.proxy, - timeout=self.timeout, - verify=self.verify, - **kwargs) + resp = session.delete(url, + cookies=self.cookie, + headers=self.header, + proxies=self.proxy, + timeout=self.timeout, + verify=self.verify, + **kwargs) except Exception as e: logger.log('ERROR', e.args) return None diff --git a/common/request.py b/common/request.py index 9a14c08..9705305 100644 --- a/common/request.py +++ b/common/request.py @@ -2,7 +2,6 @@ import json from threading import Thread from queue import Queue - import tqdm import requests from bs4 import BeautifulSoup @@ -161,6 +160,7 @@ def get_session(): verify = settings.request_ssl_verify redirect_limit = settings.request_redirect_limit session = requests.Session() + session.trust_env = False session.headers = header session.verify = verify session.max_redirects = redirect_limit diff --git a/common/utils.py b/common/utils.py index d9b9aec..ae8bb67 100644 --- a/common/utils.py +++ b/common/utils.py @@ -521,8 +521,10 @@ def check_net(): 'http://www.apple.com', 'http://microsoft.com'] url = random.choice(urls) logger.log('INFOR', f'Trying to access {url}') + session = requests.Session() + session.trust_env = False try: - rsp = requests.get(url) + rsp = session.get(url, proxies=get_proxy()) except Exception as e: logger.log('ERROR', e.args) logger.log('ALERT', 'Can not access Internet, retrying') @@ -565,8 +567,10 @@ def check_version(local): proxy = get_proxy() timeout = settings.request_timeout_second verify = settings.request_ssl_verify + session = requests.Session() + session.trust_env = False try: - resp = requests.get(url=api, headers=header, proxies=proxy, + resp = session.get(url=api, headers=header, proxies=proxy, timeout=timeout, verify=verify) resp_json = resp.json() latest = resp_json['tag_name'] @@ -705,8 +709,10 @@ def get_url_resp(url): logger.log('INFOR', f'Attempting to request {url}') timeout = settings.request_timeout_second verify = settings.request_ssl_verify + session = requests.Session() + session.trust_env = False try: - resp = requests.get(url, params=None, timeout=timeout, verify=verify) + resp = session.get(url, params=None, timeout=timeout, verify=verify) except Exception as e: logger.log('ALERT', f'Error request {url}') logger.log('DEBUG', e.args) diff --git a/modules/search/github_api.py b/modules/search/github_api.py index b862f2a..c71f2d9 100644 --- a/modules/search/github_api.py +++ b/modules/search/github_api.py @@ -12,6 +12,7 @@ class GithubAPI(Search): self.addr = 'https://api.github.com/search/code' self.domain = domain self.session = requests.Session() + self.session.trust_env = False self.auth_url = 'https://api.github.com' self.token = settings.github_api_token