mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 04:47:48 +08:00
Jing Ling
133 lines
4.0 KiB
Python
133 lines
4.0 KiB
Python
import re
|
|
from urllib import parse
|
|
|
|
from common import utils
|
|
from common import request
|
|
from common.module import Module
|
|
from config import setting
|
|
|
|
|
|
class Finder(Module):
|
|
def __init__(self, domain):
|
|
Module.__init__(self)
|
|
self.module = 'Finder'
|
|
self.source = ''
|
|
self.domain = domain
|
|
self.ts = utils.get_timestring()
|
|
self.name = f'found_subdomains_{self.domain}_{self.ts}'
|
|
self.path = setting.temp_save_dir.joinpath(self.name)
|
|
|
|
def save_subdomain(self, data, path):
|
|
if not path:
|
|
path = self.path
|
|
utils.save_data(path, data)
|
|
|
|
def remove_subdomain(self, path):
|
|
if not path:
|
|
path = self.path
|
|
utils.remove_data(path)
|
|
|
|
def find_url(self):
|
|
pass
|
|
|
|
def batch_find_url(self):
|
|
pass
|
|
|
|
def find_subdomain(self, html):
|
|
subdomains = self.match_subdomains(html, fuzzy=False)
|
|
self.subdomains.add(subdomains)
|
|
|
|
def find_js(self):
|
|
pass
|
|
|
|
def filter_js(self):
|
|
pass
|
|
|
|
|
|
# Regular expression comes from https://github.com/GerbenJavado/LinkFinder
|
|
def find_url(html):
|
|
pattern_raw = r"""
|
|
(?:"|') # Start newline delimiter
|
|
(
|
|
((?:[a-zA-Z]{1,10}://|//) # Match a scheme [a-Z]*1-10 or //
|
|
[^"'/]{1,}\. # Match a domain name (any character + dot)
|
|
[a-zA-Z]{2,}[^"']{0,}) # The domain extension and/or path
|
|
|
|
|
((?:/|\.\./|\./) # Start with /,../,./
|
|
[^"'><,;| *()(%%$^/\\\[\]] # Next character can't be...
|
|
[^"'><,;|()]{1,}) # Rest of the characters can't be
|
|
|
|
|
([a-zA-Z0-9_\-/]{1,}/ # Relative endpoint with /
|
|
[a-zA-Z0-9_\-/]{1,} # Resource name
|
|
\.(?:[a-zA-Z]{1,4}|action) # Rest + extension (length 1-4 or action)
|
|
(?:[\?|/][^"|']{0,}|)) # ? mark with parameters
|
|
|
|
|
([a-zA-Z0-9_\-]{1,} # filename
|
|
\.(?:js) # . + extension
|
|
(?:\?[^"|']{0,}|)) # ? mark with parameters
|
|
)
|
|
(?:"|') # End newline delimiter
|
|
"""
|
|
pattern = re.compile(pattern_raw, re.VERBOSE)
|
|
result = re.finditer(pattern, html)
|
|
if result is None:
|
|
return None
|
|
urls = set()
|
|
for match in result:
|
|
url = match.group().strip('"').strip("'")
|
|
urls.add(url)
|
|
return urls
|
|
|
|
|
|
def process_url(req_url, rel_url):
|
|
black_url = ["javascript:"] # Add some keyword for filter url.
|
|
raw_url = parse.urlparse(req_url)
|
|
netloc = raw_url.netloc
|
|
scheme = raw_url.scheme
|
|
if rel_url[0:2] == "//":
|
|
result = scheme + ":" + rel_url
|
|
elif rel_url[0:4] == "http":
|
|
result = rel_url
|
|
elif rel_url[0:2] != "//" and rel_url not in black_url:
|
|
if rel_url[0:1] == "/":
|
|
result = scheme + "://" + netloc + rel_url
|
|
else:
|
|
if rel_url[0:1] == ".":
|
|
if rel_url[0:2] == "..":
|
|
result = scheme + "://" + netloc + rel_url[2:]
|
|
else:
|
|
result = scheme + "://" + netloc + rel_url[1:]
|
|
else:
|
|
result = scheme + "://" + netloc + "/" + rel_url
|
|
else:
|
|
result = req_url
|
|
return result
|
|
|
|
|
|
def filter_url(urs):
|
|
pass
|
|
|
|
|
|
def run_finder(domain, data):
|
|
existing_subdomains = set(map(lambda x: x.get('subdomain'), data)) # 已有的子域
|
|
found_subdomains = set()
|
|
found_urls = set()
|
|
finder = Finder(domain)
|
|
for item in data:
|
|
req_url = item.get('url')
|
|
rsp_html = item.get('response')
|
|
found_subdomains.add(finder.match_subdomains(rsp_html))
|
|
urls = find_url(rsp_html)
|
|
if not urls:
|
|
continue
|
|
for rel_url in urls:
|
|
found_url = process_url(req_url, rel_url)
|
|
found_urls.add(found_url)
|
|
|
|
resp_data = request.urls_request(found_urls)
|
|
new_subdomains = found_subdomains - existing_subdomains
|
|
|
|
|
|
def save_db(domain, data):
|
|
pass
|