# Make a regex for redirect(word), no quotes or anything.
# Run it on every file, see what it outputs, the ones that it is successful for we run PyT on.
# redirect(url_for('helpTopics'))
import csv
import os
import re
import subprocess
from collections import namedtuple
# Part 1, make regex
# test_strings = [
# "redirect(url_for('helpTopics'))", # 1
# "redirect(hey)", # 2
# "redirect", # 3
# "hhredirect", # 4
# "return redirect(foo)", # 5
# "redirect(", # 6
# "redirect()", # 7
# "return redirect(request.GET.get('redirect',
# '/taskManager/'))" # 8
# ]
# for i, string in enumerate(test_strings):
# match = re.match(".*redirect\([a-zA-Z0-9_]+\)", string)
# if match:
# print(str(i + 1)+" was a match.")
# request_as_arg = re.match(".*redirect\(request.*\)", string)
# if request_as_arg:
# print(str(i + 1)+" was a match to request_as_arg.")
first_regex_vulns = set()
second_regex_vulns = set()
def test_match_with_regex(string, file_name):
global first_regex_vulns
global second_regex_vulns
if string.strip().startswith('#'):
return False
variable_as_arg = re.match(".*redirect\([a-zA-Z0-9_]+\).*", string)
if variable_as_arg:
print(string + " was a match to variable_as_arg.")
first_regex_vulns.add(
VulnerabilityConfig(
file_name=file_name,
line_matching_regex=string.strip()
)
)
return True
request_as_arg = re.match(".*redirect\(request.*\).*", string)
if request_as_arg:
# Maybe some guy on Hackerone would argue it was a vuln, but it's much lamer.
just_the_referrer = re.match(".*redirect\(request\.referrer\).*", string)
if just_the_referrer:
return False
just_the_url = re.match(".*redirect\(request\.url\).*", string)
if just_the_url:
return False
print(string + " was a match to request_as_arg.")
second_regex_vulns.add(
VulnerabilityConfig(
file_name=file_name,
line_matching_regex=string.strip()
)
)
return True
return False
# Part 2, grab repos and run the regex on them.
# https://github.com/mattmakai/choose-your-own-adventure-presentations.git, /cyoa/views.py
def get_repo_name(url):
"""Obtains the repo name repo URL.
This allows for local file saving, as compared to the URL, which indicates WHERE to clone from.
:type url: string
"""
# e.g. 'git@github.com:pre-commit/pre-commit-hooks' -> pre-commit-hooks
name = url.split('/')[-1]
# The url_or_path will still work without the `.git` suffix.
if name.endswith('.git'):
return name[:-4]
return name
RepoConfig = namedtuple(
'RepoConfig',
[
'repo_url',
'controller_file'
]
)
VulnerabilityConfig = namedtuple(
'VulnerabilityConfig',
[
'file_name',
'line_matching_regex'
]
)
repo_config = RepoConfig(
"https://github.com/mattmakai/choose-your-own-adventure-presentations.git",
"/cyoa/views.py"
)
def clone_from_github(repo_config):
import os
import signal
from subprocess import Popen, PIPE, TimeoutExpired
from time import monotonic as timer
start = timer()
with Popen('git clone '+repo_config.repo_url, shell=True, stdout=PIPE, preexec_fn=os.setsid) as process:
try:
output = process.communicate(timeout=1)[0]
except TimeoutExpired:
os.killpg(process.pid, signal.SIGINT) # send signal to the process group
output = process.communicate()[0]
except subprocess.CalledProcessError as e:
error_msg = e.output.decode('ascii')
# Ignore this message, because it's expected if the repo has already been cloned.
match = re.match(r"fatal: destination path '[^']+' already exists", error_msg)
if not match:
raise
print("Repo destination already existed.")
print('Elapsed seconds: {:.2f}'.format(timer() - start))
# # Clone from Github
# try:
# subprocess.check_output([
# 'git',
# 'clone',
# repo_config.repo_url,
# ],
# stderr=subprocess.STDOUT,
# timeout=2)
file_not_found_error_count = 0
boom_count = 0
possible_files = set()
unique_files = set()
possible_lines_first = set()
possible_lines_second = set()
def run_regex_on_repo_config(repo_config):
# global vars
global boom_count
global file_not_found_error_count
global possible_files
global unique_files
folder_name = get_repo_name(repo_config.repo_url)
# The csv has some files that start with a slash, so we need to remove it.
# path_to_file = os.path.join(folder_name, repo_config.controller_file)
path_to_file = repo_config.controller_file
if path_to_file in unique_files:
return
unique_files.add(path_to_file)
try:
with open(path_to_file) as foo:
print("opened ", path_to_file)
old_boom_count = boom_count
for line in foo:
if test_match_with_regex(line, file_name=path_to_file):
print(path_to_file + " matches the regexes")
print("Boom.")
possible_files.add(path_to_file)
boom_count = boom_count + 1
break
if old_boom_count == boom_count:
print(path_to_file + " does not match the regexes")
except FileNotFoundError:
file_not_found_error_count = file_not_found_error_count + 1
print("FileNotFoundError on ", path_to_file)
# See if the file matches the regex
# Print Success.
# Open flask_open_source_apps.csv,
# # BEGIN REGEX CODE
# FLASK_APPS_CSV = 'flask_open_source_apps.csv'
# flask_csv_reader = csv.reader(open(FLASK_APPS_CSV), delimiter=",")
# row_count = 0
# for row in flask_csv_reader:
# row_count = row_count + 1
# path_to_file = row[1].strip()
# repo_config = RepoConfig(
# repo_url=row[0].strip(),
# controller_file=path_to_file[1:] if path_to_file.startswith('/') else path_to_file
# )
# print("repo_config is ", repo_config)
# clone_from_github(repo_config)
# run_regex_on_repo_config(repo_config)
# print("file_not_found_error_count is", str(file_not_found_error_count))
# print("row_count is", str(row_count))
# print("boom_count is", str(boom_count))
# print("len(unique_files) is ", len(unique_files))
# print("len(possible_files) is ", len(possible_files))
# # End real code
# # print("possible_lines_first is ", possible_lines_first)
# for first_regex_vuln in first_regex_vulns:
# print("first_regex_vuln is", first_regex_vuln)
# for second_regex_vuln in second_regex_vulns:
# print("second_regex_vuln is", second_regex_vuln)
# # print("possible_lines_second is ", possible_lines_second)
# # END REGEX CODE
# I could generate a new CSV?
# Let me see where the current one gets us on boom_count
# So it gets us nowhere.
# FileNotFoundError on flaskRestCrud/flaskRestCrud/project/__init__.py
# repo_config is RepoConfig(repo_url='https://github.com/mustafawm/Flask-LocationApp', controller_file='Flask-LocationApp/routes.py')
# Cloning into 'Flask-LocationApp'...
# remote: Counting objects: 49, done.
# remote: Total 49 (delta 0), reused 0 (delta 0), pack-reused 49
# Unpacking objects: 100% (49/49), done.
# Checking connectivity... done.
# Elapsed seconds: 0.44
# FileNotFoundError on Flask-LocationApp/Flask-LocationApp/routes.py
# repo_config is RepoConfig(repo_url='https://github.com/Original-heapsters/FlaskPortal', controller_file='FlaskPortal/Portal_Main/app.py')
# Cloning into 'FlaskPortal'...
# remote: Counting objects: 253, done.
# remote: Total 253 (delta 0), reused 0 (delta 0), pack-reused 253
# Receiving objects: 100% (253/253), 186.70 KiB | 0 bytes/s, done.
# Resolving deltas: 100% (121/121), done.
# Checking connectivity... done.
# Elapsed seconds: 0.40
# FileNotFoundError on FlaskPortal/FlaskPortal/Portal_Main/app.py
# repo_config is RepoConfig(repo_url='https://github.com/neilmaldy/flask_upload', controller_file='flask_upload/test.py')
# Cloning into 'flask_upload'...
# remote: Counting objects: 20, done.
# remote: Total 20 (delta 0), reused 0 (delta 0), pack-reused 20
# Unpacking objects: 100% (20/20), done.
# Checking connectivity... done.
# Elapsed seconds: 0.35
# FileNotFoundError on flask_upload/flask_upload/test.py
# file_not_found_error_count is 728
# row_count is 728
# path_to_file = 'Flask_SQLite/draw_member.py'
# with open(path_to_file) as foo:
# print("opened ", path_to_file)
# old_boom_count = boom_count
# for line in foo:
# print("line is ", line)
# if test_match_with_regex(line):
# print(path_to_file + " matches the regexes")
# print("Boom.")
# boom_count = boom_count + 1
# break
# if old_boom_count == boom_count:
# print(path_to_file + " does not match the regexes")
# file_not_found_error_count is 79
# row_count is 812
# boom_count is 34
# So how what was it 34/(812-79)
# # 4.6 percent matched the regexess
# # 34/733
# len(unique_files) is 20/(547-66) -> 20/481
# len(possible_files) is 20
# ~4.1 percent matched the regexes
# possible_files is {'simple-web-proxy/app.py',
# 'multunus-puzzle/src/app.py',
# 'FLASKHW/directory.py',
# 'flask-pastebin/pastebin.py',
# 'python-indieweb/indieweb.py',
# 'flask-oauthlib/flask_oauthlib/provider/oauth1.py',
# 'flaskoktaapp/flaskoktaapp/__init__.py',
# 'flask_shortener/app.py',
# 'flask_upload/test.py',
# 'twitter/hello.py',
# 'rdflib-web/rdflib_web/lod.py',
# 'cheapskate/cheapskate.py',
# 'honest_site/run.py',
# 'okta-pysaml2-example/app.py',
# 'pandaflask_old/pandachrome.py',
# 'oauth-flask-template/auth.py',
# 'Flaskly/flaskly.py',
# 'hb2_flask/hb2_flask.py',
# 'social_project_flask/app.py',
# 'Flask_SQLite/draw_member.py',
# 'python-bookmark-service/app.py',
# 'Flask_OAuth2/app.py',
# 'cs125-fooddy-flask/fooddy2.py',
# 'examples-flask/example_basic.py'
# }
# Line matching regex 1 is return redirect(foo)
# Line matching regex 1 is return redirect(UPLOAD_ONION_URL)
# Line matching regex 1 is return redirect(target)
# Line matching regex 1 is return redirect(uri)
# Line matching regex 1 is return redirect(authorization_url)
# Line matching regex 1 is return redirect(link)
# Line matching regex 1 is return redirect(redirect_to)
# Line matching regex 1 is return redirect(url)
# Line matching regex 1 is return redirect(url)
# Line matching regex 1 is return redirect(link_target)
# Line matching regex 1 is return redirect(uri)
# Line matching regex 1 is return redirect(next_url)
# Line matching regex 1 is return redirect(url)
# Line matching regex 1 is return redirect(next)
# Line matching regex 2 is return redirect(request.url)
# Line matching regex 2 is return redirect(request.referrer or url_for('index'))
# Line matching regex 2 is return(redirect(request.referrer))
# Line matching regex 2 is # redirect(request.args.get('next') or url_for('index')) #allows login page to act as in between
# Line matching regex 2 is return redirect(request.referrer)
# Line matching regex 2 is return redirect(request.form["next"])
# Line matching regex 2 is return redirect(request.args.get("next") or url_for("index"))
# Finally I was left with the output:
# first_regex_vulns
# file_name='flaskoktaapp/flaskoktaapp/__init__.py',
# line_matching_regex='return redirect(url)'
# file_name='flask-pastebin/pastebin.py',
# line_matching_regex='return redirect(next_url)'
# file_name='flask_shortener/app.py',
# line_matching_regex='return redirect(link_target)'
# file_name='oauth-flask-template/auth.py',
# line_matching_regex='return redirect(next)'
# file_name='social_project_flask/app.py',
# line_matching_regex='return redirect(next_url)'
# file_name='flask-oauthlib/flask_oauthlib/provider/oauth1.py',
# line_matching_regex='return redirect(uri)'
# file_name='cs125-fooddy-flask/fooddy2.py',
# line_matching_regex='return redirect(auth_uri)')
# file_name='python-indieweb/indieweb.py',
# line_matching_regex='return redirect(url)'
# file_name='multunus-puzzle/src/app.py',
# line_matching_regex='return redirect(redirect_to)'
# file_name='hb2_flask/hb2_flask.py',
# line_matching_regex='return redirect(target)'
# file_name='Flask_OAuth2/app.py',
# line_matching_regex='return redirect(uri)'
# file_name='examples-flask/example_basic.py',
# line_matching_regex='return redirect(authorization_url)'
# file_name='Flaskly/flaskly.py',
# line_matching_regex='return redirect(link)'
# file_name='honest_site/run.py',
# line_matching_regex='return redirect(UPLOAD_ONION_URL)'
# file_name='cheapskate/cheapskate.py',
# line_matching_regex='return redirect(url)'
# file_name='okta-pysaml2-example/app.py',
# line_matching_regex='return redirect(url)'
# file_name='python-bookmark-service/app.py',
# line_matching_regex='return redirect(url)'
# second_regex_vulns
# file_name='simple-web-proxy/app.py',
# line_matching_regex='return redirect(request.args.get("next") or url_for("index"))'
# file_name='twitter/hello.py',
# line_matching_regex="return redirect(request.referrer or url_for('index'))"
# file_name='pandaflask_old/pandachrome.py',
# line_matching_regex='return redirect(request.form["next"])'
# 17 matching the first regex (5 'url', 2 'next_url', 2 'uri' and others)
# We cannot tell if they are vulnerable from looking at them.
# 3 matching the second_regex
interesting_files = set([
'flaskoktaapp/flaskoktaapp/__init__.py',
'flask-pastebin/pastebin.py',
'flask_shortener/app.py',
'oauth-flask-template/auth.py',
'social_project_flask/app.py',
'flask-oauthlib/flask_oauthlib/provider/oauth1.py',
'cs125-fooddy-flask/fooddy2.py',
'python-indieweb/indieweb.py',
'multunus-puzzle/src/app.py',
'hb2_flask/hb2_flask.py',
'Flask_OAuth2/app.py',
'examples-flask/example_basic.py',
'Flaskly/flaskly.py',
'honest_site/run.py',
'cheapskate/cheapskate.py',
'okta-pysaml2-example/app.py',
'python-bookmark-service/app.py',
'simple-web-proxy/app.py',
'twitter/hello.py',
'pandaflask_old/pandachrome.py'
])
print("len of interesting_files is ", len(interesting_files))
for file in interesting_files:
print('file is ', file)
flaskoktaapp/flaskoktaapp/__init__.py
# Ret
# Reports 0 vulns, (with edited flask trigger words)
# https://github.com/gene1wood/flaskoktaapp/blob/master/flaskoktaapp/__init__.py#L204
# although we don't currently have a Post-only Flask option.
#1 of 20, GOOD: true negative, not reported.
#2 of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,
# of 20,