s0md3v/Bolt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

code formatting and improved crawling

Browse Source
This commit is contained in:
Somdev Sangwan
2019-04-25 01:40:14 +05:30
committed by GitHub
parent 412e95753e 92c7e10c87
commit 734ea73705
13 changed files with 381 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
bolt.py
View File

@@ -2,11 +2,13 @@ from core.colors import green, yellow, end, run, good, info, bad, white, red
lightning = '\033[93;5m⚡\033[0m'
def banner():
print ('''
%s⚡ %sBOLT%s ⚡%s
''' % (yellow, white, yellow, end))
banner()
try:
@@ -43,9 +45,12 @@ parser = argparse.ArgumentParser()
parser.add_argument('-u', help='target url', dest='target')
parser.add_argument('-t', help='number of threads', dest='threads', type=int)
parser.add_argument('-l', help='levels to crawl', dest='level', type=int)
parser.add_argument('--delay', help='delay between requests', dest='delay', type=int)
parser.add_argument('--timeout', help='http request timeout', dest='timeout', type=int)
parser.add_argument('--headers', help='http headers', dest='add_headers', nargs='?', const=True)
parser.add_argument('--delay', help='delay between requests',
dest='delay', type=int)
parser.add_argument('--timeout', help='http request timeout',
dest='timeout', type=int)
parser.add_argument('--headers', help='http headers',
dest='add_headers', nargs='?', const=True)
args = parser.parse_args()
if not args.target:
@@ -70,11 +75,14 @@ weakTokens = []
tokenDatabase = []
insecureForms = []
print (' %s Phase: Crawling %s[%s1/6%s]%s' % (lightning, green, end, green, end))
print (' %s Phase: Crawling %s[%s1/6%s]%s' %
(lightning, green, end, green, end))
dataset = photon(target, headers, level, threadCount)
allForms = dataset[0]
print ('\r%s Crawled %i URL(s) and found %i form(s).%-10s' % (info, dataset[1], len(allForms), ' '))
print (' %s Phase: Evaluating %s[%s2/6%s]%s' % (lightning, green, end, green, end))
print ('\r%s Crawled %i URL(s) and found %i form(s).%-10s' %
(info, dataset[1], len(allForms), ' '))
print (' %s Phase: Evaluating %s[%s2/6%s]%s' %
(lightning, green, end, green, end))
evaluate(allForms, weakTokens, tokenDatabase, allTokens, insecureForms)
@@ -92,9 +100,11 @@ if insecureForms:
action = list(insecureForm.values())[0]['action']
form = action.replace(target, '')
if form:
print ('%s %s %s[%s%s%s]%s' % (bad, url, green, end, form, green, end))
print ('%s %s %s[%s%s%s]%s' %
(bad, url, green, end, form, green, end))
print (' %s Phase: Comparing %s[%s3/6%s]%s' % (lightning, green, end, green, end))
print (' %s Phase: Comparing %s[%s3/6%s]%s' %
(lightning, green, end, green, end))
uniqueTokens = set(allTokens)
if len(uniqueTokens) < len(allTokens):
print ('%s Potential Replay Attack condition found' % good)
@@ -103,7 +113,8 @@ if len(uniqueTokens) < len(allTokens):
for url, token in tokenDatabase:
for url2, token2 in tokenDatabase:
if token == token2 and url != url2:
print ('%s The same token was used on %s%s%s and %s%s%s' % (good, green, url, end, green, url2, end))
print ('%s The same token was used on %s%s%s and %s%s%s' %
(good, green, url, end, green, url2, end))
replay = True
if not replay:
print ('%s Further investigation shows that it was a false positive.')
@@ -127,6 +138,7 @@ if matches:
for name in matches:
print (' %s>%s %s' % (yellow, end, name))
def fuzzy(tokens):
averages = []
for token in tokens:
@@ -143,13 +155,16 @@ def fuzzy(tokens):
averages.append(average)
return statistics.mean(averages)
try:
similarity = fuzzy(allTokens)
print ('%s Tokens are %s%i%%%s similar to each other on an average' % (info, green, similarity, end))
print ('%s Tokens are %s%i%%%s similar to each other on an average' %
(info, green, similarity, end))
except statistics.StatisticsError:
print ('%s No CSRF protection to test' % bad)
quit()
def staticParts(allTokens):
strings = list(set(allTokens.copy()))
commonSubstrings = {}
@@ -165,6 +180,8 @@ def staticParts(allTokens):
if string not in commonSubstrings[commonSubstring]:
commonSubstrings[commonSubstring].append(string)
return commonSubstrings
result = {k: v for k, v in staticParts(allTokens).items() if v}
if result:
@@ -173,9 +190,11 @@ if result:
simTokens = []
print (' %s Phase: Observing %s[%s4/6%s]%s' % (lightning, green, end, green, end))
print (' %s Phase: Observing %s[%s4/6%s]%s' %
(lightning, green, end, green, end))
print ('%s 100 simultaneous requests are being made, please wait.' % info)
def extractForms(url):
response = requester(url, {}, headers, True, 0).text
forms = zetanize(url, response)
@@ -188,6 +207,7 @@ def extractForms(url):
if strength(value) > 10:
simTokens.append(value)
while True:
sample = random.choice(tokenDatabase)
goodToken = list(sample.values())[0]
@@ -196,7 +216,8 @@ while True:
break
threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=30)
futures = (threadpool.submit(extractForms, goodCandidate) for goodCandidate in [goodCandidate] * 30)
futures = (threadpool.submit(extractForms, goodCandidate)
for goodCandidate in [goodCandidate] * 30)
for i in concurrent.futures.as_completed(futures):
pass
@@ -208,7 +229,8 @@ if simTokens:
else:
print ('%s Different tokens were issued for simultaneous requests.' % info)
print (' %s Phase: Testing %s[%s5/6%s]%s' % (lightning, green, end, green, end))
print (' %s Phase: Testing %s[%s5/6%s]%s' %
(lightning, green, end, green, end))
parsed = ''
print ('%s Finding a suitable form for further testing. It may take a while.' % run)
@@ -298,7 +320,8 @@ for index in range(len(allTokens[0])):
else:
difference = abs(originalLength - len(response.text))
if difference <= tolerableDifference:
print ('%s Last %i chars of token aren\'t being checked' % (good, index + 1))
print ('%s Last %i chars of token aren\'t being checked' %
(good, index + 1))
else:
break
@@ -318,7 +341,8 @@ if response.status_code == originalCode:
else:
print ('%s It didn\'t work' % bad)
print (' %s Phase: Analysing %s[%s6/6%s]%s' % (lightning, green, end, green, end))
print (' %s Phase: Analysing %s[%s6/6%s]%s' %
(lightning, green, end, green, end))
binary = stringToBinary(''.join(allTokens))
result = isRandom(binary)

6
core/colors.py
View File

@@ -1,9 +1,9 @@
import sys
colors = True # Output should be colored
machine = sys.platform # Detecting the os of current system
colors = True # Output should be colored
machine = sys.platform # Detecting the os of current system
if machine.lower().startswith(('os', 'win', 'darwin', 'ios')):
colors = False # Colors shouldn't be displayed in mac & windows
colors = False # Colors shouldn't be displayed in mac & windows
if not colors:
end = red = white = green = yellow = run = bad = good = info = que = ''
lightning = '⚡'

3
core/config.py
View File

@@ -1,6 +1,7 @@
password = 'xXx!69!xXx'
email = 'testing@gmail.com'
strings = ['red', 'bob', 'admin', 'alex', 'testing', 'test', 'lol', 'yes', 'dragon', 'bad']
strings = ['red', 'bob', 'admin', 'alex', 'testing',
'test', 'lol', 'yes', 'dragon', 'bad']
commonNames = ['csrf', 'auth', 'token', 'verify', 'hash']
tokenPattern = r'^[\w\-_+=/]{14,256}$'

1
core/datanize.py
View File

@@ -3,6 +3,7 @@ import re
from core.config import password, email, tokenPattern, strings
def datanize(forms, tolerate=False):
parsedForms = list(forms.values())
for oneForm in parsedForms:

339
core/entropy.py
View File

@@ -7,23 +7,36 @@ import scipy.fftpack as sff
import scipy.stats as sst
from functools import reduce
def sumi(x): return 2 * x - 1
def su(x, y): return x + y
def sus(x): return (x - 0.5) ** 2
def sq(x): return int(x) ** 2
def logo(x): return x * np.log(x)
def pr(u, x):
if u == 0:
out=1.0 * np.exp(-x)
out = 1.0 * np.exp(-x)
else:
out=1.0 * x * np.exp(2*-x) * (2**-u) * spc.hyp1f1(u + 1, 2, x)
out = 1.0 * x * np.exp(2*-x) * (2**-u) * spc.hyp1f1(u + 1, 2, x)
return out
def stringpart(binin, num):
blocks = [binin[xs * num:num + xs * num:] for xs in range(floor(len(binin) / num))]
blocks = [binin[xs * num:num + xs * num:]
for xs in range(floor(len(binin) / num))]
return blocks
def randgen(num):
'''Spits out a stream of random numbers like '1001001' with the length num'''
@@ -36,6 +49,7 @@ def randgen(num):
stream += str(c >> i & 1)
return stream
def monobitfrequencytest(binin):
''' The focus of the test is the proportion of zeroes and ones for the entire sequence. The purpose of this test is to determine whether that number of ones and zeros in a sequence are approximately the same as would be expected for a truly random sequence. The test assesses the closeness of the fraction of ones to 1/2, that is, the number of ones and zeroes in a sequence should be about the same.'''
@@ -46,24 +60,30 @@ def monobitfrequencytest(binin):
pval = spc.erfc(sobs / np.sqrt(2))
return pval
def blockfrequencytest(binin, nu=20):
''' The focus of the test is the proportion of zeroes and ones within M-bit blocks. The purpose of this test is to determine whether the frequency of ones is an M-bit block is approximately M/2.'''
ss = [int(el) for el in binin]
tt = [1.0 * sum(ss[xs * nu:nu + xs * nu:]) / nu for xs in range(floor(len(ss) / nu))]
tt = [1.0 * sum(ss[xs * nu:nu + xs * nu:]) /
nu for xs in range(floor(len(ss) / nu))]
uu = list(map(sus, tt))
chisqr = 4 * nu * reduce(su, uu, 0)
pval = spc.gammaincc(len(tt) / 2.0, chisqr / 2.0)
return pval
def runstest(binin):
''' The focus of this test is the total number of zero and one runs in the entire sequence, where a run is an uninterrupted sequence of identical bits. A run of length k means that a run consists of exactly k identical bits and is bounded before and after with a bit of the opposite value. The purpose of the runs test is to determine whether the number of runs of ones and zeros of various lengths is as expected for a random sequence. In particular, this test determines whether the oscillation between such substrings is too fast or too slow.'''
ss = [int(el) for el in binin]
n = len(binin)
pi = 1.0 * reduce(su, ss) / n
vobs = len(binin.replace('0', ' ').split()) + len(binin.replace('1' , ' ').split())
pval = spc.erfc(abs(vobs-2*n*pi*(1-pi)) / (2 * pi * (1 - pi) * np.sqrt(2*n)))
vobs = len(binin.replace('0', ' ').split()) + \
len(binin.replace('1', ' ').split())
pval = spc.erfc(abs(vobs-2*n*pi*(1-pi)) /
(2 * pi * (1 - pi) * np.sqrt(2*n)))
return pval
def longestrunones8(binin):
''' The focus of the test is the longest run of ones within M-bit blocks. The purpose of this test is to determine whether the length of the longest run of ones within the tested sequence is consistent with the length of the longest run of ones that would be expected in a random sequence. Note that an irregularity in the expected length of the longest run of ones implies that there is also an irregularity in the expected length of the longest run of zeroes. Long runs of zeroes were not evaluated separately due to a concern about statistical independence among the tests.'''
m = 8
@@ -71,29 +91,33 @@ def longestrunones8(binin):
pik = [0.2148, 0.3672, 0.2305, 0.1875]
blocks = [binin[xs*m:m+xs*m:] for xs in range(len(binin) / m)]
n = len(blocks)
counts1 = [xs+'01' for xs in blocks] # append the string 01 to guarantee the length of 1
counts = [xs.replace('0',' ').split() for xs in counts1] # split into all parts
# append the string 01 to guarantee the length of 1
counts1 = [xs+'01' for xs in blocks]
counts = [xs.replace('0', ' ').split()
for xs in counts1] # split into all parts
counts2 = [list(map(len, xx)) for xx in counts]
counts4 = [(4 if xx > 4 else xx) for xx in map(max,counts2)]
counts4 = [(4 if xx > 4 else xx) for xx in map(max, counts2)]
freqs = [counts4.count(spi) for spi in [1, 2, 3, 4]]
chisqr1 = [(freqs[xx]-n*pik[xx])**2/(n*pik[xx]) for xx in range(4)]
chisqr = reduce(su, chisqr1)
pval = spc.gammaincc(k / 2.0, chisqr / 2.0)
return pval
def longestrunones128(binin): # not well tested yet
if len(binin) > 128:
m = 128
k = 5
n = len(binin)
pik = [ 0.1174, 0.2430, 0.2493, 0.1752, 0.1027, 0.1124 ]
pik = [0.1174, 0.2430, 0.2493, 0.1752, 0.1027, 0.1124]
blocks = [binin[xs * m:m + xs * m:] for xs in range(len(binin) / m)]
n = len(blocks)
counts = [xs.replace('0', ' ').split() for xs in blocks]
counts2 = [list(map(len, xx)) for xx in counts]
counts3 = [(1 if xx < 1 else xx) for xx in map(max, counts2)]
counts4 = [(4 if xx > 4 else xx) for xx in counts3]
chisqr1 = [(counts4[xx] - n * pik[xx]) ** 2 / (n * pik[xx]) for xx in range(len(counts4))]
chisqr1 = [(counts4[xx] - n * pik[xx]) ** 2 / (n * pik[xx])
for xx in range(len(counts4))]
chisqr = reduce(su, chisqr1)
pval = spc.gammaincc(k / 2.0, chisqr / 2.0)
else:
@@ -101,20 +125,23 @@ def longestrunones128(binin): # not well tested yet
pval = 0
return pval
def longestrunones10000(binin): # not well tested yet
''' The focus of the test is the longest run of ones within M-bit blocks. The purpose of this test is to determine whether the length of the longest run of ones within the tested sequence is consistent with the length of the longest run of ones that would be expected in a random sequence. Note that an irregularity in the expected length of the longest run of ones implies that there is also an irregularity in the expected length of the longest run of zeroes. Long runs of zeroes were not evaluated separately due to a concern about statistical independence among the tests.'''
if len(binin) > 128:
m = 10000
k = 6
pik = [0.0882, 0.2092, 0.2483, 0.1933, 0.1208, 0.0675, 0.0727]
blocks = [binin[xs * m:m + xs * m:] for xs in range(floor(len(binin) / m))]
blocks = [binin[xs * m:m + xs * m:]
for xs in range(floor(len(binin) / m))]
n = len(blocks)
counts = [xs.replace('0', ' ').split() for xs in blocks]
counts2 = [list(map(len, xx)) for xx in counts]
counts3 = [(10 if xx < 10 else xx) for xx in map(max, counts2)]
counts4 = [(16 if xx > 16 else xx) for xx in counts3]
freqs = [counts4.count(spi) for spi in [10,11,12,13,14,15,16]]
chisqr1 = [(freqs[xx] - n * pik[xx]) ** 2 / (n * pik[xx]) for xx in range(len(freqs))]
freqs = [counts4.count(spi) for spi in [10, 11, 12, 13, 14, 15, 16]]
chisqr1 = [(freqs[xx] - n * pik[xx]) ** 2 / (n * pik[xx])
for xx in range(len(freqs))]
chisqr = reduce(su, chisqr1)
pval = spc.gammaincc(k / 2.0, chisqr / 2.0)
else:
@@ -123,6 +150,8 @@ def longestrunones10000(binin): # not well tested yet
return pval
# test 2.06
def spectraltest(binin):
'''The focus of this test is the peak heights in the discrete Fast Fourier Transform. The purpose of this test is to detect periodic features (i.e., repetitive patterns that are near each other) in the tested sequence that would indicate a deviation from the assumption of randomness. '''
@@ -133,11 +162,12 @@ def spectraltest(binin):
af = abs(ft)[1:floor(n/2)+1:]
t = np.sqrt(np.log(1/0.05)*n)
n0 = 0.95*n/2
n1 = len(np.where(af<t)[0])
n1 = len(np.where(af < t)[0])
d = (n1 - n0)/np.sqrt(n*0.95*0.05/4)
pval = spc.erfc(abs(d)/np.sqrt(2))
return pval
def nonoverlappingtemplatematchingtest(binin, mat="000000001", num=9):
''' The focus of this test is the number of occurrences of pre-defined target substrings. The purpose of this test is to reject sequences that exhibit too many occurrences of a given non-periodic (aperiodic) pattern. For this test and for the Overlapping Template Matching test, an m-bit window is used to search for a specific m-bit pattern. If the pattern is not found, the window slides one bit position. For this test, when the pattern is found, the window is reset to the bit after the found pattern, and the search resumes.'''
n = len(binin)
@@ -146,21 +176,23 @@ def nonoverlappingtemplatematchingtest(binin, mat="000000001", num=9):
blocks = [binin[xs*M:M+xs*M:] for xs in range(floor(n/M))]
counts = [xx.count(mat) for xx in blocks]
avg = 1.0 * (M-m+1)/2 ** m
var = M*(2**-m -(2*m-1)*2**(-2*m))
var = M*(2**-m - (2*m-1)*2**(-2*m))
chisqr = reduce(su, [(xs - avg) ** 2 for xs in counts]) / var
pval = spc.gammaincc(1.0 * len(blocks) / 2, chisqr / 2)
return pval
def occurances(string, sub):
count=start=0
count = start = 0
while True:
start=string.find(sub,start)+1
if start>0:
count+=1
start = string.find(sub, start)+1
if start > 0:
count += 1
else:
return count
def overlappingtemplatematchingtest(binin,mat="111111111",num=1032,numi=9):
def overlappingtemplatematchingtest(binin, mat="111111111", num=1032, numi=9):
''' The focus of this test is the number of pre-defined target substrings. The purpose of this test is to reject sequences that show deviations from the expected number of runs of ones of a given length. Note that when there is a deviation from the expected number of ones of a given length, there is also a deviation in the runs of zeroes. Runs of zeroes were not evaluated separately due to a concern about statistical independence among the tests. For this test and for the Non-overlapping Template Matching test, an m-bit window is used to search for a specific m-bit pattern. If the pattern is not found, the window slides one bit position. For this test, when the pattern is found, the window again slides one bit, and the search is resumed.'''
n = len(binin)
bign = int(n / num)
@@ -172,15 +204,17 @@ def overlappingtemplatematchingtest(binin,mat="111111111",num=1032,numi=9):
v = [0 for x in range(numi + 1)]
blocks = stringpart(binin, num)
blocklen = len(blocks[0])
counts = [occurances(i,mat) for i in blocks]
counts = [occurances(i, mat) for i in blocks]
counts2 = [(numi if xx > numi else xx) for xx in counts]
for i in counts2: v[i] = v[i] + 1
chisqr = reduce(su, [(v[i]-bign*pi[i])** 2 / (bign*pi[i]) for i in range(numi + 1)])
for i in counts2:
v[i] = v[i] + 1
chisqr = reduce(su, [(v[i]-bign*pi[i]) ** 2 / (bign*pi[i])
for i in range(numi + 1)])
pval = spc.gammaincc(0.5*numi, 0.5*chisqr)
return pval
def maurersuniversalstatistictest(binin,l=6,q=640):
def maurersuniversalstatistictest(binin, l=6, q=640):
''' The focus of this test is the number of bits between matching patterns. The purpose of the test is to detect whether or not the sequence can be significantly compressed without loss of information. An overly compressible sequence is considered to be non-random.'''
ru = [
[0.7326495, 0.690],
@@ -199,22 +233,23 @@ def maurersuniversalstatistictest(binin,l=6,q=640):
[13.167693, 3.416],
[14.167488, 3.419],
[15.167379, 3.421],
]
]
blocks = [int(li, 2) + 1 for li in stringpart(binin, l)]
k = len(blocks) - q
states = [0 for x in range(2**l)]
for x in range(q):
states[blocks[x]-1]=x+1
sumi=0.0
for x in range(q,len(blocks)):
sumi+=np.log2((x+1)-states[blocks[x]-1])
states[blocks[x]-1] = x+1
sumi = 0.0
for x in range(q, len(blocks)):
sumi += np.log2((x+1)-states[blocks[x]-1])
states[blocks[x]-1] = x+1
fn = sumi / k
c=0.7-(0.8/l)+(4+(32.0/l))*((k**(-3.0/l))/15)
sigma=c*np.sqrt((ru[l-1][1])/k)
c = 0.7-(0.8/l)+(4+(32.0/l))*((k**(-3.0/l))/15)
sigma = c*np.sqrt((ru[l-1][1])/k)
pval = spc.erfc(abs(fn-ru[l-1][0]) / (np.sqrt(2)*sigma))
return pval
def lempelzivcompressiontest1(binin):
''' The focus of this test is the number of cumulatively distinct patterns (words) in the sequence. The purpose of the test is to determine how far the tested sequence can be compressed. The sequence is considered to be non-random if it can be significantly compressed. A random sequence will have a characteristic number of distinct patterns.'''
i = 1
@@ -223,29 +258,31 @@ def lempelzivcompressiontest1(binin):
mu = 69586.25
sigma = 70.448718
words = []
while (i+j)<=n:
tmp=binin[i:i+j:]
if words.count(tmp)>0:
j+=1
else:
words.append(tmp)
i+=j+1
j=0
while (i+j) <= n:
tmp = binin[i:i+j:]
if words.count(tmp) > 0:
j += 1
else:
words.append(tmp)
i += j+1
j = 0
wobs = len(words)
pval = 0.5*spc.erfc((mu-wobs)/np.sqrt(2.0*sigma))
return pval
# test 2.11
def serialtest(binin):
m = int(log(len(binin), 2) - 3)
''' The focus of this test is the frequency of each and every overlapping m-bit pattern across the entire sequence. The purpose of this test is to determine whether the number of occurrences of the 2m m-bit overlapping patterns is approximately the same as would be expected for a random sequence. The pattern can overlap.'''
n = len(binin)
hbin=binin+binin[0:m-1:]
hbin = binin+binin[0:m-1:]
f1a = [hbin[xs:m+xs:] for xs in range(n)]
oo=set(f1a)
oo = set(f1a)
f1 = [f1a.count(xs)**2 for xs in oo]
f1 = list(map(f1a.count,oo))
cou=f1a.count
f1 = list(map(f1a.count, oo))
cou = f1a.count
f2a = [hbin[xs:m-1+xs:] for xs in range(n)]
f2 = [f2a.count(xs)**2 for xs in set(f2a)]
f3a = [hbin[xs:m-2+xs:] for xs in range(n)]
@@ -254,13 +291,13 @@ def serialtest(binin):
psim2 = 0
psim3 = 0
if m >= 0:
suss = reduce(su,f1)
suss = reduce(su, f1)
psim1 = 1.0 * 2 ** m * suss / n - n
if m >= 1:
suss = reduce(su,f2)
suss = reduce(su, f2)
psim2 = 1.0 * 2 ** (m - 1) * suss / n - n
if m >= 2:
suss = reduce(su,f3)
suss = reduce(su, f3)
psim3 = 1.0 * 2 ** (m - 2) * suss / n - n
d1 = psim1-psim2
d2 = psim1-2 * psim2 + psim3
@@ -281,57 +318,64 @@ def cumultativesumstest(binin):
stop = int(np.floor(0.25 * np.floor(n / z) - 1))
pv1 = []
for k in range(start, stop + 1):
pv1.append(sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)) - sst.norm.cdf((4 * k - 1) * z / np.sqrt(n)))
pv1.append(sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)) -
sst.norm.cdf((4 * k - 1) * z / np.sqrt(n)))
start = int(np.floor(0.25 * np.floor(-n / z - 3)))
stop = int(np.floor(0.25 * np.floor(n / z) - 1))
pv2 = []
for k in range(start, stop + 1):
pv2.append(sst.norm.cdf((4 * k + 3) * z / np.sqrt(n)) - sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)))
pv2.append(sst.norm.cdf((4 * k + 3) * z / np.sqrt(n)) -
sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)))
pval = 1
pval -= reduce(su, pv1)
pval += reduce(su, pv2)
return pval
def cumultativesumstestreverse(binin):
'''The focus of this test is the maximal excursion (from zero) of the random walk defined by the cumulative sum of adjusted (-1, +1) digits in the sequence. The purpose of the test is to determine whether the cumulative sum of the partial sequences occurring in the tested sequence is too large or too small relative to the expected behavior of that cumulative sum for random sequences. This cumulative sum may be considered as a random walk. For a random sequence, the random walk should be near zero. For non-random sequences, the excursions of this random walk away from zero will be too large. '''
pval=cumultativesumstest(binin[::-1])
pval = cumultativesumstest(binin[::-1])
return pval
def pik(k,x):
if k==0:
out=1-1.0/(2*np.abs(x))
elif k>=5:
out=(1.0/(2*np.abs(x)))*(1-1.0/(2*np.abs(x)))**4
def pik(k, x):
if k == 0:
out = 1-1.0/(2*np.abs(x))
elif k >= 5:
out = (1.0/(2*np.abs(x)))*(1-1.0/(2*np.abs(x)))**4
else:
out=(1.0/(4*x*x))*(1-1.0/(2*np.abs(x)))**(k-1)
out = (1.0/(4*x*x))*(1-1.0/(2*np.abs(x)))**(k-1)
return out
def randomexcursionstest(binin):
''' The focus of this test is the number of cycles having exactly K visits in a cumulative sum random walk. The cumulative sum random walk is found if partial sums of the (0,1) sequence are adjusted to (-1, +1). A random excursion of a random walk consists of a sequence of n steps of unit length taken at random that begin at and return to the origin. The purpose of this test is to determine if the number of visits to a state within a random walk exceeds what one would expect for a random sequence.'''
xvals=[-4, -3, -2, -1, 1, 2, 3, 4]
xvals = [-4, -3, -2, -1, 1, 2, 3, 4]
ss = [int(el) for el in binin]
sc = list(map(sumi,ss))
sc = list(map(sumi, ss))
cumsum = np.cumsum(sc)
cumsum = np.append(cumsum,0)
cumsum = np.append(0,cumsum)
posi=np.where(cumsum==0)[0]
cycles=([cumsum[posi[x]:posi[x+1]+1] for x in range(len(posi)-1)])
j=len(cycles)
sct=[]
cumsum = np.append(cumsum, 0)
cumsum = np.append(0, cumsum)
posi = np.where(cumsum == 0)[0]
cycles = ([cumsum[posi[x]:posi[x+1]+1] for x in range(len(posi)-1)])
j = len(cycles)
sct = []
for ii in cycles:
sct.append(([len(np.where(ii==xx)[0]) for xx in xvals]))
sct=np.transpose(np.clip(sct,0,5))
su=[]
sct.append(([len(np.where(ii == xx)[0]) for xx in xvals]))
sct = np.transpose(np.clip(sct, 0, 5))
su = []
for ii in range(6):
su.append([(xx==ii).sum() for xx in sct])
su=np.transpose(su)
pikt=([([pik(uu,xx) for uu in range(6)]) for xx in xvals])
su.append([(xx == ii).sum() for xx in sct])
su = np.transpose(su)
pikt = ([([pik(uu, xx) for uu in range(6)]) for xx in xvals])
# chitab=1.0*((su-j*pikt)**2)/(j*pikt)
chitab=np.sum(1.0*(np.array(su)-j*np.array(pikt))**2/(j*np.array(pikt)),axis=1)
pval=([spc.gammaincc(2.5,cs/2.0) for cs in chitab])
chitab = np.sum(1.0*(np.array(su)-j*np.array(pikt))
** 2/(j*np.array(pikt)), axis=1)
pval = ([spc.gammaincc(2.5, cs/2.0) for cs in chitab])
return pval
def getfreq(linn, nu):
val = 0
for (x, y) in linn:
@@ -339,6 +383,7 @@ def getfreq(linn, nu):
val = y
return val
def randomexcursionsvarianttest(binin):
''' The focus of this test is the number of times that a particular state occurs in a cumulative sum random walk. The purpose of this test is to detect deviations from the expected number of occurrences of various states in the random walk.'''
ss = [int(el) for el in binin]
@@ -353,9 +398,11 @@ def randomexcursionsvarianttest(binin):
for xs in range(-9, 9 + 1):
if not xs == 0:
# pval.append([xs, spc.erfc(np.abs(getfreq(li, xs) - j) / np.sqrt(2 * j * (4 * np.abs(xs) - 2)))])
pval.append(spc.erfc(np.abs(getfreq(li, xs) - j) / np.sqrt(2 * j * (4 * np.abs(xs) - 2))))
pval.append(spc.erfc(np.abs(getfreq(li, xs) - j) /
np.sqrt(2 * j * (4 * np.abs(xs) - 2))))
return pval
def aproximateentropytest(binin, m=5):
''' The focus of this test is the frequency of each and every overlapping m-bit pattern. The purpose of the test is to compare the frequency of overlapping blocks of two consecutive/adjacent lengths (m and m+1) against the expected result for a random sequence.'''
n = len(binin)
@@ -372,82 +419,94 @@ def aproximateentropytest(binin, m=5):
pval = spc.gammaincc(2 ** (m - 1), chisqr / 2.0)
return pval
def matrank(mat): ## old function, does not work as advertized - gives the matrix rank, but not binary
def matrank(mat): # old function, does not work as advertized - gives the matrix rank, but not binary
u, s, v = np.linalg.svd(mat)
rank = np.sum(s > 1e-10)
return rank
def mrank(matrix): # matrix rank as defined in the NIST specification
m=len(matrix)
leni=len(matrix[0])
def mrank(matrix): # matrix rank as defined in the NIST specification
m = len(matrix)
leni = len(matrix[0])
def proc(mat):
for i in range(m):
if mat[i][i]==0:
for j in range(i+1,m):
if mat[j][i]==1:
mat[j],mat[i]=mat[i],mat[j]
if mat[i][i] == 0:
for j in range(i+1, m):
if mat[j][i] == 1:
mat[j], mat[i] = mat[i], mat[j]
break
if mat[i][i]==1:
for j in range(i+1,m):
if mat[j][i]==1: mat[j]=[mat[i][x]^mat[j][x] for x in range(leni)]
if mat[i][i] == 1:
for j in range(i+1, m):
if mat[j][i] == 1:
mat[j] = [mat[i][x] ^ mat[j][x] for x in range(leni)]
return mat
maa=proc(matrix)
maa = proc(matrix)
maa.reverse()
mu=[i[::-1] for i in maa]
muu=proc(mu)
ra=np.sum(np.sign([xx.sum() for xx in np.array(mu)]))
mu = [i[::-1] for i in maa]
muu = proc(mu)
ra = np.sum(np.sign([xx.sum() for xx in np.array(mu)]))
return ra
def binarymatrixranktest(binin,m=32,q=32):
def binarymatrixranktest(binin, m=32, q=32):
''' The focus of the test is the rank of disjoint sub-matrices of the entire sequence. The purpose of this test is to check for linear dependence among fixed length substrings of the original sequence.'''
p1 = 1.0
for x in range(1,50): p1*=1-(1.0/(2**x))
for x in range(1, 50):
p1 *= 1-(1.0/(2**x))
p2 = 2*p1
p3 = 1-p1-p2;
n=len(binin)
u=[int(el) for el in binin] # the input string as numbers, to generate the dot product
p3 = 1-p1-p2
n = len(binin)
# the input string as numbers, to generate the dot product
u = [int(el) for el in binin]
f1a = [u[xs*m:xs*m+m:] for xs in range(floor(n/m))]
n=len(f1a)
n = len(f1a)
f2a = [f1a[xs*q:xs*q+q:] for xs in range(floor(n/q))]
# r=map(matrank,f2a)
r=list(map(mrank,f2a))
n=len(r)
fm=r.count(m);
fm1=r.count(m-1);
chisqr=((fm-p1*n)**2)/(p1*n)+((fm1-p2*n)**2)/(p2*n)+((n-fm-fm1-p3*n)**2)/(p3*n);
pval=np.exp(-0.5*chisqr)
r = list(map(mrank, f2a))
n = len(r)
fm = r.count(m)
fm1 = r.count(m-1)
chisqr = ((fm-p1*n)**2)/(p1*n)+((fm1-p2*n)**2) / \
(p2*n)+((n-fm-fm1-p3*n)**2)/(p3*n)
pval = np.exp(-0.5*chisqr)
return pval
def lincomplex(binin):
lenn=len(binin)
c=b=np.zeros(lenn)
c[0]=b[0]=1
l=0
m=-1
n=0
u=[int(el) for el in binin] # the input string as numbers, to generate the dot product
p=99
while n<lenn:
v=u[(n-l):n] # was n-l..n-1
lenn = len(binin)
c = b = np.zeros(lenn)
c[0] = b[0] = 1
l = 0
m = -1
n = 0
# the input string as numbers, to generate the dot product
u = [int(el) for el in binin]
p = 99
while n < lenn:
v = u[(n-l):n] # was n-l..n-1
v.reverse()
cc=c[1:l+1] # was 2..l+1
d=(u[n]+np.dot(v,cc))%2
if d==1:
tmp=c
p=np.zeros(lenn)
for i in range(0,l): # was 1..l+1
if b[i]==1:
p[i+n-m]=1
c=(c+p)%2;
if l<=0.5*n: # was if 2l <= n
l=n+1-l
m=n
b=tmp
n+=1
cc = c[1:l+1] # was 2..l+1
d = (u[n]+np.dot(v, cc)) % 2
if d == 1:
tmp = c
p = np.zeros(lenn)
for i in range(0, l): # was 1..l+1
if b[i] == 1:
p[i+n-m] = 1
c = (c+p) % 2
if l <= 0.5*n: # was if 2l <= n
l = n+1-l
m = n
b = tmp
n += 1
return l
# test 2.10
def linearcomplexitytest(binin,m=500):
def linearcomplexitytest(binin, m=500):
''' The focus of this test is the length of a generating feedback register. The purpose of this test is to determine whether or not the sequence is complex enough to be considered random. Random sequences are characterized by a longer feedback register. A short feedback register implies non-randomness.'''
k = 6
pi = [0.01047, 0.03125, 0.125, 0.5, 0.25, 0.0625, 0.020833]
@@ -456,14 +515,17 @@ def linearcomplexitytest(binin,m=500):
bign = len(blocks)
lc = ([lincomplex(chunk) for chunk in blocks])
t = ([-1.0*(((-1)**m)*(chunk-avg)+2.0/9) for chunk in lc])
vg=np.histogram(t,bins=[-9999999999,-2.5,-1.5,-0.5,0.5,1.5,2.5,9999999999])[0][::-1]
im=([((vg[ii]-bign*pi[ii])**2)/(bign*pi[ii]) for ii in range(7)])
chisqr=reduce(su,im)
pval=spc.gammaincc(k/2.0,chisqr/2.0)
vg = np.histogram(t, bins=[-9999999999, -2.5, -
1.5, -0.5, 0.5, 1.5, 2.5, 9999999999])[0][::-1]
im = ([((vg[ii]-bign*pi[ii])**2)/(bign*pi[ii]) for ii in range(7)])
chisqr = reduce(su, im)
pval = spc.gammaincc(k/2.0, chisqr/2.0)
return pval
def isRandom(bits):
result = {}
def adder(name, p):
if 'list' in str(type(p)):
count = 0
@@ -503,11 +565,13 @@ def isRandom(bits):
except:
pass
try:
adder('Non-overlapping template matching test',nonoverlappingtemplatematchingtest(bits[:1048576], '11111', 8))
adder('Non-overlapping template matching test',
nonoverlappingtemplatematchingtest(bits[:1048576], '11111', 8))
except:
pass
try:
adder('Overlapping template matching test',overlappingtemplatematchingtest(bits[:998976], '0000001', 12, 5))
adder('Overlapping template matching test',
overlappingtemplatematchingtest(bits[:998976], '0000001', 12, 5))
except:
pass
try:
@@ -523,23 +587,26 @@ def isRandom(bits):
except:
pass
try:
adder('Random excursions variant test', randomexcursionsvarianttest(bits[:1000000]))
adder('Random excursions variant test',
randomexcursionsvarianttest(bits[:1000000]))
except:
pass
try:
adder('Linear complexity test',linearcomplexitytest(bits[:1000000],10))
adder('Linear complexity test',
linearcomplexitytest(bits[:1000000], 10))
except:
pass
try:
adder('Longest runs test',longestrunones10000(bits))
adder('Longest runs test', longestrunones10000(bits))
except:
pass
try:
adder('Maurers universal statistic test',maurersuniversalstatistictest(bits[:387840], 6, 640))
adder('Maurers universal statistic test',
maurersuniversalstatistictest(bits[:387840], 6, 640))
except:
pass
try:
adder('Random excursions test',randomexcursionstest(bits[:1000000]))
adder('Random excursions test', randomexcursionstest(bits[:1000000]))
except:
pass
return result

7
core/evaluate.py
View File

@@ -2,6 +2,7 @@ from re import match
from core.utils import strength
from core.config import commonNames
def evaluate(dataset, weakTokens, tokenDatabase, allTokens, insecureForms):
done = []
for i in dataset:
@@ -21,10 +22,10 @@ def evaluate(dataset, weakTokens, tokenDatabase, allTokens, insecureForms):
protected = True
break
elif name.lower() in commonNames:
weakTokens.append({url : {name : value}})
weakTokens.append({url: {name: value}})
if not protected and action not in done:
done.append(done)
insecureForms.append({url : each})
insecureForms.append({url: each})
for token in localTokens:
allTokens.append(token)
tokenDatabase.append({url : localTokens})
tokenDatabase.append({url: localTokens})

37
core/photon.py
View File

@@ -1,21 +1,23 @@
# Let's import what we need
from re import findall
import concurrent.futures
from urllib.parse import urlparse # for python3
from urllib.parse import urlparse # for python3
from core.colors import run
from core.zetanize import zetanize
from core.requester import requester
from core.utils import getUrl, getParams
from core.utils import getUrl, getParams, remove_file
def photon(seedUrl, headers, depth, threadCount):
forms = [] # web forms
processed = set() # urls that have been crawled
storage = set() # urls that belong to the target i.e. in-scope
forms = [] # web forms
processed = set() # urls that have been crawled
storage = set() # urls that belong to the target i.e. in-scope
scheme = urlparse(seedUrl).scheme
host = urlparse(seedUrl).netloc
main_url = scheme + '://' + host
storage.add(seedUrl)
def rec(url):
processed.add(url)
urlPrint = (url + (' ' * 60))[:60]
@@ -26,22 +28,31 @@ def photon(seedUrl, headers, depth, threadCount):
inps = []
for name, value in params.items():
inps.append({'name': name, 'value': value})
forms.append({url : {0: {'action': url, 'method': 'get', 'inputs': inps}}})
forms.append(
{url: {0: {'action': url, 'method': 'get', 'inputs': inps}}})
response = requester(url, params, headers, True, 0).text
forms.append({url : zetanize(url, response)})
matches = findall(r'<[aA].*href=["\']{0,1}(.*?)["\']', response)
for link in matches: # iterate over the matches
link = link.split('#')[0].lstrip(' ') # remove everything after a "#" to deal with in-page anchors
forms.append({url: zetanize(url, response)})
matches = findall(
r'<[aA][^>]*?(href|HREF)=["\']{0,1}(.*?)["\']', response)
for link in matches: # iterate over the matches
# remove everything after a "#" to deal with in-page anchors
link = link[1].split('#')[0].lstrip(' ')
if link[:4] == 'http':
if link.startswith(main_url):
storage.add(link)
elif link[:2] == '//':
if link.split('/')[2].startswith(host):
storage.add(schema + link)
storage.add(scheme + '://' + link)
elif link[:1] == '/':
storage.add(main_url + link)
storage.add(remove_file(url) + link)
else:
storage.add(main_url + '/' + link)
usable_url = remove_file(url)
if usable_url.endswith('/'):
storage.add(usable_url + link)
elif link.startswith('/'):
storage.add(usable_url + link)
else:
storage.add(usable_url + '/' + link)
for x in range(depth):
urls = storage - processed
threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=10)

1
core/prompt.py
View File

@@ -1,6 +1,7 @@
import os
import tempfile
def prompt(default=None):
editor = 'nano'
with tempfile.NamedTemporaryFile(mode='r+') as tmpfile:

18
core/ranger.py
View File

@@ -1,10 +1,10 @@
def ranger(tokens):
digits = set()
alphabets = set()
for token in tokens:
for char in token:
if char in '0123456789':
digits.add(char)
elif char in 'abcdefghijklmnopqrstuvwxyz':
alphabets.add(char)
return [list(digits), list(alphabets)]
digits = set()
alphabets = set()
for token in tokens:
for char in token:
if char in '0123456789':
digits.add(char)
elif char in 'abcdefghijklmnopqrstuvwxyz':
alphabets.add(char)
return [list(digits), list(alphabets)]

14
core/requester.py
View File

@@ -3,18 +3,20 @@ import random
import warnings
import requests
warnings.filterwarnings('ignore') # Disable SSL related warnings
warnings.filterwarnings('ignore') # Disable SSL related warnings
def requester(url, data, headers, GET, delay):
time.sleep(delay)
user_agents = ['Mozilla/5.0 (X11; Linux i686; rv:60.0) Gecko/20100101 Firefox/60.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991']
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991']
if headers:
if 'User-Agent' not in headers:
headers['User-Agent'] = random.choice(user_agents)
if 'User-Agent' not in headers:
headers['User-Agent'] = random.choice(user_agents)
if GET:
response = requests.get(url, params=data, headers=headers, verify=False)
response = requests.get(
url, params=data, headers=headers, verify=False)
else:
response = requests.post(url, data=data, headers=headers, verify=False)
return response

79
core/tweaker.py
View File

@@ -2,43 +2,44 @@ from core.config import tokenPattern
import random
import re
def tweaker(data, strategy, index=0, seeds=[None, None]):
digits = seeds[0]
alphabets = seeds[1]
newData = {}
if strategy == 'clear':
for name, value in data.items():
if re.match(tokenPattern, value):
value = ''
newData[name] = value
return newData
elif strategy == 'remove':
for name, value in data.items():
if not re.match(tokenPattern, value):
newData[name] = value
elif strategy == 'break':
for name, value in data.items():
if re.match(tokenPattern, value):
value = value[:index]
for i in index:
value += random.choice(digits + alphabets)
newData[name] = value
elif strategy == 'generate':
for name, value in data.items():
if re.match(tokenPattern, value):
newToken = ''
for char in list(value):
if char in digits:
newToken += random.choice(digits)
elif char in alphabets:
newToken += random.choice(alphabets)
else:
newToken += char
newData[name] = newToken
else:
newData[name] = value
elif strategy == 'replace':
for name, value in data.items():
if re.match(tokenPattern, value):
value
return newData
digits = seeds[0]
alphabets = seeds[1]
newData = {}
if strategy == 'clear':
for name, value in data.items():
if re.match(tokenPattern, value):
value = ''
newData[name] = value
return newData
elif strategy == 'remove':
for name, value in data.items():
if not re.match(tokenPattern, value):
newData[name] = value
elif strategy == 'break':
for name, value in data.items():
if re.match(tokenPattern, value):
value = value[:index]
for i in index:
value += random.choice(digits + alphabets)
newData[name] = value
elif strategy == 'generate':
for name, value in data.items():
if re.match(tokenPattern, value):
newToken = ''
for char in list(value):
if char in digits:
newToken += random.choice(digits)
elif char in alphabets:
newToken += random.choice(alphabets)
else:
newToken += char
newData[name] = newToken
else:
newData[name] = value
elif strategy == 'replace':
for name, value in data.items():
if re.match(tokenPattern, value):
value
return newData

42
core/utils.py
View File

@@ -1,23 +1,26 @@
import re
from core.config import tokenPattern
def longestCommonSubstring(s1, s2):
m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
longest, x_longest = 0, 0
for x in range(1, 1 + len(s1)):
for y in range(1, 1 + len(s2)):
if s1[x - 1] == s2[y - 1]:
m[x][y] = m[x - 1][y - 1] + 1
if m[x][y] > longest:
longest = m[x][y]
x_longest = x
else:
m[x][y] = 0
return s1[x_longest - longest: x_longest]
m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
longest, x_longest = 0, 0
for x in range(1, 1 + len(s1)):
for y in range(1, 1 + len(s2)):
if s1[x - 1] == s2[y - 1]:
m[x][y] = m[x - 1][y - 1] + 1
if m[x][y] > longest:
longest = m[x][y]
x_longest = x
else:
m[x][y] = 0
return s1[x_longest - longest: x_longest]
def stringToBinary(string):
return ''.join(format(ord(x), 'b') for x in string)
def strength(string):
digits = re.findall(r'\d', string)
lowerAlphas = re.findall(r'[a-z]', string)
@@ -27,6 +30,7 @@ def strength(string):
entropy = entropy/2
return entropy
def isProtected(parsed):
protected = False
parsedForms = list(parsed.values())
@@ -40,6 +44,7 @@ def isProtected(parsed):
protected = True
return protected
def extractHeaders(headers):
headers = headers.replace('\\n', '\n')
sorted_headers = {}
@@ -55,12 +60,14 @@ def extractHeaders(headers):
pass
return sorted_headers
def getUrl(url, data, GET):
if GET:
return url.split('?')[0]
else:
return url
def getParams(url, data, GET):
params = {}
if GET:
@@ -78,3 +85,14 @@ def getParams(url, data, GET):
except IndexError:
params = None
return params
def remove_file(url):
if url.count('/') > 2:
replacable = re.search(r'/[^/]*?$', url).group()
if replacable != '/':
return url.replace(replacable, '')
else:
return url
else:
return url

13
core/zetanize.py
View File

@@ -1,9 +1,11 @@
import re
from urllib.parse import urlparse
def zetanize(url, response):
parsedUrl = urlparse(url)
mainUrl = parsedUrl.scheme + '://' + parsedUrl.netloc
def e(string):
return string.encode('utf-8')
@@ -25,7 +27,8 @@ def zetanize(url, response):
else:
action = mainUrl + '/' + action
forms[num]['action'] = action.replace('&amp;', '&') if page else ''
forms[num]['method'] = d(e(method.group(1)).lower()) if method else 'get'
forms[num]['method'] = d(
e(method.group(1)).lower()) if method else 'get'
forms[num]['inputs'] = []
inputs = re.findall(r'(?i)(?s)<input.*?>', response)
for inp in inputs:
@@ -34,14 +37,14 @@ def zetanize(url, response):
inpType = re.search(r'(?i)type=[\'"](.*?)[\'"]', inp)
inpValue = re.search(r'(?i)value=[\'"](.*?)[\'"]', inp)
inpName = d(e(inpName.group(1)))
inpType = d(e(inpType.group(1)) )if inpType else ''
inpType = d(e(inpType.group(1)))if inpType else ''
inpValue = d(e(inpValue.group(1))) if inpValue else ''
if inpType.lower() == 'submit' and inpValue == '':
inpValue = 'Submit Query'
inpDict = {
'name' : inpName,
'type' : inpType,
'value' : inpValue
'name': inpName,
'type': inpType,
'value': inpValue
}
forms[num]['inputs'].append(inpDict)
num += 1