code formatting and improved crawling

bolt.py

@@ -2,11 +2,13 @@ from core.colors import green, yellow, end, run, good, info, bad, white, red
 
 lightning = '\033[93;5m⚡\033[0m'
 
+
 def banner():
     print ('''
      %s⚡ %sBOLT%s  ⚡%s
     ''' % (yellow, white, yellow, end))
 
+
 banner()
 
 try:
@@ -43,9 +45,12 @@ parser = argparse.ArgumentParser()
 parser.add_argument('-u', help='target url', dest='target')
 parser.add_argument('-t', help='number of threads', dest='threads', type=int)
 parser.add_argument('-l', help='levels to crawl', dest='level', type=int)
-parser.add_argument('--delay', help='delay between requests', dest='delay', type=int)
+parser.add_argument('--delay', help='delay between requests',
-parser.add_argument('--timeout', help='http request timeout', dest='timeout', type=int)
+                    dest='delay', type=int)
-parser.add_argument('--headers', help='http headers', dest='add_headers', nargs='?', const=True)
+parser.add_argument('--timeout', help='http request timeout',
+                    dest='timeout', type=int)
+parser.add_argument('--headers', help='http headers',
+                    dest='add_headers', nargs='?', const=True)
 args = parser.parse_args()
 
 if not args.target:
@@ -70,11 +75,14 @@ weakTokens = []
 tokenDatabase = []
 insecureForms = []
 
-print (' %s Phase: Crawling %s[%s1/6%s]%s' % (lightning, green, end, green, end))
+print (' %s Phase: Crawling %s[%s1/6%s]%s' %
+       (lightning, green, end, green, end))
 dataset = photon(target, headers, level, threadCount)
 allForms = dataset[0]
-print ('\r%s Crawled %i URL(s) and found %i form(s).%-10s' % (info, dataset[1], len(allForms), ' '))
+print ('\r%s Crawled %i URL(s) and found %i form(s).%-10s' %
-print (' %s Phase: Evaluating %s[%s2/6%s]%s' % (lightning, green, end, green, end))
+       (info, dataset[1], len(allForms), ' '))
+print (' %s Phase: Evaluating %s[%s2/6%s]%s' %
+       (lightning, green, end, green, end))
 
 evaluate(allForms, weakTokens, tokenDatabase, allTokens, insecureForms)
 
@@ -92,9 +100,11 @@ if insecureForms:
         action = list(insecureForm.values())[0]['action']
         form = action.replace(target, '')
         if form:
-            print ('%s %s %s[%s%s%s]%s' % (bad, url, green, end, form, green, end))
+            print ('%s %s %s[%s%s%s]%s' %
+                   (bad, url, green, end, form, green, end))
 
-print (' %s Phase: Comparing %s[%s3/6%s]%s' % (lightning, green, end, green, end))
+print (' %s Phase: Comparing %s[%s3/6%s]%s' %
+       (lightning, green, end, green, end))
 uniqueTokens = set(allTokens)
 if len(uniqueTokens) < len(allTokens):
     print ('%s Potential Replay Attack condition found' % good)
@@ -103,7 +113,8 @@ if len(uniqueTokens) < len(allTokens):
     for url, token in tokenDatabase:
         for url2, token2 in tokenDatabase:
             if token == token2 and url != url2:
-                print ('%s The same token was used on %s%s%s and %s%s%s' % (good, green, url, end, green, url2, end))
+                print ('%s The same token was used on %s%s%s and %s%s%s' %
+                       (good, green, url, end, green, url2, end))
                 replay = True
     if not replay:
         print ('%s Further investigation shows that it was a false positive.')
@@ -127,6 +138,7 @@ if matches:
     for name in matches:
         print ('    %s>%s %s' % (yellow, end, name))
 
+
 def fuzzy(tokens):
     averages = []
     for token in tokens:
@@ -143,13 +155,16 @@ def fuzzy(tokens):
         averages.append(average)
     return statistics.mean(averages)
 
+
 try:
     similarity = fuzzy(allTokens)
-    print ('%s Tokens are %s%i%%%s similar to each other on an average' % (info, green, similarity, end))
+    print ('%s Tokens are %s%i%%%s similar to each other on an average' %
+           (info, green, similarity, end))
 except statistics.StatisticsError:
     print ('%s No CSRF protection to test' % bad)
     quit()
 
+
 def staticParts(allTokens):
     strings = list(set(allTokens.copy()))
     commonSubstrings = {}
@@ -165,6 +180,8 @@ def staticParts(allTokens):
                 if string not in commonSubstrings[commonSubstring]:
                     commonSubstrings[commonSubstring].append(string)
     return commonSubstrings
+
+
 result = {k: v for k, v in staticParts(allTokens).items() if v}
 
 if result:
@@ -173,9 +190,11 @@ if result:
 
 simTokens = []
 
-print (' %s Phase: Observing %s[%s4/6%s]%s' % (lightning, green, end, green, end))
+print (' %s Phase: Observing %s[%s4/6%s]%s' %
+       (lightning, green, end, green, end))
 print ('%s 100 simultaneous requests are being made, please wait.' % info)
 
+
 def extractForms(url):
     response = requester(url, {}, headers, True, 0).text
     forms = zetanize(url, response)
@@ -188,6 +207,7 @@ def extractForms(url):
                 if strength(value) > 10:
                     simTokens.append(value)
 
+
 while True:
     sample = random.choice(tokenDatabase)
     goodToken = list(sample.values())[0]
@@ -196,7 +216,8 @@ while True:
         break
 
 threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=30)
-futures = (threadpool.submit(extractForms, goodCandidate) for goodCandidate in [goodCandidate] * 30)
+futures = (threadpool.submit(extractForms, goodCandidate)
+           for goodCandidate in [goodCandidate] * 30)
 for i in concurrent.futures.as_completed(futures):
     pass
 
@@ -208,7 +229,8 @@ if simTokens:
 else:
     print ('%s Different tokens were issued for simultaneous requests.' % info)
 
-print (' %s Phase: Testing %s[%s5/6%s]%s' % (lightning, green, end, green, end))
+print (' %s Phase: Testing %s[%s5/6%s]%s' %
+       (lightning, green, end, green, end))
 
 parsed = ''
 print ('%s Finding a suitable form for further testing. It may take a while.' % run)
@@ -298,7 +320,8 @@ for index in range(len(allTokens[0])):
         else:
             difference = abs(originalLength - len(response.text))
             if difference <= tolerableDifference:
-                print ('%s Last %i chars of token aren\'t being checked' % (good, index + 1))
+                print ('%s Last %i chars of token aren\'t being checked' %
+                       (good, index + 1))
     else:
         break
 
@@ -318,7 +341,8 @@ if response.status_code == originalCode:
 else:
     print ('%s It didn\'t work' % bad)
 
-print (' %s Phase: Analysing %s[%s6/6%s]%s' % (lightning, green, end, green, end))
+print (' %s Phase: Analysing %s[%s6/6%s]%s' %
+       (lightning, green, end, green, end))
 
 binary = stringToBinary(''.join(allTokens))
 result = isRandom(binary)

core/colors.py

@@ -1,9 +1,9 @@
 import sys
 
-colors = True # Output should be colored
+colors = True  # Output should be colored
-machine = sys.platform # Detecting the os of current system
+machine = sys.platform  # Detecting the os of current system
 if machine.lower().startswith(('os', 'win', 'darwin', 'ios')):
-    colors = False # Colors shouldn't be displayed in mac & windows
+    colors = False  # Colors shouldn't be displayed in mac & windows
 if not colors:
     end = red = white = green = yellow = run = bad = good = info = que = ''
     lightning = '⚡'
@@ -19,4 +19,4 @@ else:
     bad = '\033[91m[-]\033[0m'
     good = '\033[92m[+]\033[0m'
     run = '\033[97m[~]\033[0m'
-    lightning = '\033[93;5m⚡\033[0m'
+    lightning = '\033[93;5m⚡\033[0m'

core/config.py

@@ -1,6 +1,7 @@
 password = 'xXx!69!xXx'
 email = 'testing@gmail.com'
-strings = ['red', 'bob', 'admin', 'alex', 'testing', 'test', 'lol', 'yes', 'dragon', 'bad']
+strings = ['red', 'bob', 'admin', 'alex', 'testing',
+           'test', 'lol', 'yes', 'dragon', 'bad']
 commonNames = ['csrf', 'auth', 'token', 'verify', 'hash']
 tokenPattern = r'^[\w\-_+=/]{14,256}$'
 
@@ -11,4 +12,4 @@ headers = {  # default headers
     'Connection': 'close',
     'DNT': '1',
     'Upgrade-Insecure-Requests': '1',
-}
+}

core/datanize.py

@@ -3,6 +3,7 @@ import re
 
 from core.config import password, email, tokenPattern, strings
 
+
 def datanize(forms, tolerate=False):
     parsedForms = list(forms.values())
     for oneForm in parsedForms:
@@ -34,4 +35,4 @@ def datanize(forms, tolerate=False):
         if protected:
             if not login or tolerate:
                 return [GET, action, data]
-    return None
+    return None

core/entropy.py

@@ -7,23 +7,36 @@ import scipy.fftpack as sff
 import scipy.stats as sst
 from functools import reduce
 
+
 def sumi(x): return 2 * x - 1
+
+
 def su(x, y): return x + y
+
+
 def sus(x): return (x - 0.5) ** 2
+
+
 def sq(x): return int(x) ** 2
+
+
 def logo(x): return x * np.log(x)
 
+
 def pr(u, x):
     if u == 0:
-        out=1.0 * np.exp(-x)
+        out = 1.0 * np.exp(-x)
     else:
-        out=1.0 * x * np.exp(2*-x) * (2**-u) * spc.hyp1f1(u + 1, 2, x)
+        out = 1.0 * x * np.exp(2*-x) * (2**-u) * spc.hyp1f1(u + 1, 2, x)
     return out
 
+
 def stringpart(binin, num):
-    blocks = [binin[xs * num:num + xs * num:] for xs in range(floor(len(binin) / num))]
+    blocks = [binin[xs * num:num + xs * num:]
+              for xs in range(floor(len(binin) / num))]
     return blocks
 
+
 def randgen(num):
     '''Spits out a stream of random numbers like '1001001' with the length num'''
 
@@ -36,6 +49,7 @@ def randgen(num):
             stream += str(c >> i & 1)
     return stream
 
+
 def monobitfrequencytest(binin):
     ''' The focus of the test is the proportion of zeroes and ones for the entire sequence. The purpose of this test is to determine whether that number of ones and zeros in a sequence are approximately the same as would be expected for a truly random sequence. The test assesses the closeness of the fraction of ones to 1/2, that is, the number of ones and zeroes in a sequence should be about the same.'''
 
@@ -46,24 +60,30 @@ def monobitfrequencytest(binin):
     pval = spc.erfc(sobs / np.sqrt(2))
     return pval
 
+
 def blockfrequencytest(binin, nu=20):
     ''' The focus of the test is the proportion of zeroes and ones within M-bit blocks. The purpose of this test is to determine whether the frequency of ones is an M-bit block is approximately M/2.'''
     ss = [int(el) for el in binin]
-    tt = [1.0 * sum(ss[xs * nu:nu + xs * nu:]) / nu for xs in range(floor(len(ss) / nu))]
+    tt = [1.0 * sum(ss[xs * nu:nu + xs * nu:]) /
+          nu for xs in range(floor(len(ss) / nu))]
     uu = list(map(sus, tt))
     chisqr = 4 * nu * reduce(su, uu, 0)
     pval = spc.gammaincc(len(tt) / 2.0, chisqr / 2.0)
     return pval
 
+
 def runstest(binin):
-    ''' The focus of this test is the total number of zero and one runs in the entire sequence, where a run is an uninterrupted sequence of identical bits. A run of length k means that a run consists of exactly k identical bits and is bounded before and after with a bit of the opposite value. The purpose of the runs test is to determine whether the number of runs of ones and zeros of various lengths is as expected for a random sequence. In particular, this test determines whether the oscillation between such substrings is too fast or too slow.''' 
+    ''' The focus of this test is the total number of zero and one runs in the entire sequence, where a run is an uninterrupted sequence of identical bits. A run of length k means that a run consists of exactly k identical bits and is bounded before and after with a bit of the opposite value. The purpose of the runs test is to determine whether the number of runs of ones and zeros of various lengths is as expected for a random sequence. In particular, this test determines whether the oscillation between such substrings is too fast or too slow.'''
     ss = [int(el) for el in binin]
     n = len(binin)
     pi = 1.0 * reduce(su, ss) / n
-    vobs = len(binin.replace('0', ' ').split()) + len(binin.replace('1' , ' ').split())
+    vobs = len(binin.replace('0', ' ').split()) + \
-    pval = spc.erfc(abs(vobs-2*n*pi*(1-pi)) / (2 * pi * (1 - pi) * np.sqrt(2*n)))
+        len(binin.replace('1', ' ').split())
+    pval = spc.erfc(abs(vobs-2*n*pi*(1-pi)) /
+                    (2 * pi * (1 - pi) * np.sqrt(2*n)))
     return pval
 
+
 def longestrunones8(binin):
     ''' The focus of the test is the longest run of ones within M-bit blocks. The purpose of this test is to determine whether the length of the longest run of ones within the tested sequence is consistent with the length of the longest run of ones that would be expected in a random sequence. Note that an irregularity in the expected length of the longest run of ones implies that there is also an irregularity in the expected length of the longest run of zeroes. Long runs of zeroes were not evaluated separately due to a concern about statistical independence among the tests.'''
     m = 8
@@ -71,29 +91,33 @@ def longestrunones8(binin):
     pik = [0.2148, 0.3672, 0.2305, 0.1875]
     blocks = [binin[xs*m:m+xs*m:] for xs in range(len(binin) / m)]
     n = len(blocks)
-    counts1 = [xs+'01' for xs in blocks] # append the string 01 to guarantee the length of 1
+    # append the string 01 to guarantee the length of 1
-    counts = [xs.replace('0',' ').split() for xs in counts1] # split into all parts
+    counts1 = [xs+'01' for xs in blocks]
+    counts = [xs.replace('0', ' ').split()
+              for xs in counts1]  # split into all parts
     counts2 = [list(map(len, xx)) for xx in counts]
-    counts4 = [(4 if xx > 4 else xx) for xx in map(max,counts2)]
+    counts4 = [(4 if xx > 4 else xx) for xx in map(max, counts2)]
     freqs = [counts4.count(spi) for spi in [1, 2, 3, 4]]
     chisqr1 = [(freqs[xx]-n*pik[xx])**2/(n*pik[xx]) for xx in range(4)]
     chisqr = reduce(su, chisqr1)
     pval = spc.gammaincc(k / 2.0, chisqr / 2.0)
     return pval
 
+
 def longestrunones128(binin):  # not well tested yet
     if len(binin) > 128:
         m = 128
         k = 5
         n = len(binin)
-        pik = [ 0.1174, 0.2430, 0.2493, 0.1752, 0.1027, 0.1124 ]
+        pik = [0.1174, 0.2430, 0.2493, 0.1752, 0.1027, 0.1124]
         blocks = [binin[xs * m:m + xs * m:] for xs in range(len(binin) / m)]
         n = len(blocks)
         counts = [xs.replace('0', ' ').split() for xs in blocks]
         counts2 = [list(map(len, xx)) for xx in counts]
         counts3 = [(1 if xx < 1 else xx) for xx in map(max, counts2)]
         counts4 = [(4 if xx > 4 else xx) for xx in counts3]
-        chisqr1 = [(counts4[xx] - n * pik[xx]) ** 2 / (n * pik[xx]) for xx in range(len(counts4))]
+        chisqr1 = [(counts4[xx] - n * pik[xx]) ** 2 / (n * pik[xx])
+                   for xx in range(len(counts4))]
         chisqr = reduce(su, chisqr1)
         pval = spc.gammaincc(k / 2.0, chisqr / 2.0)
     else:
@@ -101,20 +125,23 @@ def longestrunones128(binin):  # not well tested yet
         pval = 0
     return pval
 
+
 def longestrunones10000(binin):  # not well tested yet
     ''' The focus of the test is the longest run of ones within M-bit blocks. The purpose of this test is to determine whether the length of the longest run of ones within the tested sequence is consistent with the length of the longest run of ones that would be expected in a random sequence. Note that an irregularity in the expected length of the longest run of ones implies that there is also an irregularity in the expected length of the longest run of zeroes. Long runs of zeroes were not evaluated separately due to a concern about statistical independence among the tests.'''
     if len(binin) > 128:
         m = 10000
         k = 6
         pik = [0.0882, 0.2092, 0.2483, 0.1933, 0.1208, 0.0675, 0.0727]
-        blocks = [binin[xs * m:m + xs * m:] for xs in range(floor(len(binin) / m))]
+        blocks = [binin[xs * m:m + xs * m:]
+                  for xs in range(floor(len(binin) / m))]
         n = len(blocks)
         counts = [xs.replace('0', ' ').split() for xs in blocks]
         counts2 = [list(map(len, xx)) for xx in counts]
         counts3 = [(10 if xx < 10 else xx) for xx in map(max, counts2)]
         counts4 = [(16 if xx > 16 else xx) for xx in counts3]
-        freqs = [counts4.count(spi) for spi in [10,11,12,13,14,15,16]]
+        freqs = [counts4.count(spi) for spi in [10, 11, 12, 13, 14, 15, 16]]
-        chisqr1 = [(freqs[xx] - n * pik[xx]) ** 2 / (n * pik[xx]) for xx in range(len(freqs))]
+        chisqr1 = [(freqs[xx] - n * pik[xx]) ** 2 / (n * pik[xx])
+                   for xx in range(len(freqs))]
         chisqr = reduce(su, chisqr1)
         pval = spc.gammaincc(k / 2.0, chisqr / 2.0)
     else:
@@ -123,6 +150,8 @@ def longestrunones10000(binin):  # not well tested yet
     return pval
 
 # test 2.06
+
+
 def spectraltest(binin):
     '''The focus of this test is the peak heights in the discrete Fast Fourier Transform. The purpose of this test is to detect periodic features (i.e., repetitive patterns that are near each other) in the tested sequence that would indicate a deviation from the assumption of randomness. '''
 
@@ -133,11 +162,12 @@ def spectraltest(binin):
     af = abs(ft)[1:floor(n/2)+1:]
     t = np.sqrt(np.log(1/0.05)*n)
     n0 = 0.95*n/2
-    n1 = len(np.where(af<t)[0])
+    n1 = len(np.where(af < t)[0])
     d = (n1 - n0)/np.sqrt(n*0.95*0.05/4)
     pval = spc.erfc(abs(d)/np.sqrt(2))
     return pval
 
+
 def nonoverlappingtemplatematchingtest(binin, mat="000000001", num=9):
     ''' The focus of this test is the number of occurrences of pre-defined target substrings. The purpose of this test is to reject sequences that exhibit too many occurrences of a given non-periodic (aperiodic) pattern. For this test and for the Overlapping Template Matching test, an m-bit window is used to search for a specific m-bit pattern. If the pattern is not found, the window slides one bit position. For this test, when the pattern is found, the window is reset to the bit after the found pattern, and the search resumes.'''
     n = len(binin)
@@ -146,21 +176,23 @@ def nonoverlappingtemplatematchingtest(binin, mat="000000001", num=9):
     blocks = [binin[xs*M:M+xs*M:] for xs in range(floor(n/M))]
     counts = [xx.count(mat) for xx in blocks]
     avg = 1.0 * (M-m+1)/2 ** m
-    var = M*(2**-m -(2*m-1)*2**(-2*m))
+    var = M*(2**-m - (2*m-1)*2**(-2*m))
     chisqr = reduce(su, [(xs - avg) ** 2 for xs in counts]) / var
     pval = spc.gammaincc(1.0 * len(blocks) / 2, chisqr / 2)
     return pval
 
+
 def occurances(string, sub):
-    count=start=0
+    count = start = 0
     while True:
-        start=string.find(sub,start)+1
+        start = string.find(sub, start)+1
-        if start>0:
+        if start > 0:
-            count+=1
+            count += 1
         else:
             return count
 
-def overlappingtemplatematchingtest(binin,mat="111111111",num=1032,numi=9):
+
+def overlappingtemplatematchingtest(binin, mat="111111111", num=1032, numi=9):
     ''' The focus of this test is the number of pre-defined target substrings. The purpose of this test is to reject sequences that show deviations from the expected number of runs of ones of a given length. Note that when there is a deviation from the expected number of ones of a given length, there is also a deviation in the runs of zeroes. Runs of zeroes were not evaluated separately due to a concern about statistical independence among the tests. For this test and for the Non-overlapping Template Matching test, an m-bit window is used to search for a specific m-bit pattern. If the pattern is not found, the window slides one bit position. For this test, when the pattern is found, the window again slides one bit, and the search is resumed.'''
     n = len(binin)
     bign = int(n / num)
@@ -172,15 +204,17 @@ def overlappingtemplatematchingtest(binin,mat="111111111",num=1032,numi=9):
     v = [0 for x in range(numi + 1)]
     blocks = stringpart(binin, num)
     blocklen = len(blocks[0])
-    counts = [occurances(i,mat) for i in blocks]
+    counts = [occurances(i, mat) for i in blocks]
     counts2 = [(numi if xx > numi else xx) for xx in counts]
-    for i in counts2: v[i] = v[i] + 1
+    for i in counts2:
-    chisqr = reduce(su, [(v[i]-bign*pi[i])** 2 / (bign*pi[i]) for i in range(numi + 1)])
+        v[i] = v[i] + 1
+    chisqr = reduce(su, [(v[i]-bign*pi[i]) ** 2 / (bign*pi[i])
+                         for i in range(numi + 1)])
     pval = spc.gammaincc(0.5*numi, 0.5*chisqr)
     return pval
 
 
-def maurersuniversalstatistictest(binin,l=6,q=640):
+def maurersuniversalstatistictest(binin, l=6, q=640):
     ''' The focus of this test is the number of bits between matching patterns. The purpose of the test is to detect whether or not the sequence can be significantly compressed without loss of information. An overly compressible sequence is considered to be non-random.'''
     ru = [
         [0.7326495, 0.690],
@@ -199,22 +233,23 @@ def maurersuniversalstatistictest(binin,l=6,q=640):
         [13.167693, 3.416],
         [14.167488, 3.419],
         [15.167379, 3.421],
-        ]
+    ]
     blocks = [int(li, 2) + 1 for li in stringpart(binin, l)]
     k = len(blocks) - q
     states = [0 for x in range(2**l)]
     for x in range(q):
-        states[blocks[x]-1]=x+1
+        states[blocks[x]-1] = x+1
-    sumi=0.0
+    sumi = 0.0
-    for x in range(q,len(blocks)):
+    for x in range(q, len(blocks)):
-        sumi+=np.log2((x+1)-states[blocks[x]-1])
+        sumi += np.log2((x+1)-states[blocks[x]-1])
         states[blocks[x]-1] = x+1
     fn = sumi / k
-    c=0.7-(0.8/l)+(4+(32.0/l))*((k**(-3.0/l))/15)
+    c = 0.7-(0.8/l)+(4+(32.0/l))*((k**(-3.0/l))/15)
-    sigma=c*np.sqrt((ru[l-1][1])/k)
+    sigma = c*np.sqrt((ru[l-1][1])/k)
     pval = spc.erfc(abs(fn-ru[l-1][0]) / (np.sqrt(2)*sigma))
     return pval
 
+
 def lempelzivcompressiontest1(binin):
     ''' The focus of this test is the number of cumulatively distinct patterns (words) in the sequence. The purpose of the test is to determine how far the tested sequence can be compressed. The sequence is considered to be non-random if it can be significantly compressed. A random sequence will have a characteristic number of distinct patterns.'''
     i = 1
@@ -223,29 +258,31 @@ def lempelzivcompressiontest1(binin):
     mu = 69586.25
     sigma = 70.448718
     words = []
-    while (i+j)<=n:
+    while (i+j) <= n:
-       tmp=binin[i:i+j:] 
+        tmp = binin[i:i+j:]
-       if words.count(tmp)>0:
+        if words.count(tmp) > 0:
-           j+=1
+            j += 1
-       else:
+        else:
-           words.append(tmp)    
+            words.append(tmp)
-           i+=j+1
+            i += j+1
-           j=0
+            j = 0
     wobs = len(words)
-    pval = 0.5*spc.erfc((mu-wobs)/np.sqrt(2.0*sigma)) 
+    pval = 0.5*spc.erfc((mu-wobs)/np.sqrt(2.0*sigma))
     return pval
 
 # test 2.11
+
+
 def serialtest(binin):
     m = int(log(len(binin), 2) - 3)
     ''' The focus of this test is the frequency of each and every overlapping m-bit pattern across the entire sequence. The purpose of this test is to determine whether the number of occurrences of the 2m m-bit overlapping patterns is approximately the same as would be expected for a random sequence. The pattern can overlap.'''
     n = len(binin)
-    hbin=binin+binin[0:m-1:]
+    hbin = binin+binin[0:m-1:]
     f1a = [hbin[xs:m+xs:] for xs in range(n)]
-    oo=set(f1a)
+    oo = set(f1a)
     f1 = [f1a.count(xs)**2 for xs in oo]
-    f1 = list(map(f1a.count,oo))
+    f1 = list(map(f1a.count, oo))
-    cou=f1a.count
+    cou = f1a.count
     f2a = [hbin[xs:m-1+xs:] for xs in range(n)]
     f2 = [f2a.count(xs)**2 for xs in set(f2a)]
     f3a = [hbin[xs:m-2+xs:] for xs in range(n)]
@@ -254,13 +291,13 @@ def serialtest(binin):
     psim2 = 0
     psim3 = 0
     if m >= 0:
-        suss = reduce(su,f1)
+        suss = reduce(su, f1)
         psim1 = 1.0 * 2 ** m * suss / n - n
     if m >= 1:
-        suss = reduce(su,f2)
+        suss = reduce(su, f2)
         psim2 = 1.0 * 2 ** (m - 1) * suss / n - n
     if m >= 2:
-        suss = reduce(su,f3)
+        suss = reduce(su, f3)
         psim3 = 1.0 * 2 ** (m - 2) * suss / n - n
     d1 = psim1-psim2
     d2 = psim1-2 * psim2 + psim3
@@ -281,57 +318,64 @@ def cumultativesumstest(binin):
     stop = int(np.floor(0.25 * np.floor(n / z) - 1))
     pv1 = []
     for k in range(start, stop + 1):
-        pv1.append(sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)) - sst.norm.cdf((4 * k - 1) * z / np.sqrt(n)))
+        pv1.append(sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)) -
+                   sst.norm.cdf((4 * k - 1) * z / np.sqrt(n)))
     start = int(np.floor(0.25 * np.floor(-n / z - 3)))
     stop = int(np.floor(0.25 * np.floor(n / z) - 1))
     pv2 = []
     for k in range(start, stop + 1):
-        pv2.append(sst.norm.cdf((4 * k + 3) * z / np.sqrt(n)) - sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)))
+        pv2.append(sst.norm.cdf((4 * k + 3) * z / np.sqrt(n)) -
+                   sst.norm.cdf((4 * k + 1) * z / np.sqrt(n)))
     pval = 1
     pval -= reduce(su, pv1)
     pval += reduce(su, pv2)
 
     return pval
 
+
 def cumultativesumstestreverse(binin):
     '''The focus of this test is the maximal excursion (from zero) of the random walk defined by the cumulative sum of adjusted (-1, +1) digits in the sequence. The purpose of the test is to determine whether the cumulative sum of the partial sequences occurring in the tested sequence is too large or too small relative to the expected behavior of that cumulative sum for random sequences.  This cumulative sum may be considered as a random walk. For a random sequence, the random walk should be near zero. For non-random sequences, the excursions of this random walk away from zero will be too large. '''
-    pval=cumultativesumstest(binin[::-1])
+    pval = cumultativesumstest(binin[::-1])
     return pval
 
-def pik(k,x):
+
-    if k==0:
+def pik(k, x):
-        out=1-1.0/(2*np.abs(x))
+    if k == 0:
-    elif k>=5:
+        out = 1-1.0/(2*np.abs(x))
-        out=(1.0/(2*np.abs(x)))*(1-1.0/(2*np.abs(x)))**4
+    elif k >= 5:
+        out = (1.0/(2*np.abs(x)))*(1-1.0/(2*np.abs(x)))**4
     else:
-        out=(1.0/(4*x*x))*(1-1.0/(2*np.abs(x)))**(k-1)
+        out = (1.0/(4*x*x))*(1-1.0/(2*np.abs(x)))**(k-1)
     return out
 
+
 def randomexcursionstest(binin):
     ''' The focus of this test is the number of cycles having exactly K visits in a cumulative sum random walk. The cumulative sum random walk is found if partial sums of the (0,1) sequence are adjusted to (-1, +1). A random excursion of a random walk consists of a sequence of n steps of unit length taken at random that begin at and return to the origin. The purpose of this test is to determine if the number of visits to a state within a random walk exceeds what one would expect for a random sequence.'''
-    xvals=[-4, -3, -2, -1, 1, 2, 3, 4]
+    xvals = [-4, -3, -2, -1, 1, 2, 3, 4]
     ss = [int(el) for el in binin]
-    sc = list(map(sumi,ss))
+    sc = list(map(sumi, ss))
     cumsum = np.cumsum(sc)
-    cumsum = np.append(cumsum,0)
+    cumsum = np.append(cumsum, 0)
-    cumsum = np.append(0,cumsum)
+    cumsum = np.append(0, cumsum)
-    posi=np.where(cumsum==0)[0]
+    posi = np.where(cumsum == 0)[0]
-    cycles=([cumsum[posi[x]:posi[x+1]+1] for x in range(len(posi)-1)])
+    cycles = ([cumsum[posi[x]:posi[x+1]+1] for x in range(len(posi)-1)])
-    j=len(cycles)
+    j = len(cycles)
-    sct=[]
+    sct = []
     for ii in cycles:
-        sct.append(([len(np.where(ii==xx)[0]) for xx in xvals]))
+        sct.append(([len(np.where(ii == xx)[0]) for xx in xvals]))
-    sct=np.transpose(np.clip(sct,0,5))
+    sct = np.transpose(np.clip(sct, 0, 5))
-    su=[]
+    su = []
     for ii in range(6):
-        su.append([(xx==ii).sum() for xx in sct])
+        su.append([(xx == ii).sum() for xx in sct])
-    su=np.transpose(su)
+    su = np.transpose(su)
-    pikt=([([pik(uu,xx) for uu in range(6)]) for xx in xvals])
+    pikt = ([([pik(uu, xx) for uu in range(6)]) for xx in xvals])
     # chitab=1.0*((su-j*pikt)**2)/(j*pikt)
-    chitab=np.sum(1.0*(np.array(su)-j*np.array(pikt))**2/(j*np.array(pikt)),axis=1)
+    chitab = np.sum(1.0*(np.array(su)-j*np.array(pikt))
-    pval=([spc.gammaincc(2.5,cs/2.0) for cs in chitab])
+                    ** 2/(j*np.array(pikt)), axis=1)
+    pval = ([spc.gammaincc(2.5, cs/2.0) for cs in chitab])
     return pval
 
+
 def getfreq(linn, nu):
     val = 0
     for (x, y) in linn:
@@ -339,6 +383,7 @@ def getfreq(linn, nu):
             val = y
     return val
 
+
 def randomexcursionsvarianttest(binin):
     ''' The focus of this test is the number of times that a particular state occurs in a cumulative sum random walk. The purpose of this test is to detect deviations from the expected number of occurrences of various states in the random walk.'''
     ss = [int(el) for el in binin]
@@ -353,9 +398,11 @@ def randomexcursionsvarianttest(binin):
     for xs in range(-9, 9 + 1):
         if not xs == 0:
             # pval.append([xs, spc.erfc(np.abs(getfreq(li, xs) - j) / np.sqrt(2 * j * (4 * np.abs(xs) - 2)))])
-            pval.append(spc.erfc(np.abs(getfreq(li, xs) - j) / np.sqrt(2 * j * (4 * np.abs(xs) - 2))))
+            pval.append(spc.erfc(np.abs(getfreq(li, xs) - j) /
+                                 np.sqrt(2 * j * (4 * np.abs(xs) - 2))))
     return pval
 
+
 def aproximateentropytest(binin, m=5):
     ''' The focus of this test is the frequency of each and every overlapping m-bit pattern. The purpose of the test is to compare the frequency of overlapping blocks of two consecutive/adjacent lengths (m and m+1) against the expected result for a random sequence.'''
     n = len(binin)
@@ -372,82 +419,94 @@ def aproximateentropytest(binin, m=5):
     pval = spc.gammaincc(2 ** (m - 1), chisqr / 2.0)
     return pval
 
-def matrank(mat): ## old function, does not work as advertized - gives the matrix rank, but not binary
+
+def matrank(mat):  # old function, does not work as advertized - gives the matrix rank, but not binary
     u, s, v = np.linalg.svd(mat)
     rank = np.sum(s > 1e-10)
     return rank
 
-def mrank(matrix): # matrix rank as defined in the NIST specification
+
-    m=len(matrix)
+def mrank(matrix):  # matrix rank as defined in the NIST specification
-    leni=len(matrix[0])
+    m = len(matrix)
+    leni = len(matrix[0])
+
     def proc(mat):
         for i in range(m):
-            if mat[i][i]==0:
+            if mat[i][i] == 0:
-                for j in range(i+1,m):
+                for j in range(i+1, m):
-                    if mat[j][i]==1:
+                    if mat[j][i] == 1:
-                        mat[j],mat[i]=mat[i],mat[j]
+                        mat[j], mat[i] = mat[i], mat[j]
                         break
-            if mat[i][i]==1:
+            if mat[i][i] == 1:
-                for j in range(i+1,m):
+                for j in range(i+1, m):
-                    if mat[j][i]==1: mat[j]=[mat[i][x]^mat[j][x] for x in range(leni)]
+                    if mat[j][i] == 1:
+                        mat[j] = [mat[i][x] ^ mat[j][x] for x in range(leni)]
         return mat
-    maa=proc(matrix)
+    maa = proc(matrix)
     maa.reverse()
-    mu=[i[::-1] for i in maa]
+    mu = [i[::-1] for i in maa]
-    muu=proc(mu)
+    muu = proc(mu)
-    ra=np.sum(np.sign([xx.sum() for xx in np.array(mu)]))
+    ra = np.sum(np.sign([xx.sum() for xx in np.array(mu)]))
     return ra
 
-def binarymatrixranktest(binin,m=32,q=32):
+
+def binarymatrixranktest(binin, m=32, q=32):
     ''' The focus of the test is the rank of disjoint sub-matrices of the entire sequence. The purpose of this test is to check for linear dependence among fixed length substrings of the original sequence.'''
     p1 = 1.0
-    for x in range(1,50): p1*=1-(1.0/(2**x))
+    for x in range(1, 50):
+        p1 *= 1-(1.0/(2**x))
     p2 = 2*p1
-    p3 = 1-p1-p2;
+    p3 = 1-p1-p2
-    n=len(binin)
+    n = len(binin)
-    u=[int(el) for el in binin] # the input string as numbers, to generate the dot product 
+    # the input string as numbers, to generate the dot product
+    u = [int(el) for el in binin]
     f1a = [u[xs*m:xs*m+m:] for xs in range(floor(n/m))]
-    n=len(f1a)
+    n = len(f1a)
     f2a = [f1a[xs*q:xs*q+q:] for xs in range(floor(n/q))]
     # r=map(matrank,f2a)
-    r=list(map(mrank,f2a))
+    r = list(map(mrank, f2a))
-    n=len(r)
+    n = len(r)
-    fm=r.count(m);
+    fm = r.count(m)
-    fm1=r.count(m-1);
+    fm1 = r.count(m-1)
-    chisqr=((fm-p1*n)**2)/(p1*n)+((fm1-p2*n)**2)/(p2*n)+((n-fm-fm1-p3*n)**2)/(p3*n);
+    chisqr = ((fm-p1*n)**2)/(p1*n)+((fm1-p2*n)**2) / \
-    pval=np.exp(-0.5*chisqr)
+        (p2*n)+((n-fm-fm1-p3*n)**2)/(p3*n)
+    pval = np.exp(-0.5*chisqr)
     return pval
 
+
 def lincomplex(binin):
-    lenn=len(binin)
+    lenn = len(binin)
-    c=b=np.zeros(lenn)
+    c = b = np.zeros(lenn)
-    c[0]=b[0]=1
+    c[0] = b[0] = 1
-    l=0
+    l = 0
-    m=-1
+    m = -1
-    n=0
+    n = 0
-    u=[int(el) for el in binin] # the input string as numbers, to generate the dot product
+    # the input string as numbers, to generate the dot product
-    p=99
+    u = [int(el) for el in binin]
-    while n<lenn:
+    p = 99
-        v=u[(n-l):n] # was n-l..n-1
+    while n < lenn:
+        v = u[(n-l):n]  # was n-l..n-1
         v.reverse()
-        cc=c[1:l+1] # was 2..l+1
+        cc = c[1:l+1]  # was 2..l+1
-        d=(u[n]+np.dot(v,cc))%2
+        d = (u[n]+np.dot(v, cc)) % 2
-        if d==1:
+        if d == 1:
-            tmp=c
+            tmp = c
-            p=np.zeros(lenn)
+            p = np.zeros(lenn)
-            for i in range(0,l): # was 1..l+1
+            for i in range(0, l):  # was 1..l+1
-                 if b[i]==1: 
+                if b[i] == 1:
-                     p[i+n-m]=1
+                    p[i+n-m] = 1
-            c=(c+p)%2;
+            c = (c+p) % 2
-            if l<=0.5*n: # was if 2l <= n
+            if l <= 0.5*n:  # was if 2l <= n
-                 l=n+1-l
+                l = n+1-l
-                 m=n
+                m = n
-                 b=tmp
+                b = tmp
-        n+=1
+        n += 1
     return l
 
 # test 2.10
-def linearcomplexitytest(binin,m=500):
+
+
+def linearcomplexitytest(binin, m=500):
     ''' The focus of this test is the length of a generating feedback register. The purpose of this test is to determine whether or not the sequence is complex enough to be considered random. Random sequences are characterized by a longer feedback register. A short feedback register implies non-randomness.'''
     k = 6
     pi = [0.01047, 0.03125, 0.125, 0.5, 0.25, 0.0625, 0.020833]
@@ -456,14 +515,17 @@ def linearcomplexitytest(binin,m=500):
     bign = len(blocks)
     lc = ([lincomplex(chunk) for chunk in blocks])
     t = ([-1.0*(((-1)**m)*(chunk-avg)+2.0/9) for chunk in lc])
-    vg=np.histogram(t,bins=[-9999999999,-2.5,-1.5,-0.5,0.5,1.5,2.5,9999999999])[0][::-1]
+    vg = np.histogram(t, bins=[-9999999999, -2.5, -
-    im=([((vg[ii]-bign*pi[ii])**2)/(bign*pi[ii]) for ii in range(7)])
+                               1.5, -0.5, 0.5, 1.5, 2.5, 9999999999])[0][::-1]
-    chisqr=reduce(su,im)
+    im = ([((vg[ii]-bign*pi[ii])**2)/(bign*pi[ii]) for ii in range(7)])
-    pval=spc.gammaincc(k/2.0,chisqr/2.0)
+    chisqr = reduce(su, im)
+    pval = spc.gammaincc(k/2.0, chisqr/2.0)
     return pval
 
+
 def isRandom(bits):
     result = {}
+
     def adder(name, p):
         if 'list' in str(type(p)):
             count = 0
@@ -503,11 +565,13 @@ def isRandom(bits):
     except:
         pass
     try:
-        adder('Non-overlapping template matching test',nonoverlappingtemplatematchingtest(bits[:1048576], '11111', 8))
+        adder('Non-overlapping template matching test',
+              nonoverlappingtemplatematchingtest(bits[:1048576], '11111', 8))
     except:
         pass
     try:
-        adder('Overlapping template matching test',overlappingtemplatematchingtest(bits[:998976], '0000001', 12, 5))
+        adder('Overlapping template matching test',
+              overlappingtemplatematchingtest(bits[:998976], '0000001', 12, 5))
     except:
         pass
     try:
@@ -523,23 +587,26 @@ def isRandom(bits):
     except:
         pass
     try:
-        adder('Random excursions variant test', randomexcursionsvarianttest(bits[:1000000]))
+        adder('Random excursions variant test',
+              randomexcursionsvarianttest(bits[:1000000]))
     except:
         pass
     try:
-        adder('Linear complexity test',linearcomplexitytest(bits[:1000000],10))
+        adder('Linear complexity test',
+              linearcomplexitytest(bits[:1000000], 10))
     except:
         pass
     try:
-        adder('Longest runs test',longestrunones10000(bits))
+        adder('Longest runs test', longestrunones10000(bits))
     except:
         pass
     try:
-        adder('Maurers universal statistic test',maurersuniversalstatistictest(bits[:387840], 6, 640))
+        adder('Maurers universal statistic test',
+              maurersuniversalstatistictest(bits[:387840], 6, 640))
     except:
         pass
     try:
-        adder('Random excursions test',randomexcursionstest(bits[:1000000]))
+        adder('Random excursions test', randomexcursionstest(bits[:1000000]))
     except:
         pass
     return result

core/evaluate.py

@@ -2,6 +2,7 @@ from re import match
 from core.utils import strength
 from core.config import commonNames
 
+
 def evaluate(dataset, weakTokens, tokenDatabase, allTokens, insecureForms):
     done = []
     for i in dataset:
@@ -21,10 +22,10 @@ def evaluate(dataset, weakTokens, tokenDatabase, allTokens, insecureForms):
                             protected = True
                             break
                         elif name.lower() in commonNames:
-                            weakTokens.append({url : {name : value}})
+                            weakTokens.append({url: {name: value}})
                 if not protected and action not in done:
                     done.append(done)
-                    insecureForms.append({url : each})
+                    insecureForms.append({url: each})
             for token in localTokens:
                 allTokens.append(token)
-            tokenDatabase.append({url : localTokens})
+            tokenDatabase.append({url: localTokens})

core/photon.py

@@ -1,21 +1,23 @@
 # Let's import what we need
 from re import findall
 import concurrent.futures
-from urllib.parse import urlparse # for python3
+from urllib.parse import urlparse  # for python3
 
 from core.colors import run
 from core.zetanize import zetanize
 from core.requester import requester
-from core.utils import getUrl, getParams
+from core.utils import getUrl, getParams, remove_file
+
 
 def photon(seedUrl, headers, depth, threadCount):
-    forms = [] # web forms
+    forms = []  # web forms
-    processed = set() # urls that have been crawled
+    processed = set()  # urls that have been crawled
-    storage = set() # urls that belong to the target i.e. in-scope
+    storage = set()  # urls that belong to the target i.e. in-scope
     scheme = urlparse(seedUrl).scheme
     host = urlparse(seedUrl).netloc
     main_url = scheme + '://' + host
     storage.add(seedUrl)
+
     def rec(url):
         processed.add(url)
         urlPrint = (url + (' ' * 60))[:60]
@@ -26,26 +28,35 @@ def photon(seedUrl, headers, depth, threadCount):
             inps = []
             for name, value in params.items():
                 inps.append({'name': name, 'value': value})
-            forms.append({url : {0: {'action': url, 'method': 'get', 'inputs': inps}}})
+            forms.append(
+                {url: {0: {'action': url, 'method': 'get', 'inputs': inps}}})
         response = requester(url, params, headers, True, 0).text
-        forms.append({url : zetanize(url, response)})
+        forms.append({url: zetanize(url, response)})
-        matches = findall(r'<[aA].*href=["\']{0,1}(.*?)["\']', response)
+        matches = findall(
-        for link in matches: # iterate over the matches
+            r'<[aA][^>]*?(href|HREF)=["\']{0,1}(.*?)["\']', response)
-            link = link.split('#')[0].lstrip(' ') # remove everything after a "#" to deal with in-page anchors
+        for link in matches:  # iterate over the matches
+            # remove everything after a "#" to deal with in-page anchors
+            link = link[1].split('#')[0].lstrip(' ')
             if link[:4] == 'http':
                 if link.startswith(main_url):
                     storage.add(link)
             elif link[:2] == '//':
                 if link.split('/')[2].startswith(host):
-                    storage.add(schema + link)
+                    storage.add(scheme + '://' + link)
             elif link[:1] == '/':
-                storage.add(main_url + link)
+                storage.add(remove_file(url) + link)
             else:
-                storage.add(main_url + '/' + link)
+                usable_url = remove_file(url)
+                if usable_url.endswith('/'):
+                    storage.add(usable_url + link)
+                elif link.startswith('/'):
+                    storage.add(usable_url + link)
+                else:
+                    storage.add(usable_url + '/' + link)
     for x in range(depth):
         urls = storage - processed
         threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=10)
         futures = (threadpool.submit(rec, url) for url in urls)
         for i in concurrent.futures.as_completed(futures):
             pass
-    return [forms, len(processed)]
+    return [forms, len(processed)]

core/prompt.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 
+
 def prompt(default=None):
     editor = 'nano'
     with tempfile.NamedTemporaryFile(mode='r+') as tmpfile:
@@ -16,4 +17,4 @@ def prompt(default=None):
         else:
             os.waitpid(child_pid, 0)
             tmpfile.seek(0)
-            return tmpfile.read().strip()
+            return tmpfile.read().strip()

core/ranger.py

@@ -1,10 +1,10 @@
 def ranger(tokens):
-	digits = set()
+    digits = set()
-	alphabets = set()
+    alphabets = set()
-	for token in tokens:
+    for token in tokens:
-		for char in token:
+        for char in token:
-			if char in '0123456789':
+            if char in '0123456789':
-				digits.add(char)
+                digits.add(char)
-			elif char in 'abcdefghijklmnopqrstuvwxyz':
+            elif char in 'abcdefghijklmnopqrstuvwxyz':
-				alphabets.add(char)
+                alphabets.add(char)
-	return [list(digits), list(alphabets)]
+    return [list(digits), list(alphabets)]

core/requester.py

@@ -3,18 +3,20 @@ import random
 import warnings
 import requests
 
-warnings.filterwarnings('ignore') # Disable SSL related warnings
+warnings.filterwarnings('ignore')  # Disable SSL related warnings
+
 
 def requester(url, data, headers, GET, delay):
     time.sleep(delay)
     user_agents = ['Mozilla/5.0 (X11; Linux i686; rv:60.0) Gecko/20100101 Firefox/60.0',
-'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
+                   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
-'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991']
+                   'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991']
     if headers:
-    	if 'User-Agent' not in headers:
+        if 'User-Agent' not in headers:
-    		headers['User-Agent'] = random.choice(user_agents)
+            headers['User-Agent'] = random.choice(user_agents)
     if GET:
-        response = requests.get(url, params=data, headers=headers, verify=False)
+        response = requests.get(
+            url, params=data, headers=headers, verify=False)
     else:
         response = requests.post(url, data=data, headers=headers, verify=False)
-    return response
+    return response

core/tweaker.py

@@ -2,43 +2,44 @@ from core.config import tokenPattern
 import random
 import re
 
+
 def tweaker(data, strategy, index=0, seeds=[None, None]):
-	digits = seeds[0]
+    digits = seeds[0]
-	alphabets = seeds[1]
+    alphabets = seeds[1]
-	newData = {}
+    newData = {}
-	if strategy == 'clear':
+    if strategy == 'clear':
-		for name, value in data.items():
+        for name, value in data.items():
-			if re.match(tokenPattern, value):
+            if re.match(tokenPattern, value):
-				value = ''
+                value = ''
-			newData[name] = value
+            newData[name] = value
-		return newData
+        return newData
-	elif strategy == 'remove':
+    elif strategy == 'remove':
-		for name, value in data.items():
+        for name, value in data.items():
-			if not re.match(tokenPattern, value):
+            if not re.match(tokenPattern, value):
-				newData[name] = value
+                newData[name] = value
-	elif strategy == 'break':
+    elif strategy == 'break':
-		for name, value in data.items():
+        for name, value in data.items():
-			if re.match(tokenPattern, value):
+            if re.match(tokenPattern, value):
-				value = value[:index]
+                value = value[:index]
-				for i in index:
+                for i in index:
-					value += random.choice(digits + alphabets)
+                    value += random.choice(digits + alphabets)
-			newData[name] = value
+            newData[name] = value
-	elif strategy == 'generate':
+    elif strategy == 'generate':
-		for name, value in data.items():
+        for name, value in data.items():
-			if re.match(tokenPattern, value):
+            if re.match(tokenPattern, value):
-				newToken = ''
+                newToken = ''
-				for char in list(value):
+                for char in list(value):
-					if char in digits:
+                    if char in digits:
-						newToken += random.choice(digits)
+                        newToken += random.choice(digits)
-					elif char in alphabets:
+                    elif char in alphabets:
-						newToken += random.choice(alphabets)
+                        newToken += random.choice(alphabets)
-					else:
+                    else:
-						newToken += char
+                        newToken += char
-				newData[name] = newToken
+                newData[name] = newToken
-			else:
+            else:
-				newData[name] = value
+                newData[name] = value
-	elif strategy == 'replace':
+    elif strategy == 'replace':
-		for name, value in data.items():
+        for name, value in data.items():
-			if re.match(tokenPattern, value):
+            if re.match(tokenPattern, value):
-				value
+                value
-	return newData
+    return newData

core/utils.py

@@ -1,23 +1,26 @@
 import re
 from core.config import tokenPattern
 
+
 def longestCommonSubstring(s1, s2):
-   m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
+    m = [[0] * (1 + len(s2)) for i in range(1 + len(s1))]
-   longest, x_longest = 0, 0
+    longest, x_longest = 0, 0
-   for x in range(1, 1 + len(s1)):
+    for x in range(1, 1 + len(s1)):
-       for y in range(1, 1 + len(s2)):
+        for y in range(1, 1 + len(s2)):
-           if s1[x - 1] == s2[y - 1]:
+            if s1[x - 1] == s2[y - 1]:
-               m[x][y] = m[x - 1][y - 1] + 1
+                m[x][y] = m[x - 1][y - 1] + 1
-               if m[x][y] > longest:
+                if m[x][y] > longest:
-                   longest = m[x][y]
+                    longest = m[x][y]
-                   x_longest = x
+                    x_longest = x
-           else:
+            else:
-               m[x][y] = 0
+                m[x][y] = 0
-   return s1[x_longest - longest: x_longest]
+    return s1[x_longest - longest: x_longest]
+
 
 def stringToBinary(string):
     return ''.join(format(ord(x), 'b') for x in string)
 
+
 def strength(string):
     digits = re.findall(r'\d', string)
     lowerAlphas = re.findall(r'[a-z]', string)
@@ -27,6 +30,7 @@ def strength(string):
         entropy = entropy/2
     return entropy
 
+
 def isProtected(parsed):
     protected = False
     parsedForms = list(parsed.values())
@@ -40,6 +44,7 @@ def isProtected(parsed):
                 protected = True
     return protected
 
+
 def extractHeaders(headers):
     headers = headers.replace('\\n', '\n')
     sorted_headers = {}
@@ -55,12 +60,14 @@ def extractHeaders(headers):
             pass
     return sorted_headers
 
+
 def getUrl(url, data, GET):
     if GET:
         return url.split('?')[0]
     else:
         return url
 
+
 def getParams(url, data, GET):
     params = {}
     if GET:
@@ -78,3 +85,14 @@ def getParams(url, data, GET):
         except IndexError:
             params = None
     return params
+
+
+def remove_file(url):
+    if url.count('/') > 2:
+        replacable = re.search(r'/[^/]*?$', url).group()
+        if replacable != '/':
+            return url.replace(replacable, '')
+        else:
+            return url
+    else:
+        return url

core/zetanize.py

@@ -1,9 +1,11 @@
 import re
 from urllib.parse import urlparse
 
+
 def zetanize(url, response):
     parsedUrl = urlparse(url)
     mainUrl = parsedUrl.scheme + '://' + parsedUrl.netloc
+
     def e(string):
         return string.encode('utf-8')
 
@@ -25,7 +27,8 @@ def zetanize(url, response):
             else:
                 action = mainUrl + '/' + action
         forms[num]['action'] = action.replace('&', '&') if page else ''
-        forms[num]['method'] = d(e(method.group(1)).lower()) if method else 'get'
+        forms[num]['method'] = d(
+            e(method.group(1)).lower()) if method else 'get'
         forms[num]['inputs'] = []
         inputs = re.findall(r'(?i)(?s)<input.*?>', response)
         for inp in inputs:
@@ -34,15 +37,15 @@ def zetanize(url, response):
                 inpType = re.search(r'(?i)type=[\'"](.*?)[\'"]', inp)
                 inpValue = re.search(r'(?i)value=[\'"](.*?)[\'"]', inp)
                 inpName = d(e(inpName.group(1)))
-                inpType = d(e(inpType.group(1)) )if inpType else ''
+                inpType = d(e(inpType.group(1)))if inpType else ''
                 inpValue = d(e(inpValue.group(1))) if inpValue else ''
                 if inpType.lower() == 'submit' and inpValue == '':
                     inpValue = 'Submit Query'
                 inpDict = {
-                'name' : inpName,
+                    'name': inpName,
-                'type' : inpType,
+                    'type': inpType,
-                'value' : inpValue
+                    'value': inpValue
                 }
                 forms[num]['inputs'].append(inpDict)
         num += 1
-    return forms
+    return forms