2.1.0 pypi release, fixes #41, fixes #22

CHANGELOG.md

@@ -1,3 +1,13 @@
+#### 2.1.0
+- Added `XML` method
+- `-q` option for quiet mode
+- New wordlists backed by research
+- `-oT` option for txt export
+- `-oB` option for BurpSuite export
+- `-oJ` alias for JSON export
+- Added support for custom injection point in `XML` and `JSON`
+- pypi package
+
 #### 2.0-beta
 - Added an anamoly detection algorithm with 9 factors
 - Added a HTTP response analyzer for handling errors and retrying requests

README.md

@@ -29,18 +29,17 @@ Web applications use parameters (or queries) to accept user input, take the foll
 `http://api.example.com/v1/userinfo?id=751634589`
 
 This URL seems to load user information for a specific user id, but what if there exists a parameter named `admin` which when set to `True` makes the endpoint provide more information about the user?\
-This is what Arjun does, it finds valid HTTP parameters with a huge default dictionary of 25,980 parameter names.
+This is what Arjun does, it finds valid HTTP parameters with a huge default dictionary of 10,985 parameter names.
 
-The best part? It takes less than 20 seconds to go through this huge list while making just 50-60 requests to the target. [Here's how](https://github.com/s0md3v/Arjun/wiki/How-Arjun-works%3F).
+The best part? It takes less than 10 seconds to go through this huge list while making just 20-30 requests to the target. [Here's how](https://github.com/s0md3v/Arjun/wiki/How-Arjun-works%3F).
 
 ### Why Arjun?
 
-- Anomaly detection with 9 factors
+- Supports `GET/POST/POST-JSON/POST-XML` requests
-- Supports `GET/POST/POST-JSON`
 - Automatically handles rate limits and timeouts
-- Can import targets from BurpSuite, text file or a raw request file
+- Export results to: BurpSuite, text or JSON file
+- Import targets from: BurpSuite, text file or a raw request file
 - Can passively extract parameters from JS or 3 external sources
-- Makes ~50 requests in 20 seconds for checking 25,980 parameter names
 
 ### How to use Arjun?
 
@@ -51,11 +50,11 @@ A detailed usage guide is available on [Usage](https://github.com/s0md3v/Arjun/w
 Direct links to some basic options are given below:
 
 - [Scan a single URL](https://github.com/s0md3v/Arjun/wiki/Usage#scan-a-single-url)
-- [Import multiple targets](https://github.com/s0md3v/Arjun/wiki/Usage#import-multiple-targets)
+- [Import targets](https://github.com/s0md3v/Arjun/wiki/Usage#import-multiple-targets)
-- [Save output to a file](https://github.com/s0md3v/Arjun/wiki/Usage#save-output-to-a-file)
+- [Export results](https://github.com/s0md3v/Arjun/wiki/Usage#save-output-to-a-file)
 - [Use custom HTTP headers](https://github.com/s0md3v/Arjun/wiki/Usage#use-custom-http-headers)
 
 Optionally, you can use the `--help` argument to explore Arjun on your own.
 
 ##### Credits
-The parameter names wordlist is taken from [@SecLists](https://github.com/danielmiessler/SecLists).
+The parameter names wordlist is created by extracting top parameter names from [CommonCrawl](http://commoncrawl.org) dataset and merging best words from [SecLists](https://github.com/danielmiessler/SecLists) and [param-miner](https://github.com/PortSwigger/param-miner) wordlists into that.

arjun/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.1.0'

arjun/__main__.py

@@ -1,15 +1,51 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from __future__ import print_function
+from arjun.core.colors import green, end, info, bad, good, run, res
 
-from core.colors import green, end, info, bad, good, run, res
+import os
+import argparse
+
+from urllib.parse import urlparse
+import arjun.core.config as mem
+from arjun.core.bruter import bruter
+from arjun.core.exporter import exporter
+from arjun.core.requester import requester
+from arjun.core.anomaly import define
+from arjun.core.utils import fetch_params, stable_request, random_str, slicer, confirm, populate, reader, nullify, prepare_requests
+
+from arjun.plugins.heuristic import heuristic
+
+arjun_dir = mem.__file__.replace('/core/config.py', '')
+
+parser = argparse.ArgumentParser() # defines the parser
+# Arguments that can be supplied
+parser.add_argument('-u', help='target url', dest='url')
+parser.add_argument('-o', '-oJ', help='path for json output file', dest='json_file')
+parser.add_argument('-oT', help='path for text output file', dest='text_file')
+parser.add_argument('-oB', help='port for burp suite proxy', dest='burp_port')
+parser.add_argument('-d', help='delay between requests', dest='delay', type=float, default=0)
+parser.add_argument('-t', help='number of threads', dest='threads', type=int, default=2)
+parser.add_argument('-w', help='wordlist path', dest='wordlist', default=arjun_dir+'/db/default.txt')
+parser.add_argument('-m', help='request method: GET/POST/XML/JSON', dest='method', default='GET')
+parser.add_argument('-i', help='import targets from file', dest='import_file', nargs='?', const=True)
+parser.add_argument('-T', help='http request timeout', dest='timeout', type=float, default=15)
+parser.add_argument('-c', help='chunk size/number of parameters to be sent at once', type=int, dest='chunks', default=500)
+parser.add_argument('-q', help='quiet mode, no output', dest='quiet', action='store_true')
+parser.add_argument('--headers', help='add headers', dest='headers', nargs='?', const=True)
+parser.add_argument('--passive', help='collect parameter names from passive sources', dest='passive')
+parser.add_argument('--stable', help='prefer stability over speed', dest='stable', action='store_true')
+parser.add_argument('--include', help='include this data in every request', dest='include', default={})
+args = parser.parse_args() # arguments to be parsed
+
+if args.quiet:
+    print = nullify
 
 print('''%s    _
    /_| _ '
-  (  |/ /(//) v2.0-beta
+  (  |/ /(//) v%s
       _/      %s
-''' % (green, end))
+''' % (green, __import__('arjun').__version__, end))
 
 try:
     from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -17,39 +53,6 @@ except ImportError:
     print('%s Please use Python > 3.2 to run Arjun.' % bad)
     quit()
 
-import sys
-import json
-import argparse
-
-from urllib.parse import urlparse
-
-import core.config as mem
-from core.bruter import bruter
-from core.prompt import prompt
-from core.importer import importer
-from core.requester import requester
-from core.anamoly import define
-from core.utils import fetch_params, stable_request, randomString, slicer, confirm, getParams, populate, extractHeaders, reader
-
-from plugins.heuristic import heuristic
-
-parser = argparse.ArgumentParser() # defines the parser
-# Arguments that can be supplied
-parser.add_argument('-u', help='target url', dest='url')
-parser.add_argument('-o', help='path for the output file', dest='output_file')
-parser.add_argument('-d', help='delay between requests', dest='delay', type=float, default=0)
-parser.add_argument('-t', help='number of threads', dest='threads', type=int, default=2)
-parser.add_argument('-w', help='wordlist path', dest='wordlist', default=sys.path[0]+'/db/params.txt')
-parser.add_argument('-m', help='request method: GET/POST/JSON', dest='method', default='GET')
-parser.add_argument('-i', help='import targets from file', dest='import_file', nargs='?', const=True)
-parser.add_argument('-T', help='http request timeout', dest='timeout', type=float, default=15)
-parser.add_argument('-c', help='chunk size/number of parameters to be sent at once', type=int, dest='chunks', default=500)
-parser.add_argument('--headers', help='add headers', dest='headers', nargs='?', const=True)
-parser.add_argument('--passive', help='collect parameter names from passive sources', dest='passive')
-parser.add_argument('--stable', help='prefer stability over speed', dest='stable', action='store_true')
-parser.add_argument('--include', help='include this data in every request', dest='include', default={})
-args = parser.parse_args() # arguments to be parsed
-
 mem.var = vars(args)
 
 mem.var['method'] = mem.var['method'].upper()
@@ -58,7 +61,8 @@ if mem.var['stable'] or mem.var['delay']:
     mem.var['threads'] = 1
 
 try:
-    wordlist = set(reader(args.wordlist, mode='lines'))
+    wordlist_file = arjun_dir + '/db/small.txt' if args.wordlist == 'small' else args.wordlist
+    wordlist = set(reader(wordlist_file, mode='lines'))
     if mem.var['passive']:
         host = mem.var['passive']
         if host == '-':
@@ -72,40 +76,17 @@ except FileNotFoundError:
     exit('%s The specified file for parameters doesn\'t exist' % bad)
 
 if len(wordlist) < mem.var['chunks']:
-   mem.var['chunks'] = int(len(wordlist)/2)
+    mem.var['chunks'] = int(len(wordlist)/2)
 
-if not (args.url, args.import_file):
+if not args.url and not args.import_file:
-    exit('%s No targets specified' % bad)
+    exit('%s No target(s) specified' % bad)
-
-def prepare_requests(args):
-    headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:83.0) Gecko/20100101 Firefox/83.0',
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Language': 'en-US,en;q=0.5',
-    'Accept-Encoding': 'gzip, deflate',
-    'Connection': 'close',
-    'Upgrade-Insecure-Requests': '1'
-    }
-    if type(headers) == bool:
-        headers = extractHeaders(prompt())
-    elif type(headers) == str:
-        headers = extractHeaders(headers)
-    if mem.var['method'] == 'JSON':
-        headers['Content-type'] = 'application/json'
-    if args.url:
-        params = getParams(args.include)
-        return {
-            'url': args.url,
-            'method': mem.var['method'],
-            'headers': headers,
-            'include': params
-        }
-    elif args.import_file:
-        return importer(args.import_file, mem.var['method'], headers, args.include)
-    return []
 
 
 def narrower(request, factors, param_groups):
+    """
+    takes a list of parameters and narrows it down to parameters that cause anomalies
+    returns list
+    """
     anamolous_params = []
     threadpool = ThreadPoolExecutor(max_workers=mem.var['threads'])
     futures = (threadpool.submit(bruter, request, factors, params) for params in param_groups)
@@ -116,7 +97,12 @@ def narrower(request, factors, param_groups):
             print('%s Processing chunks: %i/%-6i' % (info, i + 1, len(param_groups)), end='\r')
     return anamolous_params
 
+
 def initialize(request, wordlist):
+    """
+    handles parameter finding process for a single request object
+    returns 'skipped' (on error), list on success
+    """
     url = request['url']
     if not url.startswith('http'):
         print('%s %s is not a valid URL' % (bad, url))
@@ -126,11 +112,11 @@ def initialize(request, wordlist):
     if not stable:
         return 'skipped'
     else:
-        fuzz = randomString(6)
+        fuzz = random_str(6)
-        response_1 = requester(request, {fuzz : fuzz[::-1]})
+        response_1 = requester(request, {fuzz: fuzz[::-1]})
         print('%s Analysing HTTP response for anamolies' % run)
-        fuzz = randomString(6)
+        fuzz = random_str(6)
-        response_2 = requester(request, {fuzz : fuzz[::-1]})
+        response_2 = requester(request, {fuzz: fuzz[::-1]})
         if type(response_1) == str or type(response_2) == str:
             return 'skipped'
         factors = define(response_1, response_2, fuzz, fuzz[::-1], wordlist)
@@ -160,39 +146,45 @@ def initialize(request, wordlist):
                 print('%s name: %s, factor: %s' % (res, name, reason))
         return confirmed_params
 
-request = prepare_requests(args)
 
-final_result = {}
+def main():
+    request = prepare_requests(args)
 
-try:
+    final_result = {}
-    if type(request) == dict:
+
-        mem.var['kill'] = False
+    try:
-        url = request['url']
+        if type(request) == dict:
-        these_params = initialize(request, wordlist)
+            # in case of a single target
-        if these_params == 'skipped':
-            print('%s Skipped %s due to errors' % (bad, request['url']))
-        elif these_params:
-            final_result['url'] = url
-            final_result['params'] = these_params
-            final_result['method'] = request['method']
-    elif type(request) == list:
-        for each in request:
-            url = each['url']
             mem.var['kill'] = False
-            print('%s Scanning: %s' % (run, url))
+            url = request['url']
-            these_params = initialize(each, list(wordlist))
+            these_params = initialize(request, wordlist)
             if these_params == 'skipped':
-                print('%s Skipped %s due to errors' % (bad, url))
+                print('%s Skipped %s due to errors' % (bad, request['url']))
             elif these_params:
                 final_result[url] = {}
                 final_result[url]['params'] = these_params
-                final_result[url]['method'] = each['method']
+                final_result[url]['method'] = request['method']
-                print('%s Parameters found: %s' % (good, ', '.join(final_result[url])))
+                final_result[url]['headers'] = request['headers']
-except KeyboardInterrupt:
+        elif type(request) == list:
-    exit()
+            # in case of multiple targets
+            for each in request:
+                url = each['url']
+                mem.var['kill'] = False
+                print('%s Scanning: %s' % (run, url))
+                these_params = initialize(each, list(wordlist))
+                if these_params == 'skipped':
+                    print('%s Skipped %s due to errors' % (bad, url))
+                elif these_params:
+                    final_result[url] = {}
+                    final_result[url]['params'] = these_params
+                    final_result[url]['method'] = each['method']
+                    final_result[url]['headers'] = each['headers']
+                    print('%s Parameters found: %s' % (good, ', '.join(final_result[url])))
+    except KeyboardInterrupt:
+        exit()
 
-# Finally, export to json
+    exporter(final_result)
-if args.output_file and final_result:
+
-    with open(str(mem.var['output_file']), 'w+', encoding='utf8') as json_output:
+
-        json.dump(final_result, json_output, sort_keys=True, indent=4)
+if __name__ == '__main__':
-    print('%s Output saved to JSON file in %s' % (info, mem.var['output_file']))
+    main()

arjun/core/anomaly.py

@@ -1,17 +1,13 @@
 import re
 
-from core.utils import lcs, removeTags
+from arjun.core.utils import lcs, diff_map, remove_tags
-
-def diff_map(body_1, body_2):
-    sig = []
-    lines_1, lines_2 = body_1.split('\n'), body_2.split('\n')
-    for line_1, line_2 in zip(lines_1, lines_2):
-        if line_1 == line_2:
-            sig.append(line_1)
-    return sig
 
 
 def define(response_1, response_2, param, value, wordlist):
+    """
+    defines a rule list for detecting anomalies by comparing two HTTP response
+    returns dict
+    """
     factors = {
         'same_code': False, # if http status code is same, contains that code
         'same_body': False, # if http body is same, contains that body
@@ -23,7 +19,7 @@ def define(response_1, response_2, param, value, wordlist):
         'param_missing': False, # if param name is missing from the body, contains words that are already there
         'value_missing': False # contains whether param value is missing from the body
     }
-    if (response_1 and response_2) != None:
+    if response_1 and response_2:
         body_1, body_2 = response_1.text, response_2.text
         if response_1.status_code == response_2.status_code:
             factors['same_code'] = response_1.status_code
@@ -33,8 +29,8 @@ def define(response_1, response_2, param, value, wordlist):
             factors['same_redirect'] = response_1.url
         if response_1.text == response_2.text:
             factors['same_body'] = response_1.text
-        elif removeTags(body_1) == removeTags(body_2):
+        elif remove_tags(body_1) == remove_tags(body_2):
-            factors['same_plaintext'] = removeTags(body_1)
+            factors['same_plaintext'] = remove_tags(body_1)
         elif body_1 and body_2:
             if body_1.count('\\n') == 1:
                 factors['common_string'] = lcs(body_1, body_2)
@@ -48,6 +44,10 @@ def define(response_1, response_2, param, value, wordlist):
 
 
 def compare(response, factors, params):
+    """
+    detects anomalies by comparing a HTTP response against a rule list
+    returns string, list (anamoly, list of parameters that caused it)
+    """
     if factors['same_code'] and response.status_code != factors['same_code']:
         return ('http code', params)
     if factors['same_headers'] and list(response.headers.keys()) != factors['same_headers']:
@@ -56,7 +56,7 @@ def compare(response, factors, params):
         return ('redirection', params)
     if factors['same_body'] and response.text != factors['same_body']:
         return ('body length', params)
-    if factors['same_plaintext'] and removeTags(response.text) != factors['same_plaintext']:
+    if factors['same_plaintext'] and remove_tags(response.text) != factors['same_plaintext']:
         return ('text length', params)
     if factors['lines_diff']:
         for line in factors['lines_diff']:

arjun/core/bruter.py

@@ -1,11 +1,15 @@
-import core.config as mem
+import arjun.core.config as mem
 
-from core.anamoly import compare
+from arjun.core.anomaly import compare
-from core.requester import requester
+from arjun.core.requester import requester
-from core.error_handler import error_handler
+from arjun.core.error_handler import error_handler
 
 
 def bruter(request, factors, params, mode='bruteforce'):
+    """
+    returns anomaly detection result for a chunk of parameters
+    returns list
+    """
     if mem.var['kill']:
         return []
     response = requester(request, params)

arjun/core/config.py

@@ -0,0 +1 @@
+var = {} # all the cli arguments are added to this variable to be accessed globally

arjun/core/error_handler.py

@@ -1,12 +1,16 @@
 import time
 
-import core.config as mem
+import arjun.core.config as mem
 
-from core.colors import bad
+from arjun.core.colors import bad
 
 def connection_refused():
+	"""
+	checks if a request should be retried if the server refused connection
+	returns str
+	"""
 	if mem.var['stable']:
-		print('%s Hit rate limit, stabilizing the connection'  % bad)
+		print('%s Hit rate limit, stabilizing the connection' % bad)
 		mem.var['kill'] = False
 		time.sleep(30)
 		return 'retry'
@@ -14,11 +18,18 @@ def connection_refused():
 	return 'kill'
 
 def error_handler(response, factors):
+	"""
+	decides what to do after performing a HTTP request
+		'ok': continue normally
+		'retry': retry this request
+		'kill': stop processing this target
+	returns str
+	"""
 	if type(response) != str and response.status_code in (400, 503, 429):
 		if response.status_code == 400:
 			if factors['same_code'] != 400:
 				mem.var['kill'] = True
-				print('%s Server recieved a bad request. Try decreasing the chunk size with -c option'  % bad)
+				print('%s Server recieved a bad request. Try decreasing the chunk size with -c option' % bad)
 				return 'kill'
 			else:
 				return 'ok'
@@ -35,7 +46,7 @@ def error_handler(response, factors):
 				print('%s Connection timed out, unable to increase timeout further')
 				return 'kill'
 			else:
-				print('%s Connection timed out, increased timeout by 5 seconds'  % bad)
+				print('%s Connection timed out, increased timeout by 5 seconds' % bad)
 				mem.var['timeout'] += 5
 				return 'retry'
 		elif 'ConnectionRefused' in response:

arjun/core/exporter.py

@@ -0,0 +1,57 @@
+import json
+import re
+import requests
+
+import arjun.core.config as mem
+from arjun.core.utils import populate
+
+from arjun.core.utils import create_query_string
+
+def json_export(result):
+    """
+    exports result to a file in JSON format
+    """
+    with open(mem.var['json_file'], 'w+', encoding='utf8') as json_output:
+        json.dump(result, json_output, sort_keys=True, indent=4)
+
+def burp_export(result):
+    """
+    exports results to Burp Suite by sending request to Burp proxy
+    """
+    for url, data in result.items():
+        url = re.sub(r'://[^/]+', '://' + mem.var['burp_port'], url, 1)
+        if data['method'] == 'GET':
+            requests.get(url, params=populate(data['params']), headers=data['headers'])
+        elif data['method'] == 'POST':
+            requests.post(url, data=populate(data['params']), headers=data['headers'])
+        elif data['method'] == 'JSON':
+            requests.post(url, json=populate(data['params']), headers=data['headers'])
+
+def text_export(result):
+    """
+    exports results to a text file, one url per line
+    """
+    with open(mem.var['text_file'], 'w+', encoding='utf8') as text_file:
+        for url, data in result.items():
+            clean_url = url.lstrip('/')
+            if data['method'] == 'JSON':
+                text_file.write(clean_url + '\t' + json.dumps(populate(data['params'])) + '\n')
+            else:
+                query_string = create_query_string(data['params'])
+                if '?' in clean_url:
+                    query_string = query_string.replace('?', '&', 1)
+                if data['method'] == 'GET':
+                    text_file.write(clean_url + query_string + '\n')
+                elif data['method'] == 'POST':
+                    text_file.write(clean_url + '\t' + query_string + '\n')
+
+def exporter(result):
+    """
+    main exporter function that calls other export functions
+    """
+    if mem.var['json_file']:
+        json_export(result)
+    if mem.var['text_file']:
+        text_export(result)
+    if mem.var['burp_port']:
+        burp_export(result)

arjun/core/importer.py

@@ -0,0 +1,102 @@
+import re
+
+def reader(path, mode='string'):
+    """
+    reads a file
+    returns a string/array containing the content of the file
+    """
+    with open(path, 'r', encoding='utf-8') as file:
+        if mode == 'lines':
+            return list(filter(None, [line.rstrip('\n') for line in file]))
+        else:
+            return ''.join([line for line in file])
+
+def parse_request(string):
+    """
+    parses http request
+    returns dict
+    """
+    result = {}
+    match = re.search(r'(?:([a-zA-Z0-9]+) ([^ ]+) [^ ]+\n)?([\s\S]+\n)\n?([\s\S]+)?', string)
+    result['method'] = match.group(1)
+    result['path'] = match.group(2)
+    result['headers'] = parse_headers(match.group(3))
+    result['data'] = match.group(4)
+    return result
+
+burp_regex = re.compile(r'''(?m)^    <url><!\[CDATA\[(.+?)\]\]></url>
+    <host ip="[^"]*">[^<]+</host>
+    <port>[^<]*</port>
+    <protocol>[^<]*</protocol>
+    <method><!\[CDATA\[(.+?)\]\]></method>
+    <path>.*</path>
+    <extension>(.*)</extension>
+    <request base64="(?:false|true)"><!\[CDATA\[([\s\S]+?)]]></request>
+    <status>([^<]*)</status>
+    <responselength>([^<]*)</responselength>
+    <mimetype>([^<]*)</mimetype>''')
+
+
+def burp_import(path):
+    """
+    imports targets from burp suite
+    returns list (of request objects)
+    """
+    requests = []
+    content = reader(path)
+    matches = re.finditer(burp_regex, content)
+    for match in matches:
+        request = parse_request(match.group(4))
+        headers = request['headers']
+        if match.group(7) in ('HTML', 'JSON'):
+            requests.append({
+                'url': match.group(1),
+                'method': match.group(2),
+                'extension': match.group(3),
+                'headers': headers,
+                'include': request['data'],
+                'code': match.group(5),
+                'length': match.group(6),
+                'mime': match.group(7)
+            })
+    return requests
+
+
+def urls_import(path, method, headers, include):
+    """
+    imports urls from a newline delimited text file
+    returns list (of request objects)
+    """
+    requests = []
+    urls = reader(path, mode='lines')
+    for url in urls:
+        requests.append({
+            'url': url,
+            'method': method,
+            'headers': headers,
+            'data': include
+        })
+    return requests
+
+
+def request_import(path):
+    """
+    imports request from a raw request file
+    returns dict
+    """
+    return parse_request(reader(path))
+
+
+def importer(path, method, headers, include):
+    """
+    main importer function that calls other import functions
+    """
+    with open(path, 'r', encoding='utf-8') as file:
+        for line in file:
+            if line.startswith('<?xml'):
+                return burp_import(path)
+            elif line.startswith(('http://', 'https://')):
+                return urls_import(path, method, headers, include)
+            elif line.startswith(('GET', 'POST')):
+                return request_import(path)
+            return 'unknown'

arjun/core/prompt.py

@@ -2,6 +2,10 @@ import os
 import tempfile
 
 def prompt(default=None):
+    """
+    lets user paste input by opening a temp file in a text editor
+    returns str (content of tmp file)
+    """
     editor = 'nano'
     with tempfile.NamedTemporaryFile(mode='r+') as tmpfile:
         if default:
@@ -16,4 +20,4 @@ def prompt(default=None):
         else:
             os.waitpid(child_pid, 0)
             tmpfile.seek(0)
-            return tmpfile.read().strip()
+            return tmpfile.read().strip()

arjun/core/requester.py

@@ -0,0 +1,42 @@
+import json
+import time
+import random
+import requests
+import warnings
+
+import arjun.core.config as mem
+
+from arjun.core.utils import dict_to_xml
+
+warnings.filterwarnings('ignore') # Disable SSL related warnings
+
+def requester(request, payload={}):
+    """
+    central function for making http requests
+    returns str on error otherwise response object of requests library
+    """
+    if 'include' in request and request['include']:
+        payload.update(request['include'])
+    if mem.var['stable']:
+        mem.var['delay'] = random.choice(range(6, 12))
+    time.sleep(mem.var['delay'])
+    url = request['url']
+    if mem.var['kill']:
+        return 'killed'
+    try:
+        if request['method'] == 'GET':
+            response = requests.get(url, params=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
+        elif request['method'] == 'JSON':
+            if mem.var['include'] and '$arjun$' in mem.var['include']:
+                payload = mem.var['include'].replace('$arjun$', json.dumps(payload).rstrip('}').lstrip('{'))
+                response = requests.post(url, data=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
+            else:
+                response = requests.post(url, json=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
+        elif request['method'] == 'XML':
+            payload = mem.var['include'].replace('$arjun$', dict_to_xml(payload))
+            response = requests.post(url, data=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
+        else:
+            response = requests.post(url, data=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
+        return response
+    except Exception as e:
+        return str(e)

arjun/core/utils.py

@@ -4,13 +4,19 @@ import random
 import requests
 
 import concurrent.futures
+from dicttoxml import dicttoxml
 from urllib.parse import urlparse
 
-from plugins.otx import otx
+from arjun.core.prompt import prompt
-from plugins.wayback import wayback
+from arjun.core.importer import importer
-from plugins.commoncrawl import commoncrawl
+
+from arjun.plugins.otx import otx
+from arjun.plugins.wayback import wayback
+from arjun.plugins.commoncrawl import commoncrawl
+
+import arjun.core.config as mem
+from arjun.core.colors import info
 
-from core.colors import info
 
 def lcs(s1, s2):
     """
@@ -31,7 +37,7 @@ def lcs(s1, s2):
     return s1[x_longest - longest: x_longest]
 
 
-def extractHeaders(headers):
+def extract_headers(headers):
     """
     parses headers provided through command line
     returns dict
@@ -93,44 +99,45 @@ def stable_request(url, headers):
         return None
 
 
-def removeTags(html):
+def remove_tags(html):
     """
     removes all the html from a webpage source
     """
     return re.sub(r'(?s)<.*?>', '', html)
 
 
-def lineComparer(response1, response2):
+def diff_map(body_1, body_2):
     """
-    compares two webpage and finds the non-matching lines
+    creates a list of lines that are common between two multi-line strings
+    returns list
     """
-    response1 = response1.split('\n')
+    sig = []
-    response2 = response2.split('\n')
+    lines_1, lines_2 = body_1.split('\n'), body_2.split('\n')
-    num = 0
+    for line_1, line_2 in zip(lines_1, lines_2):
-    dynamicLines = []
+        if line_1 == line_2:
-    for line1, line2 in zip(response1, response2):
+            sig.append(line_1)
-        if line1 != line2:
+    return sig
-            dynamicLines.append(num)
-        num += 1
-    return dynamicLines
 
 
-def randomString(n):
+def random_str(n):
     """
     generates a random string of length n
     """
     return ''.join(str(random.choice(range(10))) for i in range(n))
 
 
-def getParams(include):
+def get_params(include):
     """
     loads parameters from JSON/query string
     """
     params = {}
     if include:
         if include.startswith('{'):
-            params = json.loads(str(include).replace('\'', '"'))
+            try:
-            return params
+                params = json.loads(str(include).replace('\'', '"'))
+                return params
+            except json.decoder.JSONDecodeError:
+                return {}
         else:
             cleaned = include.split('?')[-1]
             parts = cleaned.split('&')
@@ -143,6 +150,18 @@ def getParams(include):
     return params
 
 
+def create_query_string(params):
+    """
+    creates a query string from a list of parameters
+    returns str
+    """
+    query_string = ''
+    for param in params:
+        pair = param + '=' + random_str(4)
+        query_string += pair
+    return '?' + query_string
+
+
 def reader(path, mode='string'):
     """
     reads a file
@@ -225,3 +244,50 @@ def fetch_params(host):
         page += 1
     print('%s Progress: %i%%' % (info, 100), end='\r')
     return params
+
+
+def prepare_requests(args):
+    """
+    creates a list of request objects used by Arjun from targets given by user
+    returns list (of targs)
+    """
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:83.0) Gecko/20100101 Firefox/83.0',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate',
+        'Connection': 'close',
+        'Upgrade-Insecure-Requests': '1'
+    }
+    if type(headers) == bool:
+        headers = extract_headers(prompt())
+    elif type(headers) == str:
+        headers = extract_headers(headers)
+    if mem.var['method'] == 'JSON':
+        headers['Content-type'] = 'application/json'
+    if args.url:
+        params = get_params(args.include)
+        return {
+            'url': args.url,
+            'method': mem.var['method'],
+            'headers': headers,
+            'include': params
+        }
+    elif args.import_file:
+        return importer(args.import_file, mem.var['method'], headers, args.include)
+    return []
+
+
+def nullify(*args, **kwargs):
+    """
+    a function that does nothing
+    """
+    pass
+
+
+def dict_to_xml(dict_obj):
+    """
+    converts dict to xml string
+    returns str
+    """
+    return dicttoxml(dict_obj, root=False, attr_type=False).decode('utf-8')

arjun/plugins/heuristic.py

@@ -0,0 +1,26 @@
+import re
+
+from arjun.core.utils import extract_js
+
+def is_not_junk(string):
+    return re.match(r'^[A-Za-z0-9_]+$', string)
+
+def insert_words(words, wordlist, found):
+    if words:
+        for var in words:
+            if var not in found and is_not_junk(var):
+                found.append(var)
+                if var in wordlist:
+                    wordlist.remove(var)
+                wordlist.insert(0, var)
+
+def heuristic(response, wordlist):
+    found = []
+    inputs = re.findall(r'(?i)<input.+?name=["\']?([^"\'\s>]+)', response)
+    insert_words(inputs, wordlist, found)
+    for script in extract_js(response):
+        empty_vars = re.findall(r'([^\s!=<>]+)\s*=\s*[\'"`][\'"`]', script)
+        insert_words(empty_vars, wordlist, found)
+        map_keys = re.findall(r'([^\'"]+)[\'"]:\s?[\'"]', script)
+        insert_words(map_keys, wordlist, found)
+    return found

core/__init__.py

@@ -1 +0,0 @@
-

core/config.py

@@ -1 +0,0 @@
-var = {}

core/importer.py

@@ -1,64 +0,0 @@
-import re
-from core.utils import reader, parse_request
-
-burp_regex = re.compile(r'''(?m)^    <url><!\[CDATA\[(.+?)\]\]></url>
-    <host ip="[^"]*">[^<]+</host>
-    <port>[^<]*</port>
-    <protocol>[^<]*</protocol>
-    <method><!\[CDATA\[(.+?)\]\]></method>
-    <path>.*</path>
-    <extension>(.*)</extension>
-    <request base64="(?:false|true)"><!\[CDATA\[([\s\S]+?)]]></request>
-    <status>([^<]*)</status>
-    <responselength>([^<]*)</responselength>
-    <mimetype>([^<]*)</mimetype>''')
-
-
-def burp_import(path):
-	requests = []
-	content = reader(path)
-	matches = re.finditer(burp_regex, content)
-	for match in matches:
-		request = parse_request(match.group(4))
-		headers = request['headers']
-		if match.group(7) in ('HTML', 'JSON'):
-			requests.append({
-				'url': match.group(1),
-				'method': match.group(2),
-				'extension': match.group(3),
-				'headers': headers,
-				'include': request['data'],
-				'code': match.group(5),
-				'length': match.group(6),
-				'mime': match.group(7)
-			})
-	return requests
-
-
-def urls_import(path, method, headers, include):
-	requests = []
-	urls = reader(path, mode='lines')
-	for url in urls:
-		requests.append({
-			'url': url,
-			'method': method,
-			'headers': headers,
-			'data': include
-		})
-	return requests
-
-
-def request_import(path):
-	return parse_request(reader(path))
-
-
-def importer(path, method, headers, include):
-    with open(path, 'r', encoding='utf-8') as file:
-        for line in file:
-            if line.startswith('<?xml'):
-                return burp_import(path)
-            elif line.startswith(('http://', 'https://')):
-                return urls_import(path, method, headers, include)
-            elif line.startswith(('GET', 'POST')):
-                return request_import(path)
-            return 'unknown'

core/requester.py

@@ -1,33 +0,0 @@
-import re
-import json
-import time
-import random
-import requests
-import warnings
-
-import core.config as mem
-
-warnings.filterwarnings('ignore') # Disable SSL related warnings
-
-def requester(request, payload={}):
-    if 'include' in request and request['include']:
-        payload.update(request['include'])
-    if mem.var['stable']:
-        mem.var['delay'] = random.choice(range(6, 12))
-    time.sleep(mem.var['delay'])
-    url = request['url']
-    if 'Host' not in request['headers']:
-        this_host = re.search(r'https?://([^/]+)', url).group(1)
-        request['headers']['Host'] = this_host.split('@')[1] if '@' in this_host else this_host
-    if mem.var['kill']:
-        return 'killed'
-    try:
-        if request['method'] == 'GET':
-            response = requests.get(url, params=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
-        elif request['method'] == 'JSON':
-            response = requests.post(url, json=json.dumps(payload), headers=request['headers'], verify=False, timeout=mem.var['timeout'])
-        else:
-            response = requests.post(url, data=payload, headers=request['headers'], verify=False, timeout=mem.var['timeout'])
-        return response
-    except Exception as e:
-        return str(e)

plugins/__init__.py

@@ -1 +0,0 @@
-

plugins/heuristic.py

@@ -1,35 +0,0 @@
-import re
-
-from core.utils import extract_js
-
-def is_not_junk(string):
-    return re.match(r'^[A-Za-z0-9_]+$', string)
-
-def heuristic(response, paramList):
-    found = []
-    inputs = re.findall(r'(?i)<input.+?name=["\']?([^"\'\s>]+)', response)
-    if inputs:
-        for inpName in inputs:
-            if inpName not in found and is_not_junk(inpName):
-                if inpName in paramList:
-                    paramList.remove(inpName)
-                found.append(inpName)
-                paramList.insert(0, inpName)
-    for script in extract_js(response):
-        emptyJSvars = re.findall(r'([^\s!=<>]+)\s*=\s*[\'"`][\'"`]', script)
-        if emptyJSvars:
-            for var in emptyJSvars:
-                if var not in found and is_not_junk(var):
-                    found.append(var)
-                    if var in paramList:
-                        paramList.remove(var)
-                    paramList.insert(0, var)
-        arrayJSnames = re.findall(r'([^\'"]+)[\'"]:\s?[\'"]', script)
-        if arrayJSnames:
-            for var in arrayJSnames:
-                if var not in found and is_not_junk(var):
-                    found.append(var)
-                    if var in paramList:
-                        paramList.remove(var)
-                    paramList.insert(0, var)
-    return found

requirements.txt

@@ -1 +0,0 @@
-requests

setup.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import io
+from setuptools import setup, find_packages
+from os import path
+this_directory = path.abspath(path.dirname(__file__))
+with io.open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
+    desc = f.read()
+
+setup(
+    name='arjun',
+    version=__import__('arjun').__version__,
+    description='HTTP parameter discovery suite',
+    long_description=desc,
+    long_description_content_type='text/markdown',
+    author='Somdev Sangwan',
+    author_email='s0md3v@gmail.com',
+    license='GNU General Public License v3 (GPLv3)',
+    url='https://github.com/s0md3v/Arjun',
+    download_url='https://github.com/s0md3v/Arjun/archive/v%s.zip' % __import__('arjun').__version__,
+    zip_safe=False,
+    packages=find_packages(),
+    package_data={'arjun': ['db/*']},
+    install_requires=[
+        'requests',
+        'dicttoxml'
+    ],
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Information Technology',
+        'Operating System :: OS Independent',
+        'Topic :: Security',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Programming Language :: Python :: 3.4',
+    ],
+    entry_points={
+        'console_scripts': [
+            'arjun = arjun.__main__:main'
+        ]
+    },
+    keywords=['arjun', 'bug bounty', 'http', 'pentesting', 'security'],
+)