From d66f994f71bff1bef329ea02c62e3e920aabcd72 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 11 Oct 2014 11:14:47 -0400 Subject: [PATCH] Added rate limiting option. --- xsscrapy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/xsscrapy.py b/xsscrapy.py index f2e5c99..44cb45c 100755 --- a/xsscrapy.py +++ b/xsscrapy.py @@ -15,6 +15,7 @@ def get_args(): parser.add_argument('-u', '--url', help="URL to scan; -u http://example.com") parser.add_argument('-l', '--login', help="Login name; -l danmcinerney") parser.add_argument('-p', '--password', help="Password; -p pa$$w0rd") + parser.add_argument('-r', '--rate', help="Rate in requests per minute") args = parser.parse_args() return args @@ -22,8 +23,12 @@ args = get_args() url = args.url user = args.login password = args.password - +rate = args.rate +if rate is not None: + delay = 60 / float(rate) +else: + delay = 0 try: - execute(['scrapy', 'crawl', 'xsscrapy', '-a', 'url=%s' % url, '-a', 'user=%s' % user, '-a', 'pw=%s' % password]) + execute(['scrapy', 'crawl', 'xsscrapy', '-a', 'url=%s' % url, '-a', 'user=%s' % user, '-a', 'pw=%s' % password, '-s', "DOWNLOAD_DELAY=%s" % delay]) except KeyboardInterrupt: sys.exit()