Files
lib/spider.rb
lib/spider/included_in_memcached.rb
lib/spider/included_in_redis.rb
lib/spider/next_urls_in_sqs.rb
lib/spider/robot_rules.rb
lib/spider/spider_instance.rb