#!/usr/bin/env ruby # get SPF records from list of domains # # (c) Tatsuya Mori require 'rubygems' require 'resolv' require 'thread' require 'thwait' require 'progressbar' class Domain2SPF def initialize(file) # @dns = Resolv::DNS.new() @dns = Resolv::DNS.new({:nameserver=>['127.0.0.1']}) # @dns = Resolv::DNS.new({:nameserver=>['10.2.0.30']}) @queue = Queue.new @num_threads = 150 @threads = [] @mutex = Mutex.new @counter = 0 GC.start warn "now reading" cnt = parse_data(file) warn "done" @pbar = ProgressBar.new("resolving",cnt) warn "now resolving" multi_lookup ThreadsWait::all_waits(*@threads) end def parse_data(file) seen = Hash.new cnt = 0 f = open(file) while line = f.gets begin data = line.chomp.split(/\s+/) if data[0] domain = data[0].sub(/^\./,"").downcase unless seen[domain] @queue.push(domain) cnt += 1 seen[domain] = true end end rescue # for invalid chacaters... warn "invalid character in #{line}: skipping..." next end end f.close return cnt end def multi_lookup @num_threads.times { @queue.push(nil) thread = Thread.new do while domain = @queue.pop txt = get_txt(domain) ips = [] quals = [] begin ips = (txt)? get_ips_from_txt(domain, txt, [], [], 0) : [] quals = (txt)? get_qual_from_txt(domain, txt) : {"NONE"=>nil} rescue next end @mutex.synchronize do if !txt puts "NONE_TXT:\t#{domain}" elsif txt !~ /spf/ puts "NONE_SPF:\t#{domain}" elsif !ips puts "NONE_IP:\t#{domain}\t\"#{txt}\"\t#{quals.keys.join(",")}" else puts "SPF_OK:\t#{domain}\t\"#{txt}\"\t#{quals.keys.join(",")}\t#{ips.join(" ")}" end @pbar.inc end end end thread.abort_on_exception = true @threads.push(thread) } end def get_qual_from_txt(domain, txt) q = {} txt =~ /([\-+~\?]all)/ qual = $1 q[qual] = true if qual includes = txt.scan(/include:[^\s]+/) redirects = txt.scan(/redirect=[^\s]+/) if includes includes.map{|x| x.gsub(/include:/,"")}.each do |d| if d != domain t = get_txt(d) if t t =~ /([\-+~\?]all)/ qual = $1 q[qual] = true if qual end end end end if redirects redirects.map{|x| x.gsub(/redirect=/,"")}.each do |d| if d != domain t = get_txt(d) if t t =~ /([\-+~\?]all)/ qual = $1 q[qual] = true if qual end end end end return q end def get_ips_from_txt(domain, txt, ip_list, dom_list, cnt) return ip_list unless txt return ip_list unless domain tmp = get_ip_list(txt, domain) tmp.each {|x| ip_list << x unless ip_list.include?(x)} # detect loop if cnt > 5 warn "#{domain} could have recursive loop...." return ip_list else cnt += 1 end #----- include fileds ----- # includes = txt.scan(/include:[^\s]+/) if includes includes.map{|x| x.gsub(/include:/,"")}.each do |d| if !dom_list.include?(d) dom_list << d t = get_txt(d) if t get_ips_from_txt(domain, t, ip_list, dom_list, cnt).each do |x| ip_list << x unless ip_list.include?(x) end end end end end #----- redirect fileds ----- # redirects = txt.scan(/redirect=[^\s]+/) if redirects redirects.map{|x| x.gsub(/redirect=/,"")}.each do |d| if d != domain && !dom_list.include?(d) dom_list << d t = get_txt(d) if t get_ips_from_txt(domain, t, ip_list, dom_list, cnt).each do |x| ip_list << x unless ip_list.include?(x) end end end end end return ip_list end def get_ip_list(txt, domain) ip_list = [] #----- ip4 field ----- # # ips = txt.scan(/ip4:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/*\d*/) tmp = txt.scan(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/*\d*/) tmp.each {|ip| ip_list << ip unless ip_list.include?(ip)} #----- A filed ----- # if txt =~ /\s[+]*a\s/ get_a(domain).each {|ip| ip_list << ip} end #----- A fileds ----- # domains = txt.scan(/a:[^\s]+/) domains.map {|x| x.gsub(/a:/,"")}.each do |d| get_a(d).each {|ip| ip_list << ip} end #----- MX filed ----- # if txt =~ /\s[+]*mx\/*\d*\s/ get_mx(domain).each {|m| get_a(m).each {|ip| ip_list << ip}} end #----- PTR filed ----- # # Do we need this? In general, we do not know the IP address of a client. #----- MX fileds ----- # mxs = txt.scan(/mx:[^\s]+/) mxs.map {|x| x.gsub(/mx:/,"")}.each do |d| get_mx(d).each {|m| get_a(m).each {|ip| ip_list << ip}} end return ip_list end def get_mx(domain) mx_records = [] return mx_records unless domain mx = @dns.getresources(domain, Resolv::DNS::Resource::IN::MX) mx.each {|r| mx_records << r.exchange.to_s} return mx_records end def get_a(domain) a_records = [] return a_records unless domain a = @dns.getaddresses(domain) a.each {|addr| a_records << addr.to_s} return a_records end def get_txt(domain) return nil unless domain records = [] txt = @dns.getresources(domain, Resolv::DNS::Resource::IN::TXT) if txt txt.each do |r| records << r.strings end return records.join(" ").gsub(/\t/," ").gsub(/\n/," ") else return nil end end end def main file = ARGV.shift exit if file == nil Domain2SPF.new(file) end main