# File lib/webrobots/robotstxt.rb, line 606 def initialize(agentlines, rulelines) @patterns = agentlines.map { |agentline| agentline.pattern } @acls = [] @delay = nil @options = {} rulelines.each { |ruleline| case ruleline when AccessControlLine @acls << ruleline when CrawlDelayLine @delay = ruleline.delay else @options[ruleline.token.downcase] = ruleline.value end } if rulelines @acls.replace @acls.sort_by { |x| [-x.value.length, x.is_a?(AllowLine) ? -1 : 0] } end
# File lib/webrobots/robotstxt.rb, line 638 def allow?(request_uri) @acls.each { |acl| if acl.match?(request_uri) return acl.allow? end } return true end
# File lib/webrobots/robotstxt.rb, line 634 def default? @patterns.include?(%r/) end
# File lib/webrobots/robotstxt.rb, line 628 def match?(user_agent) @patterns.any? { |pattern| pattern.match(user_agent) } end