def set_headers(uri, request, options)
unless options.is_a? Hash
cur_page = options
else
raise ArgumentError.new("cur_page must be specified") unless cur_page = options[:page]
headers = options[:headers]
end
if @keep_alive
request.add_field('Connection', 'keep-alive')
request.add_field('Keep-Alive', keep_alive_time.to_s)
else
request.add_field('Connection', 'close')
end
request.add_field('Accept-Encoding', 'gzip,identity')
request.add_field('Accept-Language', 'en-us,en;q=0.5')
request.add_field('Host', uri.host)
request.add_field('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
unless @cookie_jar.empty?(uri)
cookies = @cookie_jar.cookies(uri)
cookie = cookies.length > 0 ? cookies.join("; ") : nil
if log
cookies.each do |c|
log.debug("using cookie: #{c}")
end
end
request.add_field('Cookie', cookie)
end
unless cur_page.uri.nil?
request.add_field('Referer', cur_page.uri.to_s)
end
request.add_field('User-Agent', @user_agent) if @user_agent
if @conditional_requests
if( (page = visited_page(uri)) && page.response['Last-Modified'] )
request.add_field('If-Modified-Since', page.response['Last-Modified'])
end
end
if( @auth_hash[uri.host] )
case @auth_hash[uri.host]
when :basic
request.basic_auth(@user, @password)
when :digest
@digest_response = self.gen_auth_header(uri,request,@digest) if @digest
request.add_field('Authorization', @digest_response) if @digest_response
end
end
if headers
headers.each do |k,v|
case k
when :etag then request.add_field("ETag", v)
when :if_modified_since then request.add_field("If-Modified-Since", v)
else
raise ArgumentError.new("unknown header symbol #{k}") if k.is_a? Symbol
request.add_field(k,v)
end
end
end
request
end