def set_headers(uri, request, cur_page)
if @keep_alive
request.add_field('Connection', 'keep-alive')
request.add_field('Keep-Alive', keep_alive_time.to_s)
else
request.add_field('Connection', 'close')
end
request.add_field('Accept-Encoding', 'gzip,identity')
request.add_field('Accept-Language', 'en-us,en;q0.5')
request.add_field('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
unless @cookie_jar.empty?(uri)
cookies = @cookie_jar.cookies(uri)
cookie = cookies.length > 0 ? cookies.join("; ") : nil
if log
cookies.each do |c|
log.debug("using cookie: #{c}")
end
end
request.add_field('Cookie', cookie)
end
unless cur_page.uri.nil?
request.add_field('Referer', cur_page.uri.to_s)
end
request.add_field('User-Agent', @user_agent) if @user_agent
if @conditional_requests
if( (page = visited_page(uri)) && page.response['Last-Modified'] )
request.add_field('If-Modified-Since', page.response['Last-Modified'])
end
end
if( @auth_hash[uri.host] )
case @auth_hash[uri.host]
when :basic
request.basic_auth(@user, @password)
when :digest
@digest_response ||= nil
@digest_response = self.gen_auth_header(uri,request,@digest) if @digest
request.add_field('Authorization', @digest_response) if @digest_response
end
end
request
end