#!/usr/bin/env python2.2

"""
xpath2rss by Mark Nottingham <mnot@pobox.com>

XPath -> RSS1.0 Web Scraper.

xpath2rss is Yet Another HTML->RSS scraper. This one's different in that
instead of using regular expressions, as most do, it uses XPath.

Why XPath?

The first reason to use XPath is to learn it; XPath is an excellent tool for
working with XML. Currently, it's used in XSLT and not too many other
places.

Secondly, XPath is smarter than regex because XPath is aware of XML's (and
therefore HTML's) syntax; this makes it a more natural fit for scraping HTML
and XML, and should make it more reliable.


TODO:
- look for w3c-synd profile: http://www.w3.org/2000/08/w3c-synd/
"""

# THIS SOFTWARE IS SUPPLIED WITHOUT WARRANTY OF ANY KIND, AND MAY BE
# COPIED, MODIFIED OR DISTRIBUTED IN ANY WAY, AS LONG AS THIS NOTICE
# AND ACKNOWLEDGEMENT OF AUTHORSHIP REMAIN.


import os, sys, traceback, StringIO, cgi, robotparser
from urlparse import urljoin, urlparse, urlunparse
from xml import sax
from xml.xpath import Evaluate
import RSS   # http://www.mnot.net/python/RSS.py
import HtmlDom   # http://www.mnot.net/python/HtmlDom.py


__version__ = "0.7"
x2r = 'http://www.mnot.net/xpath2rss/'   # namespace
ns = RSS.ns
UAString = "%s?%s" % (x2r, __version__)
HtmlDom.UAString = UAString


def main(config_file, out_file, console, run_ids, timeout=240):
    """
    Fetch configured URIs and run XPaths to generate
    channels.
    """
    
    dh = configParser()
    p = sax.sax2exts.make_parser()
    p.setContentHandler(dh)
    p.setFeature(sax.handler.feature_namespaces, 1)
    try:
        p.parse(config_file)
    except IOError, why:
        sys.stderr.write("ERROR - configuration - %s\n" % why)
        sys.exit(1)
    out = None

    for info, md, paths in dh.channels:
        rp = robotparser.RobotFileParser()
        channel = RSS.CollectionChannel()
        channel.setMD((ns.rss10, 'channel'), md)
        if run_ids and info[(None, 'id')] not in run_ids:
            continue
        console.write("channel: %s\n" % info[(None, 'id')])

        # check robots.txt
        rp.set_url(urljoin(md[(ns.rss10, 'link')], '/robots.txt'))
        rp.read()
        if not rp.can_fetch(UAString, md[(ns.rss10, 'link')]):
            sys.stderr.write("  ERROR - %s - robot excluded; skipping\n" % (
              info[(None, 'id')]))
            continue
 
        # fetch the DOM
        try:
            dom = HtmlDom.fetch(str(md[(ns.rss10, 'link')]), timeout)
        except Exception, why:
            sys.stderr.write("  ERROR - %s - %s\n" % (
              info[(None, 'id')], why))
            continue
            
        # get the base href
        if dom.isHtml():
            base_href = getdata(Evaluate('//BASE/@HREF', dom.documentElement))
        else:
            base_href = getdata(Evaluate('//base/@href', dom.documentElement)) 
        if not base_href:
            base_href = str(md[(ns.rss10, 'link')])
            
        # look for a declared feed
        if dom.isHtml():
            doc_feed = getdata(Evaluate('/html/head/link[@type="application/rss+xml"]/@href', ###
              dom.documentElement))
        else:
            doc_feed = getdata(Evaluate('/HTML/HEAD/LINK[@TYPE="application/rss+xml"]/@HREF',
              dom.documentElement))
        if doc_feed:
            sys.stderr.write("""\
  ERROR - %s - authoritative feed at:
          <%s>; skipping
""" % (info[(None, 'id')], urljoin(base_href, document_feed)))
            continue 
        found = Evaluate(info[(x2r, 'item_path')], dom.documentElement)
        
        for item in found:
            tmp_item = {}
            try:
                for name, path in paths.items():
                    tmp_item[name] = getdata(Evaluate(path, item))

            except:
                sys.stderr.write(
                  "  %s: ERROR - xpath\n" % info[(None, 'id')])
                traceback.print_exc(limit=2)
                continue
            if tmp_item.has_key((ns.rss10, 'link')):
                tmp_item[(ns.rss10, 'link')] = \
                  urljoin(base_href, tmp_item[(ns.rss10, 'link')])
                if info.has_key('rm_args'):
                    tmp_item[(ns.rss10, 'link')] = \
                      rm_args(info['rm_args'], 
                      tmp_item[(ns.rss10, 'link')])
                channel.addItem(tmp_item)
                        
        console.write("  found %s items\n" % (len(found)))
        if len(found) == 0:
            sys.stderr.write(
                "  WARNING - %s - no items found\n" % info[(None, 'id')])

        if info.get((None, 'reverse'), None):   ### assumes channel isn't persistent
            console.write("  reversing item order")
            channel[(ns.rss10, 'items')].reverse()

        if out_file:
            if out_file == '-':
                out = sys.stdout
            elif out is None:
                out = open(out_file, 'w')
        else:
            out = open(os.path.join(dh.output_base, info[(None, 'id')]), 'w')
        out.write(str(channel))
        if not out_file:
            out.close()

def getdata(inp):
    o = []
    for i in inp:
        if i.__nodeName == '#text':
            o.append(i._get_data())
        else:
            o.append(i._get_value())
    return " ".join(o).strip()
    
    
def rm_args(args, uri):
    del_args = args.split()
    (scheme, authority, path, params, query, fragment) = urlparse(uri)
    ql = cgi.parse_qs(query)
    q = []
    for name, values in ql.items():
        if not name in del_args:
            for value in values:
                q.append("%s=%s" % (name, value))
    query = '&'.join(q)
    return urlunparse((scheme, authority, path, params, query, fragment))   


class configParser(sax.handler.ContentHandler):
    """xpath2rss configuration file parser."""
    def __init__(self):
        sax.handler.ContentHandler.__init__(self)
        self._context = []
        self._tmp_info = {}
        self._tmp_md = {}
        self._tmp_paths = {}
        self._tmp_path = ''
        self._tmp_buf = ''
        self.channels = []
        self.output_base = ''

    def startElementNS(self, name, qname, attrs):
        self._context.append(name)
        if name == (x2r, 'channels'):  # top-level element
            self.output_base = attrs.get((None, 'output_base'), None)
        elif name == (x2r, 'channel'):  # new channel
            self._tmp_info = attrs._attrs
            self._tmp_paths = {}
            self._tmp_buf = ''
        elif name == (x2r, 'item'):
            self._tmp_info[(x2r, 'item_path')] = attrs[(None, 'path')]
        elif len(self._context) > 1 and self._context[-2] == (x2r, 'item'):
            self._tmp_path = attrs[(None, 'path')]
            
    def endElementNS(self, name, qname):
        if (x2r, 'channel') in self._context:
            if self._context[-1] == (x2r, 'channel'):  # end of a channel
                self.channels.append(
                  (self._tmp_info, self._tmp_md, self._tmp_paths))
                self._tmp_info, self._tmp_md, self_tmp_item = {}, {}, {}
            elif self._context[-2] == (x2r, 'channel') and \
              name != (x2r, 'item'):  # channel metadata
                self._tmp_md[name] = self._tmp_buf.strip()
            elif self._context[-2] == (x2r, 'item'):  # item path
                self._tmp_paths[name] = self._tmp_path.strip()
        self._tmp_buf = ''
        self._tmp_path = ''
        self._context.pop()

    def characters(self, content):
        self._tmp_buf = self._tmp_buf + content

    def ignoreableWhitespace(self, content):
        pass


def usage():
    sys.stderr.write("""\

Usage: %s [-h] [-q] [-c channel]+ [-o out_file] config_file
       c channel id to run (may be invoked multiple times)
       h this help message
       o output to specified file; use '-' for stdout
       q quiet - only prints errors

the 'o' option will overwrite channels to the file specified, in turn;
generally, it should be used in conjunction with the 'c' option.

""" % sys.argv[0])
    sys.exit(1)


if __name__ == '__main__':
    import getopt
    opthash = {}
    try:
        (optlist, args) = getopt.getopt(sys.argv[1:], 'c:ho:q')
    except getopt.error:
        usage()
    try:
        assert len(args) == 1
    except:
        usage()
    for (flag, value) in optlist:
        try:
            opthash[flag].append(value)
        except KeyError:
            opthash[flag] = [value]
    # help?
    if opthash.has_key('-h'):
        usage()
    # quiet?
    if opthash.has_key('-q'):
        console = StringIO.StringIO()
    else:
        console = sys.stdout
    # channel?
    run_ids = None
    if opthash.has_key('-c'):
        run_ids = opthash['-c']
    # out?
    out_file = None
    if opthash.has_key('-o'):
        out_file = opthash['-o'][0]
    
    main(args[0], out_file, console, run_ids)
