rss-darenet.py

#--depends-on config
#--depends-on shorturl

import difflib, hashlib, time, urllib.parse
from src import ModuleManager, utils
import feedparser

RSS_INTERVAL = 60 # 1 minute

SETTING_BIND = utils.Setting("rss-bindhost",
    "Which local address to bind to for RSS requests", example="127.0.0.1")

@utils.export("botset", utils.IntSetting("rss-interval",
    "Interval (in seconds) between RSS polls", example="120"))
@utils.export("channelset", utils.BoolSetting("rss-shorten",
    "Whether or not to shorten RSS urls"))
@utils.export("serverset", SETTING_BIND)
@utils.export("channelset", SETTING_BIND)
class Module(ModuleManager.BaseModule):
    _name = "RSS"
    def on_load(self):
        self.timers.add("rss-feeds", self._timer,
            self.bot.get_setting("rss-interval", RSS_INTERVAL))

    def _format_entry(self, server, feed_title, entry, shorten):
        title = utils.parse.line_normalise(utils.http.strip_html(
            entry["title"]))

        author = entry.get("author", None)
        author = " by %s" % author if author else ""

        link = entry.get("link", None)
        if shorten:
            try:
                link = self.exports.get("shorturl")(server, link)
            except:
                pass
        link = " - %s" % link if link else ""

        feed_title_str = "%s: " % feed_title if feed_title else ""

        return "%s%s%s%s" % (feed_title_str, title, author, link)

    def _timer(self, timer):
        start_time = time.monotonic()
        self.log.trace("Polling RSS feeds")

        timer.redo()
        hook_settings = self.bot.database.channel_settings.find_by_setting(
            "rss-hooks")

        all_urls = []
        for server_id, channel_name, urls in hook_settings:
            server = self.bot.get_server_by_id(server_id)
            if server and channel_name in server.channels:
                channel = server.channels.get(channel_name)
                for url in urls:
                    bindhost = channel.get_setting("rss-bindhost",
                        server.get_setting("rss-bindhost", None))

                    all_urls.append((server, channel, url, bindhost))

        https = set([])
        hooks = {}
        all_urls.sort(lambda u: u[2].startswith("https:"), reverse=True)
        for server, channel, url, bindhost in all_urls:
            parts = urllib.parse.urlparse(url)
            if parts.netloc.startswith("www."):
                url = url.replace("www.", "", 1)

            if parts.scheme == "https":
                https.add(parts.netloc)
            elif parts.scheme == "http" and parts.netloc in https:
                url = url.replace("http:", "https:", 1)

            key = (url, bindhost)
            if not key in hooks:
                hooks[key] = []

            hooks[key].append((server, channel))

        if not hooks:
            return

        requests = []
        for url, bindhost in hooks.keys():
            requests.append(utils.http.Request(url, id=f"{url} {bindhost}",
                bindhost=bindhost))

        pages = utils.http.request_many(requests)

        for (url, bindhost), channels in hooks.items():
            key = f"{url} {bindhost}"
            if not key in pages:
                # async url get failed
                continue

            try:
                data = pages[key].decode()
            except Exception as e:
                self.log.error("Failed to decode rss URL %s", [url],
                    exc_info=True)
                continue

            feed = feedparser.parse(data)
            feed_title = feed["feed"].get("title", None)
            max_ids = len(feed["entries"])*10

            for server, channel in channels:
                seen_ids = channel.get_setting("rss-seen-ids-%s" % url, [])
                valid = 0
                for entry in feed["entries"][::-1]:
                    entry_id, entry_id_hash = self._get_id(entry)
                    if entry_id_hash in seen_ids or entry_id in seen_ids:
                        continue

                    if valid == 3:
                        continue
                    valid += 1

                    shorten = channel.get_setting("rss-shorten", False)
                    output = self._format_entry(server, feed_title, entry,
                        shorten)

                    self.events.on("send.stdout").call(target=channel,
                        module_name="RSS", server=server, message=output)
                    seen_ids.append(entry_id_hash)

                if len(seen_ids) > max_ids:
                    seen_ids = seen_ids[len(seen_ids)-max_ids:]
                channel.set_setting("rss-seen-ids-%s" % url, seen_ids)

        total_milliseconds = (time.monotonic() - start_time) * 1000
        self.log.trace("Polled RSS feeds in %fms", [total_milliseconds])

    def _get_id(self, entry):
        entry_id = entry.get("id", entry["link"])
        entry_id_hash = hashlib.sha1(entry_id.encode("utf8")).hexdigest()
        return entry_id, "sha1:%s" % entry_id_hash

    def _get_entries(self, url, max: int=None):
        try:
            feed = feedparser.parse(utils.http.request(url).data)
        except Exception as e:
            self.log.warn("failed to parse RSS %s", [url], exc_info=True)
            feed = None
        if not feed or not feed["feed"]:
            return None, None

        entry_ids = []
        for entry in feed["entries"]:
            entry_ids.append(entry.get("id", entry["link"]))
        return feed["feed"].get("title", None), feed["entries"][:max]

    @utils.hook("received.command.rss", min_args=1, channel_only=True)
    def rss(self, event):
        """
        :help: Modify RSS/Atom configuration for the current channel
        :usage: list
        :usage: add <url>
        :usage: remove <url>
        :permission: rss
        """
        changed = False
        message = None

        rss_hooks = event["target"].get_setting("rss-hooks", [])

        subcommand = event["args_split"][0].lower()
        if subcommand == "list":
            event["stdout"].write("RSS hooks: %s" % ", ".join(rss_hooks))
        elif subcommand == "add":
            if not len(event["args_split"]) > 1:
                raise utils.EventError("Please provide a URL")

            url = utils.http.url_sanitise(event["args_split"][1])
            if url in rss_hooks:
                raise utils.EventError("That URL is already being watched")

            title, entries = self._get_entries(url)
            if entries == None:
                raise utils.EventError("Failed to read feed")

            seen_ids = [self._get_id(e)[1] for e in entries]
            event["target"].set_setting("rss-seen-ids-%s" % url, seen_ids)

            rss_hooks.append(url)
            changed = True
            message = "Added RSS feed"
        elif subcommand == "remove":
            if not len(event["args_split"]) > 1:
                raise utils.EventError("Please provide a URL")

            url = utils.http.url_sanitise(event["args_split"][1])
            if not url in rss_hooks:
                matches = difflib.get_close_matches(url, rss_hooks, cutoff=0.5)
                if matches:
                    raise utils.EventError("Did you mean %s ?" % matches[0])
                else:
                    raise utils.EventError("I'm not watching that URL")
            rss_hooks.remove(url)
            changed = True
            message = "Removed RSS feed"
        elif subcommand == "read":
            url = None
            if not len(event["args_split"]) > 1:
                if len(rss_hooks) == 1:
                    url = rss_hooks[0]
                else:
                    raise utils.EventError("Please provide a url")
            else:
                url = event["args_split"][1]

            title, entries = self._get_entries(url)
            if not entries:
                raise utils.EventError("Failed to get RSS entries")

            shorten = event["target"].get_setting("rss-shorten", False)
            out = self._format_entry(event["server"], title, entries[0],
                shorten)
            event["stdout"].write(out)
        else:
            raise utils.EventError("Unknown subcommand '%s'" % subcommand)

        if changed:
            if rss_hooks:
                event["target"].set_setting("rss-hooks", rss_hooks)
            else:
                event["target"].del_setting("rss-hooks")
            event["stdout"].write(message)