discourse/app/jobs/scheduled/poll_feed.rb

#
# Creates and Updates Topics based on an RSS or ATOM feed.
#
require 'digest/sha1'
require_dependency 'post_creator'
require_dependency 'post_revisor'
require 'open-uri'

module Jobs
  class PollFeed < Jobs::Scheduled
    every 5.minutes

    sidekiq_options retry: false

    def execute(args)
      poll_feed if SiteSetting.feed_polling_enabled? &&
                   SiteSetting.feed_polling_url.present? &&
                   not_polled_recently?
    end

    def feed_key
      "feed-modified:#{Digest::SHA1.hexdigest(SiteSetting.feed_polling_url)}"
    end

    def poll_feed
      feed = Feed.new
      import_topics(feed.topics)
    end

    private

    def not_polled_recently?
      $redis.set(
        'feed-polled-recently',
        "1",
        ex: SiteSetting.feed_polling_frequency_mins.minutes - 10.seconds,
        nx: true
      )
    end

    def import_topics(feed_topics)
      feed_topics.each do |topic|
        import_topic(topic)
      end
    end

    def import_topic(topic)
      if topic.user
        TopicEmbed.import(topic.user, topic.url, topic.title, CGI.unescapeHTML(topic.content.scrub))
      end
    end

    class Feed
      require 'simple-rss'

      if SiteSetting.embed_username_key_from_feed.present?
        SimpleRSS.item_tags << SiteSetting.embed_username_key_from_feed.to_sym
      end

      def initialize
        @feed_url = SiteSetting.feed_polling_url
        @feed_url = "http://#{@feed_url}" if @feed_url !~ /^https?\:\/\//
      end

      def topics
        feed_topics = []

        rss = fetch_rss
        return feed_topics unless rss.present?

        rss.items.each do |i|
          current_feed_topic = FeedTopic.new(i)
          feed_topics << current_feed_topic if current_feed_topic.content
        end

        return feed_topics
      end

      private

      def fetch_rss
        SimpleRSS.parse open(@feed_url, allow_redirections: :all)
      rescue OpenURI::HTTPError, SimpleRSSError
        nil
      end

    end

    class FeedTopic
      def initialize(article_rss_item)
        @article_rss_item = article_rss_item
      end

      def url
        link = @article_rss_item.link
        if url?(link)
          return link
        else
          return @article_rss_item.id
        end
      end

      def content
        @article_rss_item.content.try(:force_encoding, "UTF-8").try(:scrub) || @article_rss_item.description.try(:force_encoding, "UTF-8").try(:scrub)
      end

      def title
        @article_rss_item.title.force_encoding("UTF-8").scrub
      end

      def user
        author_user || default_user
      end

      private

      def url?(link)
        if link.blank? || link !~ /^https?\:\/\//
          return false
        else
          return true
        end
      end

      def author_username
        begin
          @article_rss_item.send(SiteSetting.embed_username_key_from_feed.to_sym)
        rescue
          nil
        end
      end

      def default_user
        find_user(SiteSetting.embed_by_username.downcase)
      end

      def author_user
        return nil if !author_username.present?

        find_user(author_username)
      end

      def find_user(user_name)
        User.where(username_lower: user_name).first
      end

    end

  end

end