Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Format .thor files (#23059)

This commit is contained in:
Jarek Radosz 2023-08-10 13:59:13 +02:00 committed by GitHub
parent 5e72f2a342
commit 8bebd8fd99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 237 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/linting.yml vendored
View File

@ -73,7 +73,7 @@ jobs:
if: ${{ !cancelled() }}
run: |
set -E
bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake')
bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake') $(git ls-files '*.thor')
- name: ESLint (core)
if: ${{ !cancelled() }}

6
lib/tasks/backfill.thor
View File

@ -4,13 +4,11 @@ class Backfill < Thor
desc "link_titles", "Backfills link titles"
def link_titles
require './config/environment'
require "./config/environment"
topic_links = TopicLink.where(crawled_at: nil, internal: false)
puts "Enqueueing Topic Links: #{topic_links.count} links found."
topic_links.pluck(:id).each do |tl|
Jobs.enqueue(:crawl_topic_link, topic_link_id: tl)
end
topic_links.pluck(:id).each { |tl| Jobs.enqueue(:crawl_topic_link, topic_link_id: tl) }
end
end

186
lib/tasks/populate.thor
View File

@ -21,14 +21,28 @@ class Populate < Thor
> $ thor populate:posts -p 10 -n 5
MD
method_option :num_posts, aliases: '-n', type: :numeric, required: true, desc: "Number of posts to make"
method_option :users, aliases: '-u', type: :array, desc: "Usernames of users who will make the posts"
method_option :title, aliases: '-t', desc: "The title of the topic, if making a new topic"
method_option :topic_id, aliases: '-i', type: :numeric, desc: "The id of the topic where the posts will be added"
method_option :num_topics, aliases: '-p', type: :numeric, default: 1, desc: "Number of topics to create"
method_option :num_posts,
aliases: "-n",
type: :numeric,
required: true,
desc: "Number of posts to make"
method_option :users,
aliases: "-u",
type: :array,
desc: "Usernames of users who will make the posts"
method_option :title, aliases: "-t", desc: "The title of the topic, if making a new topic"
method_option :topic_id,
aliases: "-i",
type: :numeric,
desc: "The id of the topic where the posts will be added"
method_option :num_topics,
aliases: "-p",
type: :numeric,
default: 1,
desc: "Number of topics to create"
def posts
require './config/environment'
require "./config/environment"
users = []
if options[:users]
options[:users].each do |u|
@ -49,10 +63,8 @@ class Populate < Thor
topic = create_topic(users) unless topic
puts "Adding posts to '#{topic.title}'"
puts "Making #{options[:num_posts]} posts"
(start_post..options[:num_posts]).each do
create_post(users, topic)
end
puts ''
(start_post..options[:num_posts]).each { create_post(users, topic) }
puts ""
puts "Done. Topic id = #{topic.id}"
end
ensure
@ -65,7 +77,12 @@ class Populate < Thor
user = User.find_by_email(user_email)
unless user
puts "Creating new account: #{user_email}"
user = User.create!(email: user_email, password: SecureRandom.hex, username: UserNameSuggester.suggest(user_email))
user =
User.create!(
email: user_email,
password: SecureRandom.hex,
username: UserNameSuggester.suggest(user_email),
)
end
user.active = true
user.save!
@ -87,35 +104,158 @@ class Populate < Thor
end
def create_post(users, topic)
print '.'
raw = rand(4) == 0 ? (rand(2) == 0 ? image_posts.sample : wikipedia_posts.sample) : generate_sentence(7)
print "."
raw =
(
if rand(4) == 0
(rand(2) == 0 ? image_posts.sample : wikipedia_posts.sample)
else
generate_sentence(7)
end
)
post_creator = PostCreator.new(users.sample, topic_id: topic.id, raw: raw)
post = post_creator.create
unless post
puts post_creator.errors.full_messages, ""
end
puts post_creator.errors.full_messages, "" unless post
post
end
def hipster_words
@hipster_words ||= ['etsy', 'twee', 'hoodie', 'Banksy', 'retro', 'synth', 'single-origin', 'coffee', 'art', 'party', 'cliche', 'artisan', 'Williamsburg', 'squid', 'helvetica', 'keytar', 'American Apparel', 'craft beer', 'food truck', "you probably haven't heard of them", 'cardigan', 'aesthetic', 'raw denim', 'sartorial', 'gentrify', 'lomo', 'Vice', 'Pitchfork', 'Austin', 'sustainable', 'salvia', 'organic', 'thundercats', 'PBR', 'iPhone', 'lo-fi', 'skateboard', 'jean shorts', 'next level', 'beard', 'tattooed', 'trust fund', 'Four Loko', 'master cleanse', 'ethical', 'high life', 'wolf moon', 'fanny pack', 'Terry Richardson', '8-bit', 'Carles', 'Shoreditch', 'seitan', 'freegan', 'keffiyeh', 'biodiesel', 'quinoa', 'farm-to-table', 'fixie', 'viral', 'chambray', 'scenester', 'leggings', 'readymade', 'Brooklyn', 'Wayfarers', 'Marfa', 'put a bird on it', 'dreamcatcher', 'photo booth', 'tofu', 'mlkshk', 'vegan', 'vinyl', 'DIY', 'banh mi', 'bicycle rights', 'before they sold out', 'gluten-free', 'yr butcher blog', 'whatever', '+1', 'Cosby Sweater', 'VHS', 'messenger bag', 'cred', 'locavore', 'mustache', 'tumblr', 'Portland', 'mixtape', 'fap', 'letterpress', "McSweeney's", 'stumptown', 'brunch', 'Wes Anderson', 'irony', 'echo park']
@hipster_words ||= [
"etsy",
"twee",
"hoodie",
"Banksy",
"retro",
"synth",
"single-origin",
"coffee",
"art",
"party",
"cliche",
"artisan",
"Williamsburg",
"squid",
"helvetica",
"keytar",
"American Apparel",
"craft beer",
"food truck",
"you probably haven't heard of them",
"cardigan",
"aesthetic",
"raw denim",
"sartorial",
"gentrify",
"lomo",
"Vice",
"Pitchfork",
"Austin",
"sustainable",
"salvia",
"organic",
"thundercats",
"PBR",
"iPhone",
"lo-fi",
"skateboard",
"jean shorts",
"next level",
"beard",
"tattooed",
"trust fund",
"Four Loko",
"master cleanse",
"ethical",
"high life",
"wolf moon",
"fanny pack",
"Terry Richardson",
"8-bit",
"Carles",
"Shoreditch",
"seitan",
"freegan",
"keffiyeh",
"biodiesel",
"quinoa",
"farm-to-table",
"fixie",
"viral",
"chambray",
"scenester",
"leggings",
"readymade",
"Brooklyn",
"Wayfarers",
"Marfa",
"put a bird on it",
"dreamcatcher",
"photo booth",
"tofu",
"mlkshk",
"vegan",
"vinyl",
"DIY",
"banh mi",
"bicycle rights",
"before they sold out",
"gluten-free",
"yr butcher blog",
"whatever",
"+1",
"Cosby Sweater",
"VHS",
"messenger bag",
"cred",
"locavore",
"mustache",
"tumblr",
"Portland",
"mixtape",
"fap",
"letterpress",
"McSweeney's",
"stumptown",
"brunch",
"Wes Anderson",
"irony",
"echo park",
]
end
def generate_sentence(num_words)
sentence = hipster_words.sample(num_words).join(' ').capitalize + '.'
sentence.force_encoding('UTF-8')
sentence = hipster_words.sample(num_words).join(" ").capitalize + "."
sentence.force_encoding("UTF-8")
end
def generate_email
email = hipster_words.sample.delete(' ') + '@' + hipster_words.sample.delete(' ') + '.com'
email.delete("'").force_encoding('UTF-8')
email = hipster_words.sample.delete(" ") + "@" + hipster_words.sample.delete(" ") + ".com"
email.delete("'").force_encoding("UTF-8")
end
def image_posts
@image_posts ||= ["http://i.imgur.com/CnRF48R.jpg\n\n", "http://i.imgur.com/2iaeK.png\n\n", "http://i.imgur.com/WSD5t61.jpg\n\n", "http://i.imgur.com/GUldmUd.jpg\n\n", "http://i.imgur.com/nJnb6Bj.jpg\n\n", "http://i.imgur.com/eljDYjm.jpg\n\n", "http://i.imgur.com/5yZMWyY.png\n\n", "http://i.imgur.com/2iCPGm2.jpg\n\n"]
@image_posts ||= [
"http://i.imgur.com/CnRF48R.jpg\n\n",
"http://i.imgur.com/2iaeK.png\n\n",
"http://i.imgur.com/WSD5t61.jpg\n\n",
"http://i.imgur.com/GUldmUd.jpg\n\n",
"http://i.imgur.com/nJnb6Bj.jpg\n\n",
"http://i.imgur.com/eljDYjm.jpg\n\n",
"http://i.imgur.com/5yZMWyY.png\n\n",
"http://i.imgur.com/2iCPGm2.jpg\n\n",
]
end
def wikipedia_posts
@wikipedia_posts ||= ["http://en.wikipedia.org/wiki/Dwarf_fortress\n\n", "http://en.wikipedia.org/wiki/Action_plan\n\n", "http://en.wikipedia.org/wiki/Chang%27e_3\n\n", "http://en.wikipedia.org/wiki/Carl_sagan\n\n", "http://en.wikipedia.org/wiki/Chasmosaurus\n\n", "http://en.wikipedia.org/wiki/Indian_Space_Research_Organisation\n\n", "http://en.wikipedia.org/wiki/Rockstar_Consortium\n\n", "http://en.wikipedia.org/wiki/Manitoulin_island\n\n"]
@wikipedia_posts ||= [
"http://en.wikipedia.org/wiki/Dwarf_fortress\n\n",
"http://en.wikipedia.org/wiki/Action_plan\n\n",
"http://en.wikipedia.org/wiki/Chang%27e_3\n\n",
"http://en.wikipedia.org/wiki/Carl_sagan\n\n",
"http://en.wikipedia.org/wiki/Chasmosaurus\n\n",
"http://en.wikipedia.org/wiki/Indian_Space_Research_Organisation\n\n",
"http://en.wikipedia.org/wiki/Rockstar_Consortium\n\n",
"http://en.wikipedia.org/wiki/Manitoulin_island\n\n",
]
end
end

136
lib/tasks/typepad.thor
View File

@ -1,21 +1,21 @@
# frozen_string_literal: true
require 'open-uri'
require "open-uri"
class Typepad < Thor
desc "import", "Imports posts from a Disqus XML export"
method_option :file, aliases: '-f', required: true, desc: "The typepad file to import"
method_option :post_as, aliases: '-p', required: true, desc: "The Discourse username to post as"
method_option :google_api, aliases: '-g', required: false, desc: "The google plus API key to use to fetch usernames"
method_option :file, aliases: "-f", required: true, desc: "The typepad file to import"
method_option :post_as, aliases: "-p", required: true, desc: "The Discourse username to post as"
method_option :google_api,
aliases: "-g",
required: false,
desc: "The google plus API key to use to fetch usernames"
def import
require './config/environment'
require "./config/environment"
backup_settings = {}
%w(blocked_email_domains).each do |s|
backup_settings[s] = SiteSetting.get(s)
end
%w[blocked_email_domains].each { |s| backup_settings[s] = SiteSetting.get(s) }
user = User.where(username_lower: options[:post_as].downcase).first
if user.nil?
@ -31,26 +31,24 @@ class Typepad < Thor
input = ""
entries = []
File.open(options[:file]).each_line do |l|
l = l.scrub
File
.open(options[:file])
.each_line do |l|
l = l.scrub
if l =~ /\A--------\z/
parsed_entry = process_entry(input)
if parsed_entry
puts "Parsed #{parsed_entry[:title]}"
entries << parsed_entry
if l =~ /\A--------\z/
parsed_entry = process_entry(input)
if parsed_entry
puts "Parsed #{parsed_entry[:title]}"
entries << parsed_entry
end
input = ""
else
input << l
end
input = ""
else
input << l
end
end
entries.each_with_index do |e, i|
if e[:title] =~ /Head/
puts "#{i}: #{e[:title]}"
end
end
entries.each_with_index { |e, i| puts "#{i}: #{e[:title]}" if e[:title] =~ /Head/ }
RateLimiter.disable
SiteSetting.blocked_email_domains = ""
@ -74,13 +72,23 @@ class Typepad < Thor
email = c[:email]
post_user = User.where(email: email).first
if post_user.blank?
post_user = User.create!(name: c[:name], email: email, username: UserNameSuggester.suggest(username))
post_user =
User.create!(
name: c[:name],
email: email,
username: UserNameSuggester.suggest(username),
)
end
else
post_user = User.where(username: username).first
if post_user.blank?
suggested = UserNameSuggester.suggest(username)
post_user = User.create!(name: c[:name], email: "#{suggested}@no-email-found.com", username: suggested)
post_user =
User.create!(
name: c[:name],
email: "#{suggested}@no-email-found.com",
username: suggested,
)
end
end
@ -89,7 +97,7 @@ class Typepad < Thor
raw: c[:body],
cooked: c[:body],
created_at: c[:date],
skip_validations: true
skip_validations: true,
}
begin
post = PostCreator.new(post_user, attrs).create
@ -100,35 +108,34 @@ class Typepad < Thor
end
end
end
ensure
RateLimiter.enable
backup_settings.each do |s, v|
SiteSetting.set(s, v)
end
backup_settings.each { |s, v| SiteSetting.set(s, v) }
end
private
def clean_type!(type)
type.downcase!
type.gsub!(/ /, '_')
type.gsub!(/ /, "_")
type
end
def parse_meta_data(section)
result = {}
section.split(/\n/).each do |l|
if l =~ /\A([A-Z\ ]+)\: (.*)\z/
key, value = Regexp.last_match[1], Regexp.last_match[2]
clean_type!(key)
value.strip!
result[key.to_sym] = value
else
result[:body] ||= ""
result[:body] << l << "\n"
section
.split(/\n/)
.each do |l|
if l =~ /\A([A-Z\ ]+)\: (.*)\z/
key, value = Regexp.last_match[1], Regexp.last_match[2]
clean_type!(key)
value.strip!
result[key.to_sym] = value
else
result[:body] ||= ""
result[:body] << l << "\n"
end
end
end
result
end
@ -159,35 +166,36 @@ class Typepad < Thor
when :comment
comment = parse_comment(value).slice(:author, :email, :url, :body, :date)
if options[:google_api] && comment[:author] =~ /plus.google.com\/(\d+)/
if options[:google_api] && comment[:author] =~ %r{plus.google.com/(\d+)}
gplus_id = Regexp.last_match[1]
from_redis = Discourse.redis.get("gplus:#{gplus_id}")
if from_redis.blank?
json = ::JSON.parse(open("https://www.googleapis.com/plus/v1/people/#{gplus_id}?key=#{options[:google_api]}").read)
from_redis = json['displayName']
json =
::JSON.parse(
open(
"https://www.googleapis.com/plus/v1/people/#{gplus_id}?key=#{options[:google_api]}",
).read,
)
from_redis = json["displayName"]
Discourse.redis.set("gplus:#{gplus_id}", from_redis)
end
comment[:author] = from_redis
end
if comment[:author] =~ /([^\.]+)\.wordpress\.com/
comment[:author] = Regexp.last_match[1] if comment[:author] =~ /([^\.]+)\.wordpress\.com/
comment[:author] = Regexp.last_match[1] if comment[:author] =~ /([^\.]+)\.blogspot\.com/
if comment[:author] =~ %r{twitter.com/([a-zA-Z0-9]+)}
comment[:author] = Regexp.last_match[1]
end
if comment[:author] =~ /([^\.]+)\.blogspot\.com/
comment[:author] = Regexp.last_match[1]
end
if comment[:author] =~ /twitter.com\/([a-zA-Z0-9]+)/
comment[:author] = Regexp.last_match[1]
end
if comment[:author] =~ /www.facebook.com\/profile.php\?id=(\d+)/
if comment[:author] =~ %r{www.facebook.com/profile.php\?id=(\d+)}
fb_id = Regexp.last_match[1]
from_redis = Discourse.redis.get("fb:#{fb_id}")
if from_redis.blank?
json = ::JSON.parse(open("http://graph.facebook.com/#{fb_id}").read)
from_redis = json['username']
from_redis = json["username"]
Discourse.redis.set("fb:#{fb_id}", from_redis)
end
comment[:author] = from_redis
@ -195,11 +203,11 @@ class Typepad < Thor
comment[:name] = comment[:author]
if comment[:author]
comment[:author].gsub!(/\A[_\.]+/, '')
comment[:author].gsub!(/[_\.]+\z/, '')
comment[:author].gsub!(/\A[_\.]+/, "")
comment[:author].gsub!(/[_\.]+\z/, "")
if comment[:author].size < 12
comment[:author].gsub!(/ /, '_')
comment[:author].gsub!(/ /, "_")
else
segments = []
current = ""
@ -225,14 +233,13 @@ class Typepad < Thor
segments << current
comment[:author] = segments[0]
if segments.size > 1 && segments[1][0] =~ /[a-zA-Z]/
comment[:author] << segments[1][0]
end
comment[:author] << segments[1][0] if segments.size > 1 && segments[1][0] =~ /[a-zA-Z]/
end
end
comment[:author] = "commenter" if comment[:author].blank?
comment[:author] = "codinghorror" if comment[:author] == "Jeff Atwood" || comment[:author] == "JeffAtwood" || comment[:author] == "Jeff_Atwood"
comment[:author] = "codinghorror" if comment[:author] == "Jeff Atwood" ||
comment[:author] == "JeffAtwood" || comment[:author] == "Jeff_Atwood"
comment[:date] = comment[:date] ? DateTime.strptime(comment[:date], "%m/%d/%Y") : Time.now
entry[:comments] << comment if comment[:body].present?
@ -241,5 +248,4 @@ class Typepad < Thor
entry[:title] && entry[:body] ? entry : nil
end
end