+{{/if}}
diff --git a/app/assets/javascripts/discourse/app/lib/text.js b/app/assets/javascripts/discourse/app/lib/text.js
index e20fb6ade31..588d0dab3d4 100644
--- a/app/assets/javascripts/discourse/app/lib/text.js
+++ b/app/assets/javascripts/discourse/app/lib/text.js
@@ -21,6 +21,7 @@ function getOpts(opts) {
customEmojiTranslation: context.site.custom_emoji_translation,
siteSettings: context.siteSettings,
formatUsername,
+ watchedWordsReplacements: context.site.watched_words_replace,
},
opts
);
diff --git a/app/assets/javascripts/discourse/tests/acceptance/admin-watched-words-test.js b/app/assets/javascripts/discourse/tests/acceptance/admin-watched-words-test.js
index 2ae807ffc9d..36a2e4db9a1 100644
--- a/app/assets/javascripts/discourse/tests/acceptance/admin-watched-words-test.js
+++ b/app/assets/javascripts/discourse/tests/acceptance/admin-watched-words-test.js
@@ -12,7 +12,11 @@ acceptance("Admin - Watched Words", function (needs) {
test("list words in groups", async function (assert) {
await visit("/admin/logs/watched_words/action/block");
- assert.ok(exists(".watched-words-list"));
+ assert.ok(
+ !exists(".watched-words-list"),
+ "Don't show bad words by default."
+ );
+
assert.ok(
!exists(".watched-words-list .watched-word"),
"Don't show bad words by default."
diff --git a/app/assets/javascripts/pretty-text/addon/pretty-text.js b/app/assets/javascripts/pretty-text/addon/pretty-text.js
index d6ef5344bb1..2143505b514 100644
--- a/app/assets/javascripts/pretty-text/addon/pretty-text.js
+++ b/app/assets/javascripts/pretty-text/addon/pretty-text.js
@@ -33,6 +33,7 @@ export function buildOptions(state) {
censoredRegexp,
disableEmojis,
customEmojiTranslation,
+ watchedWordsReplacements,
} = state;
let features = {
@@ -82,6 +83,7 @@ export function buildOptions(state) {
siteSettings.enable_advanced_editor_preview_sync,
previewing,
disableEmojis,
+ watchedWordsReplacements,
};
// note, this will mutate options due to the way the API is designed
diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words-replace.js b/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words-replace.js
new file mode 100644
index 00000000000..f006efb37f3
--- /dev/null
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words-replace.js
@@ -0,0 +1,179 @@
+function isLinkOpen(str) {
+ return /^\s]/i.test(str);
+}
+
+function isLinkClose(str) {
+ return /^<\/a\s*>/i.test(str);
+}
+
+function findAllMatches(text, matchers, useRegExp) {
+ const matches = [];
+
+ if (useRegExp) {
+ matchers.forEach((matcher) => {
+ let match;
+ while ((match = matcher.pattern.exec(text)) !== null) {
+ matches.push({
+ index: match.index,
+ text: match[0],
+ replacement: matcher.replacement,
+ });
+ }
+ });
+ } else {
+ const lowerText = text.toLowerCase();
+ matchers.forEach((matcher) => {
+ const lowerPattern = matcher.pattern.toLowerCase();
+ let index = -1;
+ while ((index = lowerText.indexOf(lowerPattern, index + 1)) !== -1) {
+ matches.push({
+ index,
+ text: text.substr(index, lowerPattern.length),
+ replacement: matcher.replacement,
+ });
+ }
+ });
+ }
+
+ return matches.sort((a, b) => a.index - b.index);
+}
+
+export function setup(helper) {
+ helper.registerOptions((opts, siteSettings) => {
+ opts.watchedWordsRegularExpressions =
+ siteSettings.watched_words_regular_expressions;
+ });
+
+ helper.registerPlugin((md) => {
+ const replacements = md.options.discourse.watchedWordsReplacements;
+ if (!replacements) {
+ return;
+ }
+
+ const matchers = Object.keys(replacements).map((word) => ({
+ pattern: md.options.discourse.watchedWordsRegularExpressions
+ ? new RegExp(word, "gi")
+ : word,
+ replacement: replacements[word],
+ }));
+
+ const cache = {};
+
+ md.core.ruler.push("watched-words-replace", (state) => {
+ for (let j = 0, l = state.tokens.length; j < l; j++) {
+ if (state.tokens[j].type !== "inline") {
+ continue;
+ }
+
+ let tokens = state.tokens[j].children;
+
+ let htmlLinkLevel = 0;
+
+ // We scan from the end, to keep position when new tags added.
+ // Use reversed logic in links start/end match
+ for (let i = tokens.length - 1; i >= 0; i--) {
+ const currentToken = tokens[i];
+
+ // Skip content of markdown links
+ if (currentToken.type === "link_close") {
+ i--;
+ while (
+ tokens[i].level !== currentToken.level &&
+ tokens[i].type !== "link_open"
+ ) {
+ i--;
+ }
+ continue;
+ }
+
+ // Skip content of html tag links
+ if (currentToken.type === "html_inline") {
+ if (isLinkOpen(currentToken.content) && htmlLinkLevel > 0) {
+ htmlLinkLevel--;
+ }
+
+ if (isLinkClose(currentToken.content)) {
+ htmlLinkLevel++;
+ }
+ }
+
+ if (htmlLinkLevel > 0) {
+ continue;
+ }
+
+ if (currentToken.type === "text") {
+ const text = currentToken.content;
+ const matches = (cache[text] =
+ cache[text] ||
+ findAllMatches(
+ text,
+ matchers,
+ md.options.discourse.watchedWordsRegularExpressions
+ ));
+
+ // Now split string to nodes
+ const nodes = [];
+ let level = currentToken.level;
+ let lastPos = 0;
+
+ let token;
+ for (let ln = 0; ln < matches.length; ln++) {
+ if (matches[ln].index < lastPos) {
+ continue;
+ }
+
+ if (matches[ln].index > lastPos) {
+ token = new state.Token("text", "", 0);
+ token.content = text.slice(lastPos, matches[ln].index);
+ token.level = level;
+ nodes.push(token);
+ }
+
+ let url = state.md.normalizeLink(matches[ln].replacement);
+ if (state.md.validateLink(url) && /^https?/.test(url)) {
+ token = new state.Token("link_open", "a", 1);
+ token.attrs = [["href", url]];
+ token.level = level++;
+ token.markup = "linkify";
+ token.info = "auto";
+ nodes.push(token);
+
+ token = new state.Token("text", "", 0);
+ token.content = matches[ln].text;
+ token.level = level;
+ nodes.push(token);
+
+ token = new state.Token("link_close", "a", -1);
+ token.level = --level;
+ token.markup = "linkify";
+ token.info = "auto";
+ nodes.push(token);
+ } else {
+ token = new state.Token("text", "", 0);
+ token.content = matches[ln].replacement;
+ token.level = level;
+ nodes.push(token);
+ }
+
+ lastPos = matches[ln].index + matches[ln].text.length;
+ }
+
+ if (lastPos < text.length) {
+ token = new state.Token("text", "", 0);
+ token.content = text.slice(lastPos);
+ token.level = level;
+ nodes.push(token);
+ }
+
+ // replace current node
+ state.tokens[j].children = tokens = md.utils.arrayReplaceAt(
+ tokens,
+ i,
+ nodes
+ );
+ }
+ }
+ }
+ });
+ });
+}
diff --git a/app/assets/stylesheets/common/admin/staff_logs.scss b/app/assets/stylesheets/common/admin/staff_logs.scss
index cf802eeaed3..e5de3d67227 100644
--- a/app/assets/stylesheets/common/admin/staff_logs.scss
+++ b/app/assets/stylesheets/common/admin/staff_logs.scss
@@ -329,28 +329,13 @@ table.screened-ip-addresses {
vertical-align: top;
}
-.admin-watched-words {
- .clear-all-row {
- display: flex;
- margin-top: 10px;
- justify-content: flex-end;
- .clear-all {
- margin-left: 5px;
- }
- }
+.watched-word-container {
+ display: flex;
+ justify-content: space-between;
}
-.watched-word-controls {
- display: flex;
- flex-wrap: wrap;
- margin-bottom: 1em;
- justify-content: space-between;
- .download-upload-controls {
- display: flex;
- }
- .download {
- justify-content: flex-end;
- }
+.watched-words-uploader {
+ display: inline-block;
}
.watched-words-list {
@@ -361,47 +346,39 @@ table.screened-ip-addresses {
.watched-word {
display: inline-block;
cursor: pointer;
+
.d-icon {
margin-right: 0.25em;
color: var(--primary-medium);
}
+
&:hover .d-icon {
color: var(--danger);
}
}
.watched-word-form {
- display: inline-block;
.success-message {
margin-left: 1em;
}
}
-.watched-words-uploader {
- margin-left: 5px;
- display: flex;
- flex-direction: column;
- align-items: flex-end;
- @media screen and (max-width: 500px) {
- flex: 1 1 100%;
- margin-top: 0.5em;
- }
- .instructions {
- font-size: $font-down-1;
- margin-top: 5px;
- }
-}
-
-.watched-words-detail {
- .about {
- margin: 0.5em 0 1em 0;
- }
+.watched-words-detail .about,
+.watched-word-form {
+ margin: 0.5em 0 1em 0;
}
.watched-words-test-modal p {
margin-top: 0;
}
+.watched-word-input {
+ label {
+ display: inline-block;
+ min-width: 120px;
+ }
+}
+
// Search logs
table.search-logs-list {
diff --git a/app/controllers/admin/watched_words_controller.rb b/app/controllers/admin/watched_words_controller.rb
index b6e013041be..b356a765eec 100644
--- a/app/controllers/admin/watched_words_controller.rb
+++ b/app/controllers/admin/watched_words_controller.rb
@@ -69,7 +69,7 @@ class Admin::WatchedWordsController < Admin::AdminController
private
def watched_words_params
- params.permit(:id, :word, :action_key)
+ params.permit(:id, :word, :replacement, :action_key)
end
end
diff --git a/app/models/watched_word.rb b/app/models/watched_word.rb
index fb4ad35c73a..200059978e3 100644
--- a/app/models/watched_word.rb
+++ b/app/models/watched_word.rb
@@ -7,7 +7,8 @@ class WatchedWord < ActiveRecord::Base
block: 1,
censor: 2,
require_approval: 3,
- flag: 4
+ flag: 4,
+ replace: 5
)
end
@@ -37,6 +38,7 @@ class WatchedWord < ActiveRecord::Base
def self.create_or_update_word(params)
new_word = normalize_word(params[:word])
w = WatchedWord.where("word ILIKE ?", new_word).first || WatchedWord.new(word: new_word)
+ w.replacement = params[:replacement] if params[:replacement]
w.action_key = params[:action_key] if params[:action_key]
w.action = params[:action] if params[:action]
w.save
@@ -57,11 +59,12 @@ end
#
# Table name: watched_words
#
-# id :integer not null, primary key
-# word :string not null
-# action :integer not null
-# created_at :datetime not null
-# updated_at :datetime not null
+# id :integer not null, primary key
+# word :string not null
+# action :integer not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# replacement :string
#
# Indexes
#
diff --git a/app/serializers/site_serializer.rb b/app/serializers/site_serializer.rb
index 7aa736c9b42..52493214498 100644
--- a/app/serializers/site_serializer.rb
+++ b/app/serializers/site_serializer.rb
@@ -28,7 +28,8 @@ class SiteSerializer < ApplicationSerializer
:default_dark_color_scheme,
:censored_regexp,
:shared_drafts_category_id,
- :custom_emoji_translation
+ :custom_emoji_translation,
+ :watched_words_replace
)
has_many :categories, serializer: SiteCategorySerializer, embed: :objects
@@ -175,6 +176,10 @@ class SiteSerializer < ApplicationSerializer
scope.can_see_shared_draft?
end
+ def watched_words_replace
+ WordWatcher.get_cached_words(:replace)
+ end
+
private
def ordered_flags(flags)
diff --git a/app/serializers/watched_word_serializer.rb b/app/serializers/watched_word_serializer.rb
index a99bfc07a25..65423716886 100644
--- a/app/serializers/watched_word_serializer.rb
+++ b/app/serializers/watched_word_serializer.rb
@@ -1,9 +1,13 @@
# frozen_string_literal: true
class WatchedWordSerializer < ApplicationSerializer
- attributes :id, :word, :action
+ attributes :id, :word, :replacement, :action
def action
WatchedWord.actions[object.action]
end
+
+ def include_replacement?
+ action == :replace
+ end
end
diff --git a/app/services/word_watcher.rb b/app/services/word_watcher.rb
index 0a1cad9bae6..92d278b6b37 100644
--- a/app/services/word_watcher.rb
+++ b/app/services/word_watcher.rb
@@ -7,7 +7,12 @@ class WordWatcher
end
def self.words_for_action(action)
- WatchedWord.where(action: WatchedWord.actions[action.to_sym]).limit(1000).pluck(:word)
+ words = WatchedWord.where(action: WatchedWord.actions[action.to_sym]).limit(1000)
+ if action.to_sym == :replace
+ words.pluck(:word, :replacement).to_h
+ else
+ words.pluck(:word)
+ end
end
def self.words_for_action_exists?(action)
@@ -26,6 +31,9 @@ class WordWatcher
def self.word_matcher_regexp(action, raise_errors: false)
words = get_cached_words(action)
if words
+ if action.to_sym == :replace
+ words = words.keys
+ end
words = words.map do |w|
word = word_to_regexp(w)
word = "(#{word})" if SiteSetting.watched_words_regular_expressions?
diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml
index 7fe355e6556..8512284510d 100644
--- a/config/locales/client.en.yml
+++ b/config/locales/client.en.yml
@@ -4603,31 +4603,33 @@ en:
title: "Watched Words"
search: "search"
clear_filter: "Clear"
- show_words: "show words"
- one_word_per_line: "One word per line"
+ show_words:
+ one: "show %{count} word"
+ other: "show %{count} words"
download: Download
clear_all: Clear All
clear_all_confirm_block: "Are you sure you want to clear all watched words for the Block action?"
clear_all_confirm_censor: "Are you sure you want to clear all watched words for the Censor action?"
clear_all_confirm_flag: "Are you sure you want to clear all watched words for the Flag action?"
clear_all_confirm_require_approval: "Are you sure you want to clear all watched words for the Require Approval action?"
- word_count:
- one: "%{count} word"
- other: "%{count} words"
actions:
block: "Block"
censor: "Censor"
require_approval: "Require Approval"
flag: "Flag"
+ replace: "Replace"
action_descriptions:
block: "Prevent posts containing these words from being posted. The user will see an error message when they try to submit their post."
censor: "Allow posts containing these words, but replace them with characters that hide the censored words."
require_approval: "Posts containing these words will require approval by staff before they can be seen."
flag: "Allow posts containing these words, but flag them as inappropriate so moderators can review them."
+ replace: "Replace words in posts with other words or links"
form:
- label: "New Word:"
+ label: "New Word"
placeholder: "full word or * as wildcard"
placeholder_regexp: "regular expression"
+ replacement_label: "Replacement"
+ replacement_placeholder: "example or https://example.com"
add: "Add"
success: "Success"
exists: "Already exists"
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index e041502430e..f5d23d879d7 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -1943,7 +1943,6 @@ en:
min_first_post_typing_time: "Minimum amount of time in milliseconds a user must type during first post, if threshold is not met post will automatically enter the needs approval queue. Set to 0 to disable (not recommended)"
auto_silence_fast_typers_on_first_post: "Automatically silence users that do not meet min_first_post_typing_time"
auto_silence_fast_typers_max_trust_level: "Maximum trust level to auto silence fast typers"
- auto_silence_first_post_regex: "Case insensitive regex that if passed will cause first post by user to be silenced and sent to approval queue. Example: raging|a[bc]a , will cause all posts containing raging or aba or aca to be silenced on first. Only applies to first post."
reviewable_claiming: "Does reviewable content need to be claimed before it can be acted upon?"
reviewable_default_topics: "Show reviewable content grouped by topic by default"
reviewable_default_visibility: "Don't show reviewable items unless they meet this priority"
@@ -4905,7 +4904,6 @@ en:
trust_level: "Users at low trust levels must have replies approved by staff. See `approve_unless_trust_level`."
new_topics_unless_trust_level: "Users at low trust levels must have topics approved by staff. See `approve_new_topics_unless_trust_level`."
fast_typer: "New user typed their first post suspiciously fast, suspected bot or spammer behavior. See `min_first_post_typing_time`."
- auto_silence_regexp: "New user whose first post matches the `auto_silence_first_post_regex` setting."
watched_word: "This post included a Watched Word. See your list of watched words."
staged: "New topics and posts for staged users must be approved by staff. See `approve_unless_staged`."
category: "Posts in this category require manual approval by staff. See the category settings."
diff --git a/config/site_settings.yml b/config/site_settings.yml
index faba7b8a3f5..46705b46d93 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -1660,7 +1660,6 @@ spam:
min_first_post_typing_time: 3000
auto_silence_fast_typers_on_first_post: true
auto_silence_fast_typers_max_trust_level: 0
- auto_silence_first_post_regex: ""
high_trust_flaggers_auto_hide_posts: true
cooldown_hours_until_reflag:
default: 24
diff --git a/db/migrate/20210204190020_move_auto_silence_first_post_regex_to_watched_words.rb b/db/migrate/20210204190020_move_auto_silence_first_post_regex_to_watched_words.rb
new file mode 100644
index 00000000000..ead20c60058
--- /dev/null
+++ b/db/migrate/20210204190020_move_auto_silence_first_post_regex_to_watched_words.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class MoveAutoSilenceFirstPostRegexToWatchedWords < ActiveRecord::Migration[6.0]
+ def up
+ execute <<~SQL
+ INSERT INTO watched_words (word, action, created_at, updated_at)
+ SELECT value, 3, created_at, updated_at
+ FROM site_settings
+ WHERE name = 'auto_silence_first_post_regex'
+ ON CONFLICT DO NOTHING
+ SQL
+
+ execute <<~SQL
+ INSERT INTO watched_words (word, action, created_at, updated_at)
+ SELECT unnest(string_to_array(value, '|')), 3, created_at, updated_at
+ FROM site_settings
+ WHERE name = 'auto_silence_first_post_regex'
+ ON CONFLICT DO NOTHING
+ SQL
+ end
+
+ def down
+ end
+end
diff --git a/db/migrate/20210204195932_add_replacement_to_watched_words.rb b/db/migrate/20210204195932_add_replacement_to_watched_words.rb
new file mode 100644
index 00000000000..8496793806b
--- /dev/null
+++ b/db/migrate/20210204195932_add_replacement_to_watched_words.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddReplacementToWatchedWords < ActiveRecord::Migration[6.0]
+ def change
+ add_column :watched_words, :replacement, :string, null: true
+ end
+end
diff --git a/lib/new_post_manager.rb b/lib/new_post_manager.rb
index 9b72719232b..cf0bf3e2728 100644
--- a/lib/new_post_manager.rb
+++ b/lib/new_post_manager.rb
@@ -54,25 +54,6 @@ class NewPostManager
manager.user.trust_level <= SiteSetting.auto_silence_fast_typers_max_trust_level
end
- def self.matches_auto_silence_regex?(manager)
- args = manager.args
-
- pattern = SiteSetting.auto_silence_first_post_regex
-
- return false unless pattern.present?
- return false unless is_first_post?(manager)
-
- begin
- regex = Regexp.new(pattern, Regexp::IGNORECASE)
- rescue => e
- Rails.logger.warn "Invalid regex in auto_silence_first_post_regex #{e}"
- return false
- end
-
- "#{args[:title]} #{args[:raw]}" =~ regex
-
- end
-
def self.exempt_user?(user)
user.staff?
end
@@ -102,8 +83,6 @@ class NewPostManager
return :fast_typer if is_fast_typer?(manager)
- return :auto_silence_regex if matches_auto_silence_regex?(manager)
-
return :staged if SiteSetting.approve_unless_staged? && user.staged?
return :category if post_needs_approval_in_its_category?(manager)
@@ -168,8 +147,6 @@ class NewPostManager
I18n.with_locale(SiteSetting.default_locale) do
if is_fast_typer?(manager)
UserSilencer.silence(manager.user, Discourse.system_user, keep_posts: true, reason: I18n.t("user.new_user_typed_too_fast"))
- elsif matches_auto_silence_regex?(manager)
- UserSilencer.silence(manager.user, Discourse.system_user, keep_posts: true, reason: I18n.t("user.content_matches_auto_silence_regex"))
elsif reason == :email_spam && is_first_post?(manager)
UserSilencer.silence(manager.user, Discourse.system_user, keep_posts: true, reason: I18n.t("user.email_in_spam_header"))
end
diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb
index 81a9e768df8..399403402a7 100644
--- a/lib/pretty_text.rb
+++ b/lib/pretty_text.rb
@@ -172,6 +172,7 @@ module PrettyText
__optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer;
__optInput.lookupUploadUrls = __lookupUploadUrls;
__optInput.censoredRegexp = #{WordWatcher.word_matcher_regexp(:censor)&.source.to_json};
+ __optInput.watchedWordsReplacements = #{WordWatcher.get_cached_words(:replace).to_json};
JS
if opts[:topicId]
diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb
index 67722c127dd..814ee851657 100644
--- a/spec/components/pretty_text_spec.rb
+++ b/spec/components/pretty_text_spec.rb
@@ -1351,6 +1351,56 @@ HTML
end
end
+ describe "watched words - replace" do
+ after(:all) { Discourse.redis.flushdb }
+
+ it "replaces words with other words" do
+ Fabricate(:watched_word, action: WatchedWord.actions[:replace], word: "dolor sit", replacement: "something else")
+
+ expect(PrettyText.cook("Lorem ipsum dolor sit amet")).to match_html(<<~HTML)
+
Lorem ipsum something else amet
+ HTML
+ end
+
+ it "replaces words with links" do
+ Fabricate(:watched_word, action: WatchedWord.actions[:replace], word: "meta", replacement: "https://meta.discourse.org")
+
+ expect(PrettyText.cook("Meta is a Discourse forum")).to match_html(<<~HTML)
+
+ HTML
+ end
+
+ it "works with regex" do
+ Fabricate(:watched_word, action: WatchedWord.actions[:replace], word: "f.o", replacement: "test")
+
+ expect(PrettyText.cook("foo")).to match_html("
+ HTML
+ end
+ end
+
it 'supports typographer' do
SiteSetting.enable_markdown_typographer = true
expect(PrettyText.cook('(tm)')).to eq('
™
')
diff --git a/spec/requests/posts_controller_spec.rb b/spec/requests/posts_controller_spec.rb
index 1d22435ac21..92d35280518 100644
--- a/spec/requests/posts_controller_spec.rb
+++ b/spec/requests/posts_controller_spec.rb
@@ -857,26 +857,6 @@ describe PostsController do
end
end
- it 'silences correctly based on auto_silence_first_post_regex' do
- SiteSetting.auto_silence_first_post_regex = "I love candy|i eat s[1-5]"
-
- post "/posts.json", params: {
- raw: 'this is the test content',
- title: 'when I eat s3 sometimes when not looking'
- }
-
- expect(response.status).to eq(200)
- parsed = response.parsed_body
-
- expect(parsed["action"]).to eq("enqueued")
- reviewable = ReviewableQueuedPost.find_by(created_by: user)
- score = reviewable.reviewable_scores.first
- expect(score.reason).to eq('auto_silence_regex')
-
- user.reload
- expect(user).to be_silenced
- end
-
it "can send a message to a group" do
group = Group.create(name: 'test_group', messageable_level: Group::ALIAS_LEVELS[:nobody])
user1 = user