Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FEATURE: Add support for case-sensitive Watched Words (#17445) 

* FEATURE: Add case-sensitivity flag to watched_words

Currently, all watched words are matched case-insensitively. This flag
allows a watched word to be flagged for case-sensitive matching.
To allow allow for backwards compatibility the flag is set to false by
default.

* FEATURE: Support case-sensitive creation of Watched Words via API

Extend admin creation and upload of Watched Words to support case
sensitive flag. This lays the ground work for supporting
case-insensitive matching of Watched Words.

Support for an extra column has also been introduced for the Watched
Words upload CSV file. The new column structure is as follows:

 word,replacement,case_sentive

* FEATURE: Enable case-sensitive matching of Watched Words

WordWatcher's word_matcher_regexp now returns a list of regular
expressions instead of one case-insensitive regular expression.

With the ability to flag a Watched Word as case-sensitive, an action
can have words of both sensitivities.This makes the use of the global
Regexp::IGNORECASE flag added to all words problematic.

To get around platform limitations around the use of subexpression level
switches/flags, a list of regular expressions is returned instead, one for each
case sensitivity.

Word matching has also been updated to use this list of regular expressions
instead of one.

* FEATURE: Use case-sensitive regular expressions for Watched Words

Update Watched Words regular expressions matching and processing to handle
the extra metadata which comes along with the introduction of
case-sensitive Watched Words.

This allows case-sensitive Watched Words to matched as such.

* DEV: Simplify type casting of case-sensitive flag from uploads

Use builtin semantics instead of a custom method for converting
string case flags in uploaded Watched Words to boolean.

* UX: Add case-sensitivity details to Admin Watched Words UI

Update Watched Word form to include a toggle for case-sensitivity.
This also adds support for, case-sensitive testing and matching of  Watched Word
in the admin UI.

* DEV: Code improvements from review feedback

 - Extract watched word regex creation out to a utility function
 - Make JS array presence check more explicit and readable

* DEV: Extract Watched Word regex creation to utility function

Clean-up work from review feedback. Reduce code duplication.

* DEV: Rename word_matcher_regexp to word_matcher_regexp_list

Since a list is returned now instead of a single regular expression,
change `word_matcher_regexp` to `word_matcher_regexp_list` to better communicate
this change.

* DEV:  Incorporate WordWatcher updates from upstream

Resolve conflicts and ensure apply_to_text does not remove non-word characters in matches
that aren't at the beginning of the line.
This commit is contained in:
Selase Krakani 2022-08-02 08:06:03 +00:00 committed by GitHub
parent df264e49a9
commit 862007fb18
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 500 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/assets/javascripts/admin/addon/components/admin-watched-word.js
View File

@ -1,5 +1,5 @@
import Component from "@ember/component";
import { equal } from "@ember/object/computed";
import { alias, equal } from "@ember/object/computed";
import bootbox from "bootbox";
import discourseComputed from "discourse-common/utils/decorators";
import { action } from "@ember/object";
@ -11,6 +11,7 @@ export default Component.extend({
isReplace: equal("actionKey", "replace"),
isTag: equal("actionKey", "tag"),
isLink: equal("actionKey", "link"),
isCaseSensitive: alias("word.case_sensitive"),
@discourseComputed("word.replacement")
tags(replacement) {

3
app/assets/javascripts/admin/addon/components/watched-word-form.js
View File

@ -14,6 +14,7 @@ export default Component.extend({
actionKey: null,
showMessage: false,
selectedTags: null,
isCaseSensitive: false,
canReplace: equal("actionKey", "replace"),
canTag: equal("actionKey", "tag"),
@ -78,6 +79,7 @@ export default Component.extend({
? this.replacement
: null,
action: this.actionKey,
isCaseSensitive: this.isCaseSensitive,
});
watchedWord
@ -90,6 +92,7 @@ export default Component.extend({
selectedTags: [],
showMessage: true,
message: I18n.t("admin.watched_words.form.success"),
isCaseSensitive: false,
});
this.action(WatchedWord.create(result));
schedule("afterRender", () =>

23
app/assets/javascripts/admin/addon/controllers/modals/admin-watched-word-test.js
View File

@ -2,6 +2,10 @@ import Controller from "@ember/controller";
import ModalFunctionality from "discourse/mixins/modal-functionality";
import discourseComputed from "discourse-common/utils/decorators";
import { equal } from "@ember/object/computed";
import {
createWatchedWordRegExp,
toWatchedWord,
} from "discourse-common/utils/watched-words";
export default Controller.extend(ModalFunctionality, {
isReplace: equal("model.nameKey", "replace"),
@ -16,16 +20,17 @@ export default Controller.extend(ModalFunctionality, {
"isTag",
"isLink"
)
matches(value, regexpString, words, isReplace, isTag, isLink) {
if (!value || !regexpString) {
matches(value, regexpList, words, isReplace, isTag, isLink) {
if (!value || regexpList.length === 0) {
return [];
}
if (isReplace || isLink) {
const matches = [];
words.forEach((word) => {
const regexp = new RegExp(word.regexp, "gi");
const regexp = createWatchedWordRegExp(word);
let match;
while ((match = regexp.exec(value)) !== null) {
matches.push({
match: match[1],
@ -37,8 +42,9 @@ export default Controller.extend(ModalFunctionality, {
} else if (isTag) {
const matches = {};
words.forEach((word) => {
const regexp = new RegExp(word.regexp, "gi");
const regexp = createWatchedWordRegExp(word);
let match;
while ((match = regexp.exec(value)) !== null) {
if (!matches[match[1]]) {
matches[match[1]] = new Set();
@ -56,7 +62,14 @@ export default Controller.extend(ModalFunctionality, {
tags: Array.from(entry[1]),
}));
} else {
return value.match(new RegExp(regexpString, "ig")) || [];
let matches = [];
regexpList.forEach((regexp) => {
const wordRegexp = createWatchedWordRegExp(toWatchedWord(regexp));
matches.push(...(value.match(wordRegexp) || []));
});
return matches;
}
},
});

1
app/assets/javascripts/admin/addon/models/watched-word.js
View File

@ -14,6 +14,7 @@ const WatchedWord = EmberObject.extend({
word: this.word,
replacement: this.replacement,
action_key: this.action,
case_sensitive: this.isCaseSensitive,
},
dataType: "json",
}

3
app/assets/javascripts/admin/addon/templates/components/admin-watched-word.hbs
View File

@ -7,3 +7,6 @@
<span class="tag">{{tag}}</span>
{{/each}}
{{/if}}
{{#if this.isCaseSensitive}}
<span class="case-sensitive">{{i18n "admin.watched_words.case_sensitive"}}</span>
{{/if}}

8
app/assets/javascripts/admin/addon/templates/components/watched-word-form.hbs
View File

@ -27,6 +27,14 @@
</div>
{{/if}}
<div class="watched-word-input">
<label for="watched-case-sensitivity">{{i18n "admin.watched_words.form.case_sensitivity_label"}}</label>
<label class="case-sensitivity-checkbox">
<Input @type="checkbox" @checked={{this.isCaseSensitive}} disabled={{this.formSubmitted}} />
{{i18n "admin.watched_words.form.case_sensitivity_description"}}
</label>
</div>
<DButton @type="submit" @class="btn btn-primary" @action={{action "submit"}} @disabled={{this.formSubmitted}} @label="admin.watched_words.form.add" />
{{#if this.showMessage}}

9
app/assets/javascripts/discourse-common/addon/utils/watched-words.js Normal file
View File

@ -0,0 +1,9 @@
export function createWatchedWordRegExp(word) {
const caseFlag = word.case_sensitive ? "" : "i";
return new RegExp(word.regexp, `${caseFlag}g`);
}
export function toWatchedWord(regexp) {
const [[regexpString, options]] = Object.entries(regexp);
return { regexp: regexpString, ...options };
}

17
app/assets/javascripts/discourse/tests/acceptance/admin-watched-words-test.js
View File

@ -7,6 +7,7 @@ import {
} from "discourse/tests/helpers/qunit-helpers";
import { click, fillIn, visit } from "@ember/test-helpers";
import { test } from "qunit";
import I18n from "I18n";
acceptance("Admin - Watched Words", function (needs) {
needs.user();
@ -68,7 +69,23 @@ acceptance("Admin - Watched Words", function (needs) {
found.push(true);
}
});
assert.strictEqual(found.length, 1);
assert.strictEqual(count(".watched-words-list .case-sensitive"), 0);
});
test("add case-sensitve words", async function (assert) {
await visit("/admin/customize/watched_words/action/block");
click(".show-words-checkbox");
fillIn(".watched-word-form input", "Discourse");
click(".case-sensitivity-checkbox");
await click(".watched-word-form button");
assert
.dom(".watched-words-list .watched-word")
.hasText(`Discourse ${I18n.t("admin.watched_words.case_sensitive")}`);
});
test("remove words", async function (assert) {

30
app/assets/javascripts/discourse/tests/fixtures/watched-words-fixtures.js vendored
View File

@ -2,18 +2,19 @@ export default {
"/admin/customize/watched_words.json": {
actions: ["block", "censor", "require_approval", "flag", "replace", "tag"],
words: [
{ id: 1, word: "liquorice", action: "block" },
{ id: 2, word: "anise", action: "block" },
{ id: 3, word: "pyramid", action: "flag" },
{ id: 4, word: "scheme", action: "flag" },
{ id: 5, word: "coupon", action: "require_approval" },
{ id: 6, word: '<img src="x">', action: "block" },
{ id: 1, word: "liquorice", action: "block", case_sensitive: false },
{ id: 2, word: "anise", action: "block", case_sensitive: false },
{ id: 3, word: "pyramid", action: "flag", case_sensitive: false },
{ id: 4, word: "scheme", action: "flag", case_sensitive: false },
{ id: 5, word: "coupon", action: "require_approval", case_sensitive: false },
{ id: 6, word: '<img src="x">', action: "block", case_sensitive: false },
{
id: 7,
word: "hi",
regexp: "(hi)",
replacement: "hello",
action: "replace",
case_sensitive: false,
},
{
id: 8,
@ -21,15 +22,20 @@ export default {
regexp: "(hello)",
replacement: "greeting",
action: "tag",
case_sensitive: false,
},
],
compiled_regular_expressions: {
block: '(?:\\W|^)(liquorice|anise|<img\\ src="x">)(?=\\W|$)',
censor: null,
require_approval: "(?:\\W|^)(coupon)(?=\\W|$)",
flag: "(?:\\W|^)(pyramid|scheme)(?=\\W|$)",
replace: "(?:\\W|^)(hi)(?=\\W|$)",
tag: "(?:\\W|^)(hello)(?=\\W|$)",
block: [
{ '(?:\\W|^)(liquorice|anise|<img\\ src="x">)(?=\\W|$)': { case_sensitive: false }, },
],
censor: [],
require_approval: [
{ "(?:\\W|^)(coupon)(?=\\W|$)": { case_sensitive: false }, },
],
flag: [{ "(?:\\W|^)(pyramid|scheme)(?=\\W|$)": {case_sensitive: false }, },],
replace: [{ "(?:\\W|^)(hi)(?=\\W|$)": { case_sensitive: false }},],
tag: [{ "(?:\\W|^)(hello)(?=\\W|$)": { case_sensitive: false }, },],
},
},
};

1
app/assets/javascripts/discourse/tests/helpers/create-pretender.js
View File

@ -823,6 +823,7 @@ export function applyDefaultHandlers(pretender) {
pretender.post("/admin/customize/watched_words.json", (request) => {
const result = parsePostData(request.requestBody);
result.id = new Date().getTime();
result.case_sensitive = result.case_sensitive === "true";
return response(200, result);
});

20
app/assets/javascripts/discourse/tests/unit/lib/pretty-text-test.js
View File

@ -1104,7 +1104,7 @@ eviltrout</p>
assert.cookedOptions(
"Pleased to meet you, but pleeeease call me later, xyz123",
{
censoredRegexp: "(xyz*|plee+ase)",
censoredRegexp: [{ "(xyz*|plee+ase)": { case_sensitive: false } }],
},
"<p>Pleased to meet you, but ■■■■■■■■■ call me later, ■■■123</p>",
"supports censoring"
@ -1710,7 +1710,12 @@ var bar = 'bar';
test("watched words replace", function (assert) {
const opts = {
watchedWordsReplace: { "(?:\\W|^)(fun)(?=\\W|$)": "times" },
watchedWordsReplace: {
"(?:\\W|^)(fun)(?=\\W|$)": {
replacement: "times",
case_sensitive: false,
},
},
};
assert.cookedOptions("test fun funny", opts, "<p>test times funny</p>");
@ -1719,7 +1724,12 @@ var bar = 'bar';
test("watched words link", function (assert) {
const opts = {
watchedWordsLink: { "(?:\\W|^)(fun)(?=\\W|$)": "https://discourse.org" },
watchedWordsLink: {
"(?:\\W|^)(fun)(?=\\W|$)": {
replacement: "https://discourse.org",
case_sensitive: false,
},
},
};
assert.cookedOptions(
@ -1733,7 +1743,9 @@ var bar = 'bar';
const maxMatches = 100; // same limit as MD watched-words-replace plugin
const opts = {
siteSettings: { watched_words_regular_expressions: true },
watchedWordsReplace: { "(\\bu?\\b)": "you" },
watchedWordsReplace: {
"(\\bu?\\b)": { replacement: "you", case_sensitive: false },
},
};
assert.cookedOptions(

27
app/assets/javascripts/pretty-text/addon/censored-words.js
View File

@ -1,15 +1,24 @@
export function censorFn(regexpString, replacementLetter) {
if (regexpString) {
let censorRegexp = new RegExp(regexpString, "ig");
import {
createWatchedWordRegExp,
toWatchedWord,
} from "discourse-common/utils/watched-words";
export function censorFn(regexpList, replacementLetter) {
if (regexpList.length) {
replacementLetter = replacementLetter || "&#9632;";
let censorRegexps = regexpList.map((regexp) => {
return createWatchedWordRegExp(toWatchedWord(regexp));
});
return function (text) {
text = text.replace(censorRegexp, (fullMatch, ...groupMatches) => {
const stringMatch = groupMatches.find((g) => typeof g === "string");
return fullMatch.replace(
stringMatch,
new Array(stringMatch.length + 1).join(replacementLetter)
);
censorRegexps.forEach((censorRegexp) => {
text = text.replace(censorRegexp, (fullMatch, ...groupMatches) => {
const stringMatch = groupMatches.find((g) => typeof g === "string");
return fullMatch.replace(
stringMatch,
new Array(stringMatch.length + 1).join(replacementLetter)
);
});
});
return text;

6
app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js
View File

@ -28,11 +28,11 @@ function censorTree(state, censor) {
export function setup(helper) {
helper.registerPlugin((md) => {
const censoredRegexp = md.options.discourse.censoredRegexp;
const censoredRegexps = md.options.discourse.censoredRegexp;
if (censoredRegexp) {
if (Array.isArray(censoredRegexps) && censoredRegexps.length > 0) {
const replacement = String.fromCharCode(9632);
const censor = censorFn(censoredRegexp, replacement);
const censor = censorFn(censoredRegexps, replacement);
md.core.ruler.push("censored", (state) => censorTree(state, censor));
}
});

21
app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js
View File

@ -1,3 +1,8 @@
import {
createWatchedWordRegExp,
toWatchedWord,
} from "discourse-common/utils/watched-words";
const MAX_MATCHES = 100;
function isLinkOpen(str) {
@ -47,10 +52,12 @@ export function setup(helper) {
if (md.options.discourse.watchedWordsReplace) {
Object.entries(md.options.discourse.watchedWordsReplace).map(
([word, replacement]) => {
([regexpString, options]) => {
const word = toWatchedWord({ [regexpString]: options });
matchers.push({
pattern: new RegExp(word, "gi"),
replacement,
pattern: createWatchedWordRegExp(word),
replacement: options.replacement,
link: false,
});
}
@ -59,10 +66,12 @@ export function setup(helper) {
if (md.options.discourse.watchedWordsLink) {
Object.entries(md.options.discourse.watchedWordsLink).map(
([word, replacement]) => {
([regexpString, options]) => {
const word = toWatchedWord({ [regexpString]: options });
matchers.push({
pattern: new RegExp(word, "gi"),
replacement,
pattern: createWatchedWordRegExp(word),
replacement: options.replacement,
link: true,
});
}

6
app/controllers/admin/watched_words_controller.rb
View File

@ -41,7 +41,8 @@ class Admin::WatchedWordsController < Admin::AdminController
watched_word = WatchedWord.create_or_update_word(
word: row[0],
replacement: has_replacement ? row[1] : nil,
action_key: action_key
action_key: action_key,
case_sensitive: "true" == row[2]&.strip&.downcase
)
if watched_word.valid?
StaffActionLogger.new(current_user).log_watched_words_creation(watched_word)
@ -95,7 +96,6 @@ class Admin::WatchedWordsController < Admin::AdminController
private
def watched_words_params
params.permit(:id, :word, :replacement, :action_key)
params.permit(:id, :word, :replacement, :action_key, :case_sensitive)
end
end

2
app/models/admin_dashboard_data.rb
View File

@ -360,7 +360,7 @@ class AdminDashboardData
def watched_words_check
WatchedWord.actions.keys.each do |action|
begin
WordWatcher.word_matcher_regexp(action, raise_errors: true)
WordWatcher.word_matcher_regexp_list(action, raise_errors: true)
rescue RegexpError => e
translated_action = I18n.t("admin_js.admin.watched_words.actions.#{action}")
I18n.t('dashboard.watched_word_regexp_error', base_path: Discourse.base_path, action: translated_action)

14
app/models/watched_word.rb
View File

@ -65,6 +65,7 @@ class WatchedWord < ActiveRecord::Base
w.replacement = params[:replacement] if params[:replacement]
w.action_key = params[:action_key] if params[:action_key]
w.action = params[:action] if params[:action]
w.case_sensitive = params[:case_sensitive] if !params[:case_sensitive].nil?
w.save
w
end
@ -94,12 +95,13 @@ end
#
# Table name: watched_words
#
# id :integer not null, primary key
# word :string not null
# action :integer not null
# created_at :datetime not null
# updated_at :datetime not null
# replacement :string
# id :integer not null, primary key
# word :string not null
# action :integer not null
# created_at :datetime not null
# updated_at :datetime not null
# replacement :string
# case_sensitive :boolean default(FALSE), not null
#
# Indexes
#

2
app/serializers/site_serializer.rb
View File

@ -178,7 +178,7 @@ class SiteSerializer < ApplicationSerializer
end
def censored_regexp
WordWatcher.word_matcher_regexp(:censor)&.source
WordWatcher.serializable_word_matcher_regexp(:censor)
end
def custom_emoji_translation

2
app/serializers/watched_word_list_serializer.rb
View File

@ -17,7 +17,7 @@ class WatchedWordListSerializer < ApplicationSerializer
def compiled_regular_expressions
expressions = {}
actions.each do |action|
expressions[action] = WordWatcher.word_matcher_regexp(action)&.source
expressions[action] = WordWatcher.serializable_word_matcher_regexp(action)
end
expressions
end

2
app/serializers/watched_word_serializer.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
class WatchedWordSerializer < ApplicationSerializer
attributes :id, :word, :regexp, :replacement, :action
attributes :id, :word, :regexp, :replacement, :action, :case_sensitive
def regexp
WordWatcher.word_to_regexp(word, whole: true)

146
app/services/word_watcher.rb
View File

@ -18,16 +18,13 @@ class WordWatcher
end
def self.words_for_action(action)
words = WatchedWord
WatchedWord
.where(action: WatchedWord.actions[action.to_sym])
.limit(WatchedWord::MAX_WORDS_PER_ACTION)
.order(:id)
if WatchedWord.has_replacement?(action.to_sym)
words.pluck(:word, :replacement).to_h
else
words.pluck(:word)
end
.pluck(:word, :replacement, :case_sensitive)
.map { |w, r, c| [w, { replacement: r, case_sensitive: c }.compact] }
.to_h
end
def self.words_for_action_exists?(action)
@ -44,42 +41,55 @@ class WordWatcher
end
end
def self.serializable_word_matcher_regexp(action)
word_matcher_regexp_list(action)
.map { |r| { r.source => { case_sensitive: !r.casefold? } } }
end
# This regexp is run in miniracer, and the client JS app
# Make sure it is compatible with major browsers when changing
# hint: non-chrome browsers do not support 'lookbehind'
def self.word_matcher_regexp(action, raise_errors: false)
def self.word_matcher_regexp_list(action, raise_errors: false)
words = get_cached_words(action)
if words
if WatchedWord.has_replacement?(action.to_sym)
words = words.keys
end
words = words.map do |w|
word = word_to_regexp(w)
word = "(#{word})" if SiteSetting.watched_words_regular_expressions?
word
end
regexp = words.join('|')
if !SiteSetting.watched_words_regular_expressions?
regexp = "(#{regexp})"
regexp = "(?:\\W|^)#{regexp}(?=\\W|$)"
end
Regexp.new(regexp, Regexp::IGNORECASE)
return [] if words.blank?
grouped_words = { case_sensitive: [], case_insensitive: [] }
words.each do |w, attrs|
word = word_to_regexp(w)
word = "(#{word})" if SiteSetting.watched_words_regular_expressions?
group_key = attrs[:case_sensitive] ? :case_sensitive : :case_insensitive
grouped_words[group_key] << word
end
regexps = grouped_words
.select { |_, w| w.present? }
.transform_values { |w| w.join("|") }
if !SiteSetting.watched_words_regular_expressions?
regexps.transform_values! do |regexp|
regexp = "(#{regexp})"
"(?:\\W|^)#{regexp}(?=\\W|$)"
end
end
regexps
.map { |c, regexp| Regexp.new(regexp, c == :case_sensitive ? nil : Regexp::IGNORECASE) }
rescue RegexpError
raise if raise_errors
nil # Admin will be alerted via admin_dashboard_data.rb
[] # Admin will be alerted via admin_dashboard_data.rb
end
def self.word_matcher_regexps(action)
if words = get_cached_words(action)
words.map { |w, r| [word_to_regexp(w, whole: true), r] }.to_h
words.map { |w, opts| [word_to_regexp(w, whole: true), opts] }.to_h
end
end
def self.word_to_regexp(word, whole: false)
if SiteSetting.watched_words_regular_expressions?
# Strip ruby regexp format if present, we're going to make the whole thing
# case insensitive anyway
# Strip ruby regexp format if present
regexp = word.start_with?("(?-mix:") ? word[7..-2] : word
regexp = "(#{regexp})" if whole
return regexp
@ -99,32 +109,34 @@ class WordWatcher
end
def self.censor(html)
regexp = word_matcher_regexp(:censor)
return html if regexp.blank?
regexps = word_matcher_regexp_list(:censor)
return html if regexps.blank?
doc = Nokogiri::HTML5::fragment(html)
doc.traverse do |node|
node.content = censor_text_with_regexp(node.content, regexp) if node.text?
regexps.each do |regexp|
node.content = censor_text_with_regexp(node.content, regexp) if node.text?
end
end
doc.to_s
end
def self.censor_text(text)
regexp = word_matcher_regexp(:censor)
return text if regexp.blank?
regexps = word_matcher_regexp_list(:censor)
return text if regexps.blank?
censor_text_with_regexp(text, regexp)
regexps.inject(text) { |txt, regexp| censor_text_with_regexp(txt, regexp) }
end
def self.apply_to_text(text)
if regexp = word_matcher_regexp(:censor)
text = censor_text_with_regexp(text, regexp)
end
text = censor_text(text)
%i[replace link]
.flat_map { |type| word_matcher_regexps(type).to_a }
.reduce(text) do |t, (word_regexp, replacement)|
t.gsub(Regexp.new(word_regexp)) { |match| "#{match[0]}#{replacement}" }
.reduce(text) do |t, (word_regexp, attrs)|
case_flag = attrs[:case_sensitive] ? nil : Regexp::IGNORECASE
replace_text_with_regexp(t, Regexp.new(word_regexp, case_flag), attrs[:replacement])
end
end
@ -151,10 +163,19 @@ class WordWatcher
end
def word_matches_for_action?(action, all_matches: false)
regexp = self.class.word_matcher_regexp(action)
if regexp
regexps = self.class.word_matcher_regexp_list(action)
return if regexps.blank?
match_list = []
regexps.each do |regexp|
match = regexp.match(@raw)
return match if !all_matches || !match
if !all_matches
return match if match
next
end
next if !match
if SiteSetting.watched_words_regular_expressions?
set = Set.new
@ -165,25 +186,44 @@ class WordWatcher
set.add(m)
end
end
matches = set.to_a
else
matches = @raw.scan(regexp)
matches.flatten!
matches.uniq!
end
matches.compact!
matches.sort!
matches
else
false
match_list.concat(matches)
end
return if match_list.blank?
match_list.compact!
match_list.uniq!
match_list.sort!
match_list
end
def word_matches?(word, case_sensitive: false)
Regexp
.new(WordWatcher.word_to_regexp(word, whole: true), case_sensitive ? nil : Regexp::IGNORECASE)
.match?(@raw)
end
def self.replace_text_with_regexp(text, regexp, replacement)
text.gsub(regexp) do |match|
prefix = ""
# match may be prefixed with a non-word character from the non-capturing group
# Ensure this isn't replaced if watched words regular expression is disabled.
if !SiteSetting.watched_words_regular_expressions? && (match[0] =~ /\W/) != nil
prefix = "#{match[0]}"
end
"#{prefix}#{replacement}"
end
end
def word_matches?(word)
Regexp.new(WordWatcher.word_to_regexp(word, whole: true), Regexp::IGNORECASE).match?(@raw)
end
private
private_class_method :replace_text_with_regexp
def self.censor_text_with_regexp(text, regexp)
text.gsub(regexp) do |match|
@ -196,4 +236,6 @@ class WordWatcher
end
end
end
private_class_method :censor_text_with_regexp
end

3
config/locales/client.en.yml
View File

@ -5109,6 +5109,7 @@ en:
show_words:
one: "show %{count} word"
other: "show %{count} words"
case_sensitive: "(case-sensitive)"
download: Download
clear_all: Clear All
clear_all_confirm: "Are you sure you want to clear all watched words for the %{action} action?"
@ -5146,6 +5147,8 @@ en:
exists: "Already exists"
upload: "Add from file"
upload_successful: "Upload successful. Words have been added."
case_sensitivity_label: "Is case-sensitive"
case_sensitivity_description: "Only words with matching character casing"
test:
button_label: "Test"
modal_title: "%{action}: Test Watched Words"

7
db/migrate/20220712040959_add_case_sensitive_to_watched_words.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
class AddCaseSensitiveToWatchedWords < ActiveRecord::Migration[7.0]
def change
add_column :watched_words, :case_sensitive, :boolean, default: false, null: false
end
end

3
lib/pretty_text.rb
View File

@ -115,6 +115,7 @@ module PrettyText
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/object")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/deprecated")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/escape")
apply_es6_file(ctx, root_path, "discourse-common/addon/utils/watched-words")
apply_es6_file(ctx, root_path, "discourse/app/lib/to-markdown")
apply_es6_file(ctx, root_path, "discourse/app/lib/utilities")
@ -213,7 +214,7 @@ module PrettyText
__optInput.customEmojiTranslation = #{Plugin::CustomEmoji.translations.to_json};
__optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer;
__optInput.lookupUploadUrls = __lookupUploadUrls;
__optInput.censoredRegexp = #{WordWatcher.word_matcher_regexp(:censor)&.source.to_json};
__optInput.censoredRegexp = #{WordWatcher.serializable_word_matcher_regexp(:censor).to_json };
__optInput.watchedWordsReplace = #{WordWatcher.word_matcher_regexps(:replace).to_json};
__optInput.watchedWordsLink = #{WordWatcher.word_matcher_regexps(:link).to_json};
__optInput.additionalOptions = #{Site.markdown_additional_options.to_json};

6
lib/topic_creator.rb
View File

@ -180,8 +180,10 @@ class TopicCreator
if watched_words.present?
word_watcher = WordWatcher.new("#{@opts[:title]} #{@opts[:raw]}")
word_watcher_tags = topic.tags.map(&:name)
watched_words.each do |word, tags|
word_watcher_tags += tags.split(",") if word_watcher.word_matches?(word)
watched_words.each do |word, opts|
if word_watcher.word_matches?(word, case_sensitive: opts[:case_sensitive])
word_watcher_tags += opts[:replacement].split(",")
end
end
DiscourseTagging.tag_topic_by_names(topic, Discourse.system_user.guardian, word_watcher_tags)
end

15
lib/validators/censored_words_validator.rb
View File

@ -2,10 +2,11 @@
class CensoredWordsValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
words_regexp = censored_words_regexp
if WordWatcher.words_for_action(:censor).present? && !words_regexp.nil?
censored_words = censor_words(value, words_regexp)
words_regexps = WordWatcher.word_matcher_regexp_list(:censor)
if WordWatcher.words_for_action_exists?(:censor).present? && words_regexps.present?
censored_words = censor_words(value, words_regexps)
return if censored_words.blank?
record.errors.add(
attribute,
:contains_censored_words,
@ -16,8 +17,8 @@ class CensoredWordsValidator < ActiveModel::EachValidator
private
def censor_words(value, regexp)
censored_words = value.scan(regexp)
def censor_words(value, regexps)
censored_words = regexps.map { |r| value.scan(r) }
censored_words.flatten!
censored_words.compact!
censored_words.map!(&:strip)
@ -31,8 +32,4 @@ class CensoredWordsValidator < ActiveModel::EachValidator
censored_words.uniq!
censored_words.join(", ")
end
def censored_words_regexp
WordWatcher.word_matcher_regexp :censor
end
end

11
spec/fixtures/csv/words_case_sensitive.csv vendored Normal file
View File

@ -0,0 +1,11 @@
hello,"tag1,tag2",True
UN,"tag1,tag3",true
world,"tag2,tag3",FALSE
test,"tag1,tag3"
Can't render this file because it has a wrong number of fields in line 10.

19
spec/lib/topic_creator_spec.rb
View File

@ -104,6 +104,25 @@ RSpec.describe TopicCreator do
end
end
context 'when assigned via matched watched words' do
fab!(:word1) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag1.name) }
fab!(:word2) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag2.name) }
fab!(:word3) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag3.name, case_sensitive: true) }
it 'adds watched words as tags' do
topic = TopicCreator.create(
user,
Guardian.new(user),
valid_attrs.merge(
title: "This is a #{word1.word} title",
raw: "#{word2.word.upcase} is not the same as #{word3.word.upcase}")
)
expect(topic).to be_valid
expect(topic.tags).to contain_exactly(tag1, tag2)
end
end
context 'staff-only tags' do
before do
create_staff_only_tags(['alpha'])

6
spec/lib/validators/censored_words_validator_spec.rb
View File

@ -9,9 +9,9 @@ RSpec.describe CensoredWordsValidator do
context "when there are censored words for action" do
let!(:watched_word) { Fabricate(:watched_word, action: WatchedWord.actions[:censor], word: 'bad') }
context "when there is a nil word_matcher_regexp" do
context "when word_matcher_regexp_list is empty" do
before do
WordWatcher.stubs(:word_matcher_regexp).returns(nil)
WordWatcher.stubs(:word_matcher_regexp_list).returns([])
end
it "adds no errors to the record" do
@ -20,7 +20,7 @@ RSpec.describe CensoredWordsValidator do
end
end
context "when there is word_matcher_regexp" do
context "when word_matcher_regexp_list is not empty" do
context "when the new value does not contain the watched word" do
let(:value) { 'some new good text' }

20
spec/models/watched_word_spec.rb
View File

@ -23,6 +23,10 @@ RSpec.describe WatchedWord do
expect(described_class.create(word: "a**les").word).to eq('a*les')
end
it "is case-insensitive by default" do
expect(described_class.create(word: "Jest").case_sensitive?).to eq(false)
end
describe "action_key=" do
let(:w) { WatchedWord.new(word: "troll") }
@ -105,5 +109,21 @@ RSpec.describe WatchedWord do
word = Fabricate(:watched_word, action: described_class.actions[:link], word: "meta3", replacement: "/test")
expect(word.replacement).to eq("http://test.localhost/test")
end
it "sets case-sensitivity of a word" do
word = described_class.create_or_update_word(word: 'joker', action_key: :block, case_sensitive: true)
expect(word.case_sensitive?).to eq(true)
word = described_class.create_or_update_word(word: 'free', action_key: :block)
expect(word.case_sensitive?).to eq(false)
end
it "updates case-sensitivity of a word" do
existing = Fabricate(:watched_word, action: described_class.actions[:block], case_sensitive: true)
updated = described_class.create_or_update_word(word: existing.word, action_key: :block, case_sensitive: false)
expect(updated.case_sensitive?).to eq(false)
end
end
end

46
spec/requests/admin/watched_words_controller_spec.rb
View File

@ -26,6 +26,37 @@ RSpec.describe Admin::WatchedWordsController do
end
end
describe '#create' do
context 'logged in as admin' do
before do
sign_in(admin)
end
it 'creates a word with default case sensitivity' do
post '/admin/customize/watched_words.json', params: {
action_key: 'flag',
word: 'Deals'
}
expect(response.status).to eq(200)
expect(WatchedWord.take.word).to eq('Deals')
end
it 'creates a word with the given case sensitivity' do
post '/admin/customize/watched_words.json', params: {
action_key: 'flag',
word: 'PNG',
case_sensitive: true
}
expect(response.status).to eq(200)
expect(WatchedWord.take.case_sensitive?).to eq(true)
expect(WatchedWord.take.word).to eq('PNG')
end
end
end
describe '#upload' do
context 'logged in as admin' do
before do
@ -69,6 +100,21 @@ RSpec.describe Admin::WatchedWordsController do
expect(WatchedWord.pluck(:action).uniq).to eq([WatchedWord.actions[:tag]])
expect(UserHistory.where(action: UserHistory.actions[:watched_word_create]).count).to eq(2)
end
it 'creates case-sensitive words from the file' do
post '/admin/customize/watched_words/upload.json', params: {
action_key: 'flag',
file: Rack::Test::UploadedFile.new(file_from_fixtures("words_case_sensitive.csv", "csv"))
}
expect(response.status).to eq(200)
expect(WatchedWord.pluck(:word, :case_sensitive)).to contain_exactly(
['hello', true],
['UN', true],
['world', false],
['test', false]
)
end
end
end

8
spec/requests/api/schemas/json/site_response.json
View File

@ -444,10 +444,10 @@
]
},
"censored_regexp": {
"type": [
"string",
"null"
]
"type": "array",
"items": {
"type": "object"
}
},
"custom_emoji_translation": {
"type": "object",

142
spec/services/word_watcher_spec.rb
View File

@ -1,27 +1,92 @@
# frozen_string_literal: true
RSpec.describe WordWatcher do
let(:raw) { "Do you like liquorice?\n\nI really like them. One could even say that I am *addicted* to liquorice. And if\nyou can mix it up with some anise, then I'm in heaven ;)" }
describe WordWatcher do
let(:raw) do
<<~RAW.strip
Do you like liquorice?
I really like them. One could even say that I am *addicted* to liquorice. And if
you can mix it up with some anise, then I'm in heaven ;)
RAW
end
after do
Discourse.redis.flushdb
end
describe '.word_matcher_regexp' do
describe ".words_for_action" do
it "returns words with metadata including case sensitivity flag" do
Fabricate(:watched_word, action: WatchedWord.actions[:censor])
word1 = Fabricate(:watched_word, action: WatchedWord.actions[:block]).word
word2 = Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word
expect(described_class.words_for_action(:block)).to include(
word1 => { case_sensitive: false },
word2 => { case_sensitive: true }
)
end
it "returns word with metadata including replacement if word has replacement" do
word = Fabricate(
:watched_word,
action: WatchedWord.actions[:link],
replacement: "http://test.localhost/"
).word
expect(described_class.words_for_action(:link)).to include(
word => { case_sensitive: false, replacement: "http://test.localhost/" }
)
end
it "returns an empty hash when no words are present" do
expect(described_class.words_for_action(:tag)).to eq({})
end
end
describe ".word_matcher_regexp_list" do
let!(:word1) { Fabricate(:watched_word, action: WatchedWord.actions[:block]).word }
let!(:word2) { Fabricate(:watched_word, action: WatchedWord.actions[:block]).word }
let!(:word3) { Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word }
let!(:word4) { Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word }
context 'format of the result regexp' do
context "format of the result regexp" do
it "is correct when watched_words_regular_expressions = true" do
SiteSetting.watched_words_regular_expressions = true
regexp = described_class.word_matcher_regexp(:block)
expect(regexp.inspect).to eq("/(#{word1})|(#{word2})/i")
regexps = described_class.word_matcher_regexp_list(:block)
expect(regexps).to be_an(Array)
expect(regexps.map(&:inspect)).to contain_exactly("/(#{word1})|(#{word2})/i", "/(#{word3})|(#{word4})/")
end
it "is correct when watched_words_regular_expressions = false" do
SiteSetting.watched_words_regular_expressions = false
regexp = described_class.word_matcher_regexp(:block)
expect(regexp.inspect).to eq("/(?:\\W|^)(#{word1}|#{word2})(?=\\W|$)/i")
regexps = described_class.word_matcher_regexp_list(:block)
expect(regexps).to be_an(Array)
expect(regexps.map(&:inspect)).to contain_exactly("/(?:\\W|^)(#{word1}|#{word2})(?=\\W|$)/i", "/(?:\\W|^)(#{word3}|#{word4})(?=\\W|$)/")
end
it "is empty for an action without watched words" do
regexps = described_class.word_matcher_regexp_list(:censor)
expect(regexps).to be_an(Array)
expect(regexps).to be_empty
end
end
context "when regular expression is invalid" do
before do
SiteSetting.watched_words_regular_expressions = true
Fabricate(:watched_word, word: "Test[\S*", action: WatchedWord.actions[:block])
end
it "does not raise an exception by default" do
expect { described_class.word_matcher_regexp_list(:block) }.not_to raise_error
end
it "raises an exception with raise_errors set to true" do
expect { described_class.word_matcher_regexp_list(:block, raise_errors: true) }.to raise_error(RegexpError)
end
end
end
@ -187,6 +252,41 @@ RSpec.describe WordWatcher do
end
end
context "when case sensitive words are present" do
before do
Fabricate(
:watched_word,
word: "Discourse",
action: WatchedWord.actions[:block],
case_sensitive: true
)
end
context "when watched_words_regular_expressions = true" do
it "respects case sensitivity flag in matching words" do
SiteSetting.watched_words_regular_expressions = true
Fabricate(:watched_word, word: "p(rivate|ublic)", action: WatchedWord.actions[:block])
matches = described_class
.new("PUBLIC: Discourse is great for public discourse")
.word_matches_for_action?(:block, all_matches: true)
expect(matches).to contain_exactly("PUBLIC", "Discourse", "public")
end
end
context "when watched_words_regular_expressions = false" do
it "repects case sensitivity flag in matching" do
SiteSetting.watched_words_regular_expressions = false
Fabricate(:watched_word, word: "private", action: WatchedWord.actions[:block])
matches = described_class
.new("PRIVATE: Discourse is also great private discourse")
.word_matches_for_action?(:block, all_matches: true)
expect(matches).to contain_exactly("PRIVATE", "Discourse", "private")
end
end
end
end
end
@ -200,5 +300,31 @@ RSpec.describe WordWatcher do
expected = "hello #{described_class::REPLACEMENT_LETTER * 8} world replaced https://discourse.org"
expect(described_class.apply_to_text(text)).to eq(expected)
end
context "when watched_words_regular_expressions = true" do
it "replaces captured non-word prefix" do
SiteSetting.watched_words_regular_expressions = true
Fabricate(
:watched_word,
word: "\\Wplaceholder",
replacement: "replacement",
action: WatchedWord.actions[:replace]
)
text = "is \tplaceholder in https://notdiscourse.org"
expected = "is replacement in https://discourse.org"
expect(described_class.apply_to_text(text)).to eq(expected)
end
end
context "when watched_words_regular_expressions = false" do
it "maintains non-word character prefix" do
SiteSetting.watched_words_regular_expressions = false
text = "to replace and\thttps://notdiscourse.org"
expected = "replaced and\thttps://discourse.org"
expect(described_class.apply_to_text(text)).to eq(expected)
end
end
end
end