FEATURE: Add support for case-sensitive Watched Words (#17445)

* FEATURE: Add case-sensitivity flag to watched_words

Currently, all watched words are matched case-insensitively. This flag
allows a watched word to be flagged for case-sensitive matching.
To allow allow for backwards compatibility the flag is set to false by
default.

* FEATURE: Support case-sensitive creation of Watched Words via API

Extend admin creation and upload of Watched Words to support case
sensitive flag. This lays the ground work for supporting
case-insensitive matching of Watched Words.

Support for an extra column has also been introduced for the Watched
Words upload CSV file. The new column structure is as follows:

 word,replacement,case_sentive

* FEATURE: Enable case-sensitive matching of Watched Words

WordWatcher's word_matcher_regexp now returns a list of regular
expressions instead of one case-insensitive regular expression.

With the ability to flag a Watched Word as case-sensitive, an action
can have words of both sensitivities.This makes the use of the global
Regexp::IGNORECASE flag added to all words problematic.

To get around platform limitations around the use of subexpression level
switches/flags, a list of regular expressions is returned instead, one for each
case sensitivity.

Word matching has also been updated to use this list of regular expressions
instead of one.

* FEATURE: Use case-sensitive regular expressions for Watched Words

Update Watched Words regular expressions matching and processing to handle
the extra metadata which comes along with the introduction of
case-sensitive Watched Words.

This allows case-sensitive Watched Words to matched as such.

* DEV: Simplify type casting of case-sensitive flag from uploads

Use builtin semantics instead of a custom method for converting
string case flags in uploaded Watched Words to boolean.

* UX: Add case-sensitivity details to Admin Watched Words UI

Update Watched Word form to include a toggle for case-sensitivity.
This also adds support for, case-sensitive testing and matching of  Watched Word
in the admin UI.

* DEV: Code improvements from review feedback

 - Extract watched word regex creation out to a utility function
 - Make JS array presence check more explicit and readable

* DEV: Extract Watched Word regex creation to utility function

Clean-up work from review feedback. Reduce code duplication.

* DEV: Rename word_matcher_regexp to word_matcher_regexp_list

Since a list is returned now instead of a single regular expression,
change `word_matcher_regexp` to `word_matcher_regexp_list` to better communicate
this change.

* DEV:  Incorporate WordWatcher updates from upstream

Resolve conflicts and ensure apply_to_text does not remove non-word characters in matches
that aren't at the beginning of the line.

app/assets/javascripts/admin/addon/components/admin-watched-word.js

@@ -1,5 +1,5 @@
 import Component from "@ember/component";
-import { equal } from "@ember/object/computed";
+import { alias, equal } from "@ember/object/computed";
 import bootbox from "bootbox";
 import discourseComputed from "discourse-common/utils/decorators";
 import { action } from "@ember/object";
@@ -11,6 +11,7 @@ export default Component.extend({
   isReplace: equal("actionKey", "replace"),
   isTag: equal("actionKey", "tag"),
   isLink: equal("actionKey", "link"),
+  isCaseSensitive: alias("word.case_sensitive"),
 
   @discourseComputed("word.replacement")
   tags(replacement) {

app/assets/javascripts/admin/addon/components/watched-word-form.js

@@ -14,6 +14,7 @@ export default Component.extend({
   actionKey: null,
   showMessage: false,
   selectedTags: null,
+  isCaseSensitive: false,
 
   canReplace: equal("actionKey", "replace"),
   canTag: equal("actionKey", "tag"),
@@ -78,6 +79,7 @@ export default Component.extend({
               ? this.replacement
               : null,
           action: this.actionKey,
+          isCaseSensitive: this.isCaseSensitive,
         });
 
         watchedWord
@@ -90,6 +92,7 @@ export default Component.extend({
               selectedTags: [],
               showMessage: true,
               message: I18n.t("admin.watched_words.form.success"),
+              isCaseSensitive: false,
             });
             this.action(WatchedWord.create(result));
             schedule("afterRender", () =>

app/assets/javascripts/admin/addon/controllers/modals/admin-watched-word-test.js

@@ -2,6 +2,10 @@ import Controller from "@ember/controller";
 import ModalFunctionality from "discourse/mixins/modal-functionality";
 import discourseComputed from "discourse-common/utils/decorators";
 import { equal } from "@ember/object/computed";
+import {
+  createWatchedWordRegExp,
+  toWatchedWord,
+} from "discourse-common/utils/watched-words";
 
 export default Controller.extend(ModalFunctionality, {
   isReplace: equal("model.nameKey", "replace"),
@@ -16,16 +20,17 @@ export default Controller.extend(ModalFunctionality, {
     "isTag",
     "isLink"
   )
-  matches(value, regexpString, words, isReplace, isTag, isLink) {
+  matches(value, regexpList, words, isReplace, isTag, isLink) {
-    if (!value || !regexpString) {
+    if (!value || regexpList.length === 0) {
       return [];
     }
 
     if (isReplace || isLink) {
       const matches = [];
       words.forEach((word) => {
-        const regexp = new RegExp(word.regexp, "gi");
+        const regexp = createWatchedWordRegExp(word);
         let match;
+
         while ((match = regexp.exec(value)) !== null) {
           matches.push({
             match: match[1],
@@ -37,8 +42,9 @@ export default Controller.extend(ModalFunctionality, {
     } else if (isTag) {
       const matches = {};
       words.forEach((word) => {
-        const regexp = new RegExp(word.regexp, "gi");
+        const regexp = createWatchedWordRegExp(word);
         let match;
+
         while ((match = regexp.exec(value)) !== null) {
           if (!matches[match[1]]) {
             matches[match[1]] = new Set();
@@ -56,7 +62,14 @@ export default Controller.extend(ModalFunctionality, {
         tags: Array.from(entry[1]),
       }));
     } else {
-      return value.match(new RegExp(regexpString, "ig")) || [];
+      let matches = [];
+      regexpList.forEach((regexp) => {
+        const wordRegexp = createWatchedWordRegExp(toWatchedWord(regexp));
+
+        matches.push(...(value.match(wordRegexp) || []));
+      });
+
+      return matches;
     }
   },
 });

app/assets/javascripts/admin/addon/models/watched-word.js

@@ -14,6 +14,7 @@ const WatchedWord = EmberObject.extend({
           word: this.word,
           replacement: this.replacement,
           action_key: this.action,
+          case_sensitive: this.isCaseSensitive,
         },
         dataType: "json",
       }

app/assets/javascripts/admin/addon/templates/components/admin-watched-word.hbs

@@ -7,3 +7,6 @@
     <span class="tag">{{tag}}</span>
   {{/each}}
 {{/if}}
+{{#if this.isCaseSensitive}}
+  <span class="case-sensitive">{{i18n "admin.watched_words.case_sensitive"}}</span>
+{{/if}}

app/assets/javascripts/admin/addon/templates/components/watched-word-form.hbs

@@ -27,6 +27,14 @@
   </div>
 {{/if}}
 
+<div class="watched-word-input">
+  <label for="watched-case-sensitivity">{{i18n "admin.watched_words.form.case_sensitivity_label"}}</label>
+  <label class="case-sensitivity-checkbox">
+    <Input @type="checkbox" @checked={{this.isCaseSensitive}} disabled={{this.formSubmitted}} />
+    {{i18n "admin.watched_words.form.case_sensitivity_description"}}
+  </label>
+</div>
+
 <DButton @type="submit" @class="btn btn-primary" @action={{action "submit"}} @disabled={{this.formSubmitted}} @label="admin.watched_words.form.add" />
 
 {{#if this.showMessage}}

app/assets/javascripts/discourse-common/addon/utils/watched-words.js

@@ -0,0 +1,9 @@
+export function createWatchedWordRegExp(word) {
+  const caseFlag = word.case_sensitive ? "" : "i";
+  return new RegExp(word.regexp, `${caseFlag}g`);
+}
+
+export function toWatchedWord(regexp) {
+  const [[regexpString, options]] = Object.entries(regexp);
+  return { regexp: regexpString, ...options };
+}

app/assets/javascripts/discourse/tests/acceptance/admin-watched-words-test.js

@@ -7,6 +7,7 @@ import {
 } from "discourse/tests/helpers/qunit-helpers";
 import { click, fillIn, visit } from "@ember/test-helpers";
 import { test } from "qunit";
+import I18n from "I18n";
 
 acceptance("Admin - Watched Words", function (needs) {
   needs.user();
@@ -68,7 +69,23 @@ acceptance("Admin - Watched Words", function (needs) {
         found.push(true);
       }
     });
+
     assert.strictEqual(found.length, 1);
+    assert.strictEqual(count(".watched-words-list .case-sensitive"), 0);
+  });
+
+  test("add case-sensitve words", async function (assert) {
+    await visit("/admin/customize/watched_words/action/block");
+
+    click(".show-words-checkbox");
+    fillIn(".watched-word-form input", "Discourse");
+    click(".case-sensitivity-checkbox");
+
+    await click(".watched-word-form button");
+
+    assert
+      .dom(".watched-words-list .watched-word")
+      .hasText(`Discourse ${I18n.t("admin.watched_words.case_sensitive")}`);
   });
 
   test("remove words", async function (assert) {

app/assets/javascripts/discourse/tests/fixtures/watched-words-fixtures.js

@@ -2,18 +2,19 @@ export default {
   "/admin/customize/watched_words.json": {
     actions: ["block", "censor", "require_approval", "flag", "replace", "tag"],
     words: [
-      { id: 1, word: "liquorice", action: "block" },
+      { id: 1, word: "liquorice", action: "block", case_sensitive: false },
-      { id: 2, word: "anise", action: "block" },
+      { id: 2, word: "anise", action: "block", case_sensitive: false },
-      { id: 3, word: "pyramid", action: "flag" },
+      { id: 3, word: "pyramid", action: "flag", case_sensitive: false },
-      { id: 4, word: "scheme", action: "flag" },
+      { id: 4, word: "scheme", action: "flag", case_sensitive: false },
-      { id: 5, word: "coupon", action: "require_approval" },
+      { id: 5, word: "coupon", action: "require_approval", case_sensitive: false },
-      { id: 6, word: '<img src="x">', action: "block" },
+      { id: 6, word: '<img src="x">', action: "block", case_sensitive: false },
       {
         id: 7,
         word: "hi",
         regexp: "(hi)",
         replacement: "hello",
         action: "replace",
+        case_sensitive: false,
       },
       {
         id: 8,
@@ -21,15 +22,20 @@ export default {
         regexp: "(hello)",
         replacement: "greeting",
         action: "tag",
+        case_sensitive: false,
       },
     ],
     compiled_regular_expressions: {
-      block: '(?:\\W|^)(liquorice|anise|<img\\ src="x">)(?=\\W|$)',
+      block: [
-      censor: null,
+        { '(?:\\W|^)(liquorice|anise|<img\\ src="x">)(?=\\W|$)': { case_sensitive: false }, },
-      require_approval: "(?:\\W|^)(coupon)(?=\\W|$)",
+      ],
-      flag: "(?:\\W|^)(pyramid|scheme)(?=\\W|$)",
+      censor: [],
-      replace: "(?:\\W|^)(hi)(?=\\W|$)",
+      require_approval: [
-      tag: "(?:\\W|^)(hello)(?=\\W|$)",
+        { "(?:\\W|^)(coupon)(?=\\W|$)": { case_sensitive: false }, },
+      ],
+      flag: [{ "(?:\\W|^)(pyramid|scheme)(?=\\W|$)": {case_sensitive: false }, },],
+      replace: [{ "(?:\\W|^)(hi)(?=\\W|$)": { case_sensitive: false }},],
+      tag: [{ "(?:\\W|^)(hello)(?=\\W|$)": { case_sensitive: false }, },],
     },
   },
 };

app/assets/javascripts/discourse/tests/helpers/create-pretender.js

@@ -823,6 +823,7 @@ export function applyDefaultHandlers(pretender) {
   pretender.post("/admin/customize/watched_words.json", (request) => {
     const result = parsePostData(request.requestBody);
     result.id = new Date().getTime();
+    result.case_sensitive = result.case_sensitive === "true";
     return response(200, result);
   });
 

app/assets/javascripts/discourse/tests/unit/lib/pretty-text-test.js

@@ -1104,7 +1104,7 @@ eviltrout</p>
     assert.cookedOptions(
       "Pleased to meet you, but pleeeease call me later, xyz123",
       {
-        censoredRegexp: "(xyz*|plee+ase)",
+        censoredRegexp: [{ "(xyz*|plee+ase)": { case_sensitive: false } }],
       },
       "<p>Pleased to meet you, but ■■■■■■■■■ call me later, ■■■123</p>",
       "supports censoring"
@@ -1710,7 +1710,12 @@ var bar = 'bar';
 
   test("watched words replace", function (assert) {
     const opts = {
-      watchedWordsReplace: { "(?:\\W|^)(fun)(?=\\W|$)": "times" },
+      watchedWordsReplace: {
+        "(?:\\W|^)(fun)(?=\\W|$)": {
+          replacement: "times",
+          case_sensitive: false,
+        },
+      },
     };
 
     assert.cookedOptions("test fun funny", opts, "<p>test times funny</p>");
@@ -1719,7 +1724,12 @@ var bar = 'bar';
 
   test("watched words link", function (assert) {
     const opts = {
-      watchedWordsLink: { "(?:\\W|^)(fun)(?=\\W|$)": "https://discourse.org" },
+      watchedWordsLink: {
+        "(?:\\W|^)(fun)(?=\\W|$)": {
+          replacement: "https://discourse.org",
+          case_sensitive: false,
+        },
+      },
     };
 
     assert.cookedOptions(
@@ -1733,7 +1743,9 @@ var bar = 'bar';
     const maxMatches = 100; // same limit as MD watched-words-replace plugin
     const opts = {
       siteSettings: { watched_words_regular_expressions: true },
-      watchedWordsReplace: { "(\\bu?\\b)": "you" },
+      watchedWordsReplace: {
+        "(\\bu?\\b)": { replacement: "you", case_sensitive: false },
+      },
     };
 
     assert.cookedOptions(

app/assets/javascripts/pretty-text/addon/censored-words.js

@@ -1,15 +1,24 @@
-export function censorFn(regexpString, replacementLetter) {
+import {
-  if (regexpString) {
+  createWatchedWordRegExp,
-    let censorRegexp = new RegExp(regexpString, "ig");
+  toWatchedWord,
+} from "discourse-common/utils/watched-words";
+
+export function censorFn(regexpList, replacementLetter) {
+  if (regexpList.length) {
     replacementLetter = replacementLetter || "■";
+    let censorRegexps = regexpList.map((regexp) => {
+      return createWatchedWordRegExp(toWatchedWord(regexp));
+    });
 
     return function (text) {
-      text = text.replace(censorRegexp, (fullMatch, ...groupMatches) => {
+      censorRegexps.forEach((censorRegexp) => {
-        const stringMatch = groupMatches.find((g) => typeof g === "string");
+        text = text.replace(censorRegexp, (fullMatch, ...groupMatches) => {
-        return fullMatch.replace(
+          const stringMatch = groupMatches.find((g) => typeof g === "string");
-          stringMatch,
+          return fullMatch.replace(
-          new Array(stringMatch.length + 1).join(replacementLetter)
+            stringMatch,
-        );
+            new Array(stringMatch.length + 1).join(replacementLetter)
+          );
+        });
       });
 
       return text;

app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js

@@ -28,11 +28,11 @@ function censorTree(state, censor) {
 
 export function setup(helper) {
   helper.registerPlugin((md) => {
-    const censoredRegexp = md.options.discourse.censoredRegexp;
+    const censoredRegexps = md.options.discourse.censoredRegexp;
 
-    if (censoredRegexp) {
+    if (Array.isArray(censoredRegexps) && censoredRegexps.length > 0) {
       const replacement = String.fromCharCode(9632);
-      const censor = censorFn(censoredRegexp, replacement);
+      const censor = censorFn(censoredRegexps, replacement);
       md.core.ruler.push("censored", (state) => censorTree(state, censor));
     }
   });

app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js

@@ -1,3 +1,8 @@
+import {
+  createWatchedWordRegExp,
+  toWatchedWord,
+} from "discourse-common/utils/watched-words";
+
 const MAX_MATCHES = 100;
 
 function isLinkOpen(str) {
@@ -47,10 +52,12 @@ export function setup(helper) {
 
     if (md.options.discourse.watchedWordsReplace) {
       Object.entries(md.options.discourse.watchedWordsReplace).map(
-        ([word, replacement]) => {
+        ([regexpString, options]) => {
+          const word = toWatchedWord({ [regexpString]: options });
+
           matchers.push({
-            pattern: new RegExp(word, "gi"),
+            pattern: createWatchedWordRegExp(word),
-            replacement,
+            replacement: options.replacement,
             link: false,
           });
         }
@@ -59,10 +66,12 @@ export function setup(helper) {
 
     if (md.options.discourse.watchedWordsLink) {
       Object.entries(md.options.discourse.watchedWordsLink).map(
-        ([word, replacement]) => {
+        ([regexpString, options]) => {
+          const word = toWatchedWord({ [regexpString]: options });
+
           matchers.push({
-            pattern: new RegExp(word, "gi"),
+            pattern: createWatchedWordRegExp(word),
-            replacement,
+            replacement: options.replacement,
             link: true,
           });
         }

app/controllers/admin/watched_words_controller.rb

@@ -41,7 +41,8 @@ class Admin::WatchedWordsController < Admin::AdminController
             watched_word = WatchedWord.create_or_update_word(
               word: row[0],
               replacement: has_replacement ? row[1] : nil,
-              action_key: action_key
+              action_key: action_key,
+              case_sensitive: "true" == row[2]&.strip&.downcase
             )
             if watched_word.valid?
               StaffActionLogger.new(current_user).log_watched_words_creation(watched_word)
@@ -95,7 +96,6 @@ class Admin::WatchedWordsController < Admin::AdminController
   private
 
   def watched_words_params
-    params.permit(:id, :word, :replacement, :action_key)
+    params.permit(:id, :word, :replacement, :action_key, :case_sensitive)
   end
-
 end

app/models/admin_dashboard_data.rb

@@ -360,7 +360,7 @@ class AdminDashboardData
   def watched_words_check
     WatchedWord.actions.keys.each do |action|
       begin
-        WordWatcher.word_matcher_regexp(action, raise_errors: true)
+        WordWatcher.word_matcher_regexp_list(action, raise_errors: true)
       rescue RegexpError => e
         translated_action = I18n.t("admin_js.admin.watched_words.actions.#{action}")
         I18n.t('dashboard.watched_word_regexp_error', base_path: Discourse.base_path, action: translated_action)

app/models/watched_word.rb

@@ -65,6 +65,7 @@ class WatchedWord < ActiveRecord::Base
     w.replacement = params[:replacement] if params[:replacement]
     w.action_key = params[:action_key] if params[:action_key]
     w.action = params[:action] if params[:action]
+    w.case_sensitive = params[:case_sensitive] if !params[:case_sensitive].nil?
     w.save
     w
   end
@@ -94,12 +95,13 @@ end
 #
 # Table name: watched_words
 #
-#  id          :integer          not null, primary key
+#  id             :integer          not null, primary key
-#  word        :string           not null
+#  word           :string           not null
-#  action      :integer          not null
+#  action         :integer          not null
-#  created_at  :datetime         not null
+#  created_at     :datetime         not null
-#  updated_at  :datetime         not null
+#  updated_at     :datetime         not null
-#  replacement :string
+#  replacement    :string
+#  case_sensitive :boolean          default(FALSE), not null
 #
 # Indexes
 #

app/serializers/site_serializer.rb

@@ -178,7 +178,7 @@ class SiteSerializer < ApplicationSerializer
   end
 
   def censored_regexp
-    WordWatcher.word_matcher_regexp(:censor)&.source
+    WordWatcher.serializable_word_matcher_regexp(:censor)
   end
 
   def custom_emoji_translation

app/serializers/watched_word_list_serializer.rb

@@ -17,7 +17,7 @@ class WatchedWordListSerializer < ApplicationSerializer
   def compiled_regular_expressions
     expressions = {}
     actions.each do |action|
-      expressions[action] = WordWatcher.word_matcher_regexp(action)&.source
+      expressions[action] = WordWatcher.serializable_word_matcher_regexp(action)
     end
     expressions
   end

app/serializers/watched_word_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class WatchedWordSerializer < ApplicationSerializer
-  attributes :id, :word, :regexp, :replacement, :action
+  attributes :id, :word, :regexp, :replacement, :action, :case_sensitive
 
   def regexp
     WordWatcher.word_to_regexp(word, whole: true)

app/services/word_watcher.rb

@@ -18,16 +18,13 @@ class WordWatcher
   end
 
   def self.words_for_action(action)
-    words = WatchedWord
+    WatchedWord
       .where(action: WatchedWord.actions[action.to_sym])
       .limit(WatchedWord::MAX_WORDS_PER_ACTION)
       .order(:id)
-
+      .pluck(:word, :replacement, :case_sensitive)
-    if WatchedWord.has_replacement?(action.to_sym)
+      .map { |w, r, c| [w, { replacement: r, case_sensitive: c }.compact] }
-      words.pluck(:word, :replacement).to_h
+      .to_h
-    else
-      words.pluck(:word)
-    end
   end
 
   def self.words_for_action_exists?(action)
@@ -44,42 +41,55 @@ class WordWatcher
     end
   end
 
+  def self.serializable_word_matcher_regexp(action)
+    word_matcher_regexp_list(action)
+      .map { |r| { r.source => { case_sensitive: !r.casefold? } } }
+  end
+
   # This regexp is run in miniracer, and the client JS app
   # Make sure it is compatible with major browsers when changing
   # hint: non-chrome browsers do not support 'lookbehind'
-  def self.word_matcher_regexp(action, raise_errors: false)
+  def self.word_matcher_regexp_list(action, raise_errors: false)
     words = get_cached_words(action)
-    if words
+    return [] if words.blank?
-      if WatchedWord.has_replacement?(action.to_sym)
+
-        words = words.keys
+    grouped_words = { case_sensitive: [], case_insensitive: [] }
-      end
+
-      words = words.map do |w|
+    words.each do |w, attrs|
-        word = word_to_regexp(w)
+      word = word_to_regexp(w)
-        word = "(#{word})" if SiteSetting.watched_words_regular_expressions?
+      word = "(#{word})" if SiteSetting.watched_words_regular_expressions?
-        word
+
-      end
+      group_key = attrs[:case_sensitive] ? :case_sensitive : :case_insensitive
-      regexp = words.join('|')
+      grouped_words[group_key] << word
-      if !SiteSetting.watched_words_regular_expressions?
-        regexp = "(#{regexp})"
-        regexp = "(?:\\W|^)#{regexp}(?=\\W|$)"
-      end
-      Regexp.new(regexp, Regexp::IGNORECASE)
     end
+
+    regexps = grouped_words
+      .select { |_, w| w.present? }
+      .transform_values { |w| w.join("|") }
+
+    if !SiteSetting.watched_words_regular_expressions?
+      regexps.transform_values! do |regexp|
+        regexp = "(#{regexp})"
+        "(?:\\W|^)#{regexp}(?=\\W|$)"
+      end
+    end
+
+    regexps
+      .map { |c, regexp| Regexp.new(regexp, c == :case_sensitive ? nil : Regexp::IGNORECASE) }
   rescue RegexpError
     raise if raise_errors
-    nil # Admin will be alerted via admin_dashboard_data.rb
+    [] # Admin will be alerted via admin_dashboard_data.rb
   end
 
   def self.word_matcher_regexps(action)
     if words = get_cached_words(action)
-      words.map { |w, r| [word_to_regexp(w, whole: true), r] }.to_h
+      words.map { |w, opts| [word_to_regexp(w, whole: true), opts] }.to_h
     end
   end
 
   def self.word_to_regexp(word, whole: false)
     if SiteSetting.watched_words_regular_expressions?
-      # Strip ruby regexp format if present, we're going to make the whole thing
+      # Strip ruby regexp format if present
-      # case insensitive anyway
       regexp = word.start_with?("(?-mix:") ? word[7..-2] : word
       regexp = "(#{regexp})" if whole
       return regexp
@@ -99,32 +109,34 @@ class WordWatcher
   end
 
   def self.censor(html)
-    regexp = word_matcher_regexp(:censor)
+    regexps = word_matcher_regexp_list(:censor)
-    return html if regexp.blank?
+    return html if regexps.blank?
 
     doc = Nokogiri::HTML5::fragment(html)
     doc.traverse do |node|
-      node.content = censor_text_with_regexp(node.content, regexp) if node.text?
+      regexps.each do |regexp|
+        node.content = censor_text_with_regexp(node.content, regexp) if node.text?
+      end
     end
+
     doc.to_s
   end
 
   def self.censor_text(text)
-    regexp = word_matcher_regexp(:censor)
+    regexps = word_matcher_regexp_list(:censor)
-    return text if regexp.blank?
+    return text if regexps.blank?
 
-    censor_text_with_regexp(text, regexp)
+    regexps.inject(text) { |txt, regexp| censor_text_with_regexp(txt, regexp) }
   end
 
   def self.apply_to_text(text)
-    if regexp = word_matcher_regexp(:censor)
+    text = censor_text(text)
-      text = censor_text_with_regexp(text, regexp)
-    end
 
     %i[replace link]
       .flat_map { |type| word_matcher_regexps(type).to_a }
-      .reduce(text) do |t, (word_regexp, replacement)|
+      .reduce(text) do |t, (word_regexp, attrs)|
-        t.gsub(Regexp.new(word_regexp)) { |match| "#{match[0]}#{replacement}" }
+        case_flag = attrs[:case_sensitive] ? nil : Regexp::IGNORECASE
+        replace_text_with_regexp(t, Regexp.new(word_regexp, case_flag), attrs[:replacement])
       end
   end
 
@@ -151,10 +163,19 @@ class WordWatcher
   end
 
   def word_matches_for_action?(action, all_matches: false)
-    regexp = self.class.word_matcher_regexp(action)
+    regexps = self.class.word_matcher_regexp_list(action)
-    if regexp
+    return if regexps.blank?
+
+    match_list = []
+    regexps.each do |regexp|
       match = regexp.match(@raw)
-      return match if !all_matches || !match
+
+      if !all_matches
+        return match if match
+        next
+      end
+
+      next if !match
 
       if SiteSetting.watched_words_regular_expressions?
         set = Set.new
@@ -165,25 +186,44 @@ class WordWatcher
             set.add(m)
           end
         end
+
         matches = set.to_a
       else
         matches = @raw.scan(regexp)
         matches.flatten!
-        matches.uniq!
       end
-      matches.compact!
+
-      matches.sort!
+      match_list.concat(matches)
-      matches
+    end
-    else
+
-      false
+    return if match_list.blank?
+
+    match_list.compact!
+    match_list.uniq!
+    match_list.sort!
+    match_list
+  end
+
+  def word_matches?(word, case_sensitive: false)
+    Regexp
+      .new(WordWatcher.word_to_regexp(word, whole: true), case_sensitive ? nil : Regexp::IGNORECASE)
+      .match?(@raw)
+  end
+
+  def self.replace_text_with_regexp(text, regexp, replacement)
+    text.gsub(regexp) do |match|
+      prefix = ""
+      # match may be prefixed with a non-word character from the non-capturing group
+      # Ensure this isn't replaced if watched words regular expression is disabled.
+      if !SiteSetting.watched_words_regular_expressions? && (match[0] =~ /\W/) != nil
+        prefix = "#{match[0]}"
+      end
+
+      "#{prefix}#{replacement}"
     end
   end
 
-  def word_matches?(word)
+  private_class_method :replace_text_with_regexp
-    Regexp.new(WordWatcher.word_to_regexp(word, whole: true), Regexp::IGNORECASE).match?(@raw)
-  end
-
-  private
 
   def self.censor_text_with_regexp(text, regexp)
     text.gsub(regexp) do |match|
@@ -196,4 +236,6 @@ class WordWatcher
       end
     end
   end
+
+  private_class_method :censor_text_with_regexp
 end

config/locales/client.en.yml

@@ -5109,6 +5109,7 @@ en:
         show_words:
           one: "show %{count} word"
           other: "show %{count} words"
+        case_sensitive: "(case-sensitive)"
         download: Download
         clear_all: Clear All
         clear_all_confirm: "Are you sure you want to clear all watched words for the %{action} action?"
@@ -5146,6 +5147,8 @@ en:
           exists: "Already exists"
           upload: "Add from file"
           upload_successful: "Upload successful. Words have been added."
+          case_sensitivity_label: "Is case-sensitive"
+          case_sensitivity_description: "Only words with matching character casing"
         test:
           button_label: "Test"
           modal_title: "%{action}: Test Watched Words"

db/migrate/20220712040959_add_case_sensitive_to_watched_words.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddCaseSensitiveToWatchedWords < ActiveRecord::Migration[7.0]
+  def change
+    add_column :watched_words, :case_sensitive, :boolean, default: false, null: false
+  end
+end

lib/pretty_text.rb

@@ -115,6 +115,7 @@ module PrettyText
     apply_es6_file(ctx, root_path, "discourse-common/addon/lib/object")
     apply_es6_file(ctx, root_path, "discourse-common/addon/lib/deprecated")
     apply_es6_file(ctx, root_path, "discourse-common/addon/lib/escape")
+    apply_es6_file(ctx, root_path, "discourse-common/addon/utils/watched-words")
     apply_es6_file(ctx, root_path, "discourse/app/lib/to-markdown")
     apply_es6_file(ctx, root_path, "discourse/app/lib/utilities")
 
@@ -213,7 +214,7 @@ module PrettyText
         __optInput.customEmojiTranslation = #{Plugin::CustomEmoji.translations.to_json};
         __optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer;
         __optInput.lookupUploadUrls = __lookupUploadUrls;
-        __optInput.censoredRegexp = #{WordWatcher.word_matcher_regexp(:censor)&.source.to_json};
+        __optInput.censoredRegexp = #{WordWatcher.serializable_word_matcher_regexp(:censor).to_json };
         __optInput.watchedWordsReplace = #{WordWatcher.word_matcher_regexps(:replace).to_json};
         __optInput.watchedWordsLink = #{WordWatcher.word_matcher_regexps(:link).to_json};
         __optInput.additionalOptions = #{Site.markdown_additional_options.to_json};

lib/topic_creator.rb

@@ -180,8 +180,10 @@ class TopicCreator
     if watched_words.present?
       word_watcher = WordWatcher.new("#{@opts[:title]} #{@opts[:raw]}")
       word_watcher_tags = topic.tags.map(&:name)
-      watched_words.each do |word, tags|
+      watched_words.each do |word, opts|
-        word_watcher_tags += tags.split(",") if word_watcher.word_matches?(word)
+        if word_watcher.word_matches?(word, case_sensitive: opts[:case_sensitive])
+          word_watcher_tags += opts[:replacement].split(",")
+        end
       end
       DiscourseTagging.tag_topic_by_names(topic, Discourse.system_user.guardian, word_watcher_tags)
     end

lib/validators/censored_words_validator.rb

@@ -2,10 +2,11 @@
 
 class CensoredWordsValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
-    words_regexp = censored_words_regexp
+    words_regexps = WordWatcher.word_matcher_regexp_list(:censor)
-    if WordWatcher.words_for_action(:censor).present? && !words_regexp.nil?
+    if WordWatcher.words_for_action_exists?(:censor).present? && words_regexps.present?
-      censored_words = censor_words(value, words_regexp)
+      censored_words = censor_words(value, words_regexps)
       return if censored_words.blank?
+
       record.errors.add(
         attribute,
         :contains_censored_words,
@@ -16,8 +17,8 @@ class CensoredWordsValidator < ActiveModel::EachValidator
 
   private
 
-  def censor_words(value, regexp)
+  def censor_words(value, regexps)
-    censored_words = value.scan(regexp)
+    censored_words = regexps.map { |r| value.scan(r) }
     censored_words.flatten!
     censored_words.compact!
     censored_words.map!(&:strip)
@@ -31,8 +32,4 @@ class CensoredWordsValidator < ActiveModel::EachValidator
     censored_words.uniq!
     censored_words.join(", ")
   end
-
-  def censored_words_regexp
-    WordWatcher.word_matcher_regexp :censor
-  end
 end

spec/fixtures/csv/words_case_sensitive.csv

@@ -0,0 +1,11 @@
+hello,"tag1,tag2",True
+
+UN,"tag1,tag3",true
+
+
+world,"tag2,tag3",FALSE
+
+
+
+test,"tag1,tag3"
+

spec/lib/topic_creator_spec.rb

@@ -104,6 +104,25 @@ RSpec.describe TopicCreator do
         end
       end
 
+      context 'when assigned via matched watched words' do
+        fab!(:word1) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag1.name) }
+        fab!(:word2) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag2.name) }
+        fab!(:word3) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag3.name, case_sensitive: true) }
+
+        it 'adds watched words as tags' do
+          topic = TopicCreator.create(
+            user,
+            Guardian.new(user),
+            valid_attrs.merge(
+              title: "This is a #{word1.word} title",
+              raw: "#{word2.word.upcase} is not the same as #{word3.word.upcase}")
+          )
+
+          expect(topic).to be_valid
+          expect(topic.tags).to contain_exactly(tag1, tag2)
+        end
+      end
+
       context 'staff-only tags' do
         before do
           create_staff_only_tags(['alpha'])

spec/lib/validators/censored_words_validator_spec.rb

@@ -9,9 +9,9 @@ RSpec.describe CensoredWordsValidator do
     context "when there are censored words for action" do
       let!(:watched_word) { Fabricate(:watched_word, action: WatchedWord.actions[:censor], word: 'bad') }
 
-      context "when there is a nil word_matcher_regexp" do
+      context "when word_matcher_regexp_list is empty" do
         before do
-          WordWatcher.stubs(:word_matcher_regexp).returns(nil)
+          WordWatcher.stubs(:word_matcher_regexp_list).returns([])
         end
 
         it "adds no errors to the record" do
@@ -20,7 +20,7 @@ RSpec.describe CensoredWordsValidator do
         end
       end
 
-      context "when there is word_matcher_regexp" do
+      context "when word_matcher_regexp_list is not empty" do
         context "when the new value does not contain the watched word" do
           let(:value) { 'some new good text' }
 

spec/models/watched_word_spec.rb

@@ -23,6 +23,10 @@ RSpec.describe WatchedWord do
     expect(described_class.create(word: "a**les").word).to eq('a*les')
   end
 
+  it "is case-insensitive by default" do
+    expect(described_class.create(word: "Jest").case_sensitive?).to eq(false)
+  end
+
   describe "action_key=" do
     let(:w) { WatchedWord.new(word: "troll") }
 
@@ -105,5 +109,21 @@ RSpec.describe WatchedWord do
       word = Fabricate(:watched_word, action: described_class.actions[:link], word: "meta3", replacement: "/test")
       expect(word.replacement).to eq("http://test.localhost/test")
     end
+
+    it "sets case-sensitivity of a word" do
+      word = described_class.create_or_update_word(word: 'joker', action_key: :block, case_sensitive: true)
+      expect(word.case_sensitive?).to eq(true)
+
+      word = described_class.create_or_update_word(word: 'free', action_key: :block)
+      expect(word.case_sensitive?).to eq(false)
+    end
+
+    it "updates case-sensitivity of a word" do
+      existing = Fabricate(:watched_word, action: described_class.actions[:block], case_sensitive: true)
+      updated = described_class.create_or_update_word(word: existing.word, action_key: :block, case_sensitive: false)
+
+      expect(updated.case_sensitive?).to eq(false)
+    end
+
   end
 end

spec/requests/admin/watched_words_controller_spec.rb

@@ -26,6 +26,37 @@ RSpec.describe Admin::WatchedWordsController do
     end
   end
 
+  describe '#create' do
+    context 'logged in as admin' do
+      before do
+        sign_in(admin)
+      end
+
+      it 'creates a word with default case sensitivity' do
+        post '/admin/customize/watched_words.json', params: {
+          action_key: 'flag',
+          word: 'Deals'
+        }
+
+        expect(response.status).to eq(200)
+        expect(WatchedWord.take.word).to eq('Deals')
+      end
+
+      it 'creates a word with the given case sensitivity' do
+        post '/admin/customize/watched_words.json', params: {
+          action_key: 'flag',
+          word: 'PNG',
+          case_sensitive: true
+        }
+
+        expect(response.status).to eq(200)
+        expect(WatchedWord.take.case_sensitive?).to eq(true)
+        expect(WatchedWord.take.word).to eq('PNG')
+      end
+
+    end
+  end
+
   describe '#upload' do
     context 'logged in as admin' do
       before do
@@ -69,6 +100,21 @@ RSpec.describe Admin::WatchedWordsController do
         expect(WatchedWord.pluck(:action).uniq).to eq([WatchedWord.actions[:tag]])
         expect(UserHistory.where(action: UserHistory.actions[:watched_word_create]).count).to eq(2)
       end
+
+      it 'creates case-sensitive words from the file' do
+        post '/admin/customize/watched_words/upload.json', params: {
+          action_key: 'flag',
+          file: Rack::Test::UploadedFile.new(file_from_fixtures("words_case_sensitive.csv", "csv"))
+        }
+
+        expect(response.status).to eq(200)
+        expect(WatchedWord.pluck(:word, :case_sensitive)).to contain_exactly(
+          ['hello', true],
+          ['UN', true],
+          ['world', false],
+          ['test', false]
+        )
+      end
     end
   end
 

spec/requests/api/schemas/json/site_response.json

@@ -444,10 +444,10 @@
       ]
     },
     "censored_regexp": {
-      "type": [
+      "type": "array",
-        "string",
+      "items": {
-        "null"
+        "type": "object"
-      ]
+      }
     },
     "custom_emoji_translation": {
       "type": "object",

spec/services/word_watcher_spec.rb

@@ -1,27 +1,92 @@
 # frozen_string_literal: true
 
-RSpec.describe WordWatcher do
+describe WordWatcher do
-  let(:raw) { "Do you like liquorice?\n\nI really like them. One could even say that I am *addicted* to liquorice. And if\nyou can mix it up with some anise, then I'm in heaven ;)" }
+  let(:raw) do
+    <<~RAW.strip
+      Do you like liquorice?
+
+
+      I really like them. One could even say that I am *addicted* to liquorice. And if
+      you can mix it up with some anise, then I'm in heaven ;)
+    RAW
+  end
 
   after do
     Discourse.redis.flushdb
   end
 
-  describe '.word_matcher_regexp' do
+  describe ".words_for_action" do
+    it "returns words with metadata including case sensitivity flag" do
+      Fabricate(:watched_word, action: WatchedWord.actions[:censor])
+      word1 = Fabricate(:watched_word, action: WatchedWord.actions[:block]).word
+      word2 = Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word
+
+      expect(described_class.words_for_action(:block)).to include(
+        word1 => { case_sensitive: false },
+        word2 => { case_sensitive: true }
+      )
+    end
+
+    it "returns word with metadata including replacement if word has replacement" do
+      word = Fabricate(
+        :watched_word,
+        action: WatchedWord.actions[:link],
+        replacement: "http://test.localhost/"
+      ).word
+
+      expect(described_class.words_for_action(:link)).to include(
+        word => { case_sensitive: false, replacement: "http://test.localhost/" }
+      )
+    end
+
+    it "returns an empty hash when no words are present" do
+      expect(described_class.words_for_action(:tag)).to eq({})
+    end
+  end
+
+  describe ".word_matcher_regexp_list" do
     let!(:word1) { Fabricate(:watched_word, action: WatchedWord.actions[:block]).word }
     let!(:word2) { Fabricate(:watched_word, action: WatchedWord.actions[:block]).word }
+    let!(:word3) { Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word }
+    let!(:word4) { Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word }
 
-    context 'format of the result regexp' do
+    context "format of the result regexp" do
       it "is correct when watched_words_regular_expressions = true" do
         SiteSetting.watched_words_regular_expressions = true
-        regexp = described_class.word_matcher_regexp(:block)
+        regexps = described_class.word_matcher_regexp_list(:block)
-        expect(regexp.inspect).to eq("/(#{word1})|(#{word2})/i")
+
+        expect(regexps).to be_an(Array)
+        expect(regexps.map(&:inspect)).to contain_exactly("/(#{word1})|(#{word2})/i", "/(#{word3})|(#{word4})/")
       end
 
       it "is correct when watched_words_regular_expressions = false" do
         SiteSetting.watched_words_regular_expressions = false
-        regexp = described_class.word_matcher_regexp(:block)
+        regexps = described_class.word_matcher_regexp_list(:block)
-        expect(regexp.inspect).to eq("/(?:\\W|^)(#{word1}|#{word2})(?=\\W|$)/i")
+
+        expect(regexps).to be_an(Array)
+        expect(regexps.map(&:inspect)).to contain_exactly("/(?:\\W|^)(#{word1}|#{word2})(?=\\W|$)/i", "/(?:\\W|^)(#{word3}|#{word4})(?=\\W|$)/")
+      end
+
+      it "is empty for an action without watched words" do
+        regexps = described_class.word_matcher_regexp_list(:censor)
+
+        expect(regexps).to be_an(Array)
+        expect(regexps).to be_empty
+      end
+    end
+
+    context "when regular expression is invalid" do
+      before do
+        SiteSetting.watched_words_regular_expressions = true
+        Fabricate(:watched_word, word: "Test[\S*", action: WatchedWord.actions[:block])
+      end
+
+      it "does not raise an exception by default" do
+        expect { described_class.word_matcher_regexp_list(:block) }.not_to raise_error
+      end
+
+      it "raises an exception with raise_errors set to true" do
+        expect { described_class.word_matcher_regexp_list(:block, raise_errors: true) }.to raise_error(RegexpError)
       end
     end
   end
@@ -187,6 +252,41 @@ RSpec.describe WordWatcher do
         end
       end
 
+      context "when case sensitive words are present" do
+        before do
+          Fabricate(
+            :watched_word,
+            word: "Discourse",
+            action: WatchedWord.actions[:block],
+            case_sensitive: true
+          )
+        end
+
+        context "when watched_words_regular_expressions = true" do
+          it "respects case sensitivity flag in matching words" do
+            SiteSetting.watched_words_regular_expressions = true
+            Fabricate(:watched_word, word: "p(rivate|ublic)", action: WatchedWord.actions[:block])
+
+            matches = described_class
+              .new("PUBLIC: Discourse is great for public discourse")
+              .word_matches_for_action?(:block, all_matches: true)
+            expect(matches).to contain_exactly("PUBLIC", "Discourse", "public")
+          end
+        end
+
+        context "when watched_words_regular_expressions = false" do
+          it "repects case sensitivity flag in matching" do
+            SiteSetting.watched_words_regular_expressions = false
+            Fabricate(:watched_word, word: "private", action: WatchedWord.actions[:block])
+
+            matches = described_class
+              .new("PRIVATE: Discourse is also great private discourse")
+              .word_matches_for_action?(:block, all_matches: true)
+
+            expect(matches).to contain_exactly("PRIVATE", "Discourse", "private")
+          end
+        end
+      end
     end
   end
 
@@ -200,5 +300,31 @@ RSpec.describe WordWatcher do
       expected = "hello #{described_class::REPLACEMENT_LETTER * 8} world replaced https://discourse.org"
       expect(described_class.apply_to_text(text)).to eq(expected)
     end
+
+    context "when watched_words_regular_expressions = true" do
+      it "replaces captured non-word prefix" do
+        SiteSetting.watched_words_regular_expressions = true
+        Fabricate(
+          :watched_word,
+          word: "\\Wplaceholder",
+          replacement: "replacement",
+          action: WatchedWord.actions[:replace]
+        )
+
+        text = "is \tplaceholder in https://notdiscourse.org"
+        expected = "is replacement in https://discourse.org"
+        expect(described_class.apply_to_text(text)).to eq(expected)
+      end
+    end
+
+    context "when watched_words_regular_expressions = false" do
+      it "maintains non-word character prefix" do
+        SiteSetting.watched_words_regular_expressions = false
+
+        text = "to replace and\thttps://notdiscourse.org"
+        expected = "replaced and\thttps://discourse.org"
+        expect(described_class.apply_to_text(text)).to eq(expected)
+      end
+    end
   end
 end