Pleased to meet you, but ■■■■■■■■■ call me later, ■■■123
", "supports censoring" @@ -1710,7 +1710,12 @@ var bar = 'bar'; test("watched words replace", function (assert) { const opts = { - watchedWordsReplace: { "(?:\\W|^)(fun)(?=\\W|$)": "times" }, + watchedWordsReplace: { + "(?:\\W|^)(fun)(?=\\W|$)": { + replacement: "times", + case_sensitive: false, + }, + }, }; assert.cookedOptions("test fun funny", opts, "test times funny
"); @@ -1719,7 +1724,12 @@ var bar = 'bar'; test("watched words link", function (assert) { const opts = { - watchedWordsLink: { "(?:\\W|^)(fun)(?=\\W|$)": "https://discourse.org" }, + watchedWordsLink: { + "(?:\\W|^)(fun)(?=\\W|$)": { + replacement: "https://discourse.org", + case_sensitive: false, + }, + }, }; assert.cookedOptions( @@ -1733,7 +1743,9 @@ var bar = 'bar'; const maxMatches = 100; // same limit as MD watched-words-replace plugin const opts = { siteSettings: { watched_words_regular_expressions: true }, - watchedWordsReplace: { "(\\bu?\\b)": "you" }, + watchedWordsReplace: { + "(\\bu?\\b)": { replacement: "you", case_sensitive: false }, + }, }; assert.cookedOptions( diff --git a/app/assets/javascripts/pretty-text/addon/censored-words.js b/app/assets/javascripts/pretty-text/addon/censored-words.js index 6de0351fe34..fd41685dd19 100644 --- a/app/assets/javascripts/pretty-text/addon/censored-words.js +++ b/app/assets/javascripts/pretty-text/addon/censored-words.js @@ -1,15 +1,24 @@ -export function censorFn(regexpString, replacementLetter) { - if (regexpString) { - let censorRegexp = new RegExp(regexpString, "ig"); +import { + createWatchedWordRegExp, + toWatchedWord, +} from "discourse-common/utils/watched-words"; + +export function censorFn(regexpList, replacementLetter) { + if (regexpList.length) { replacementLetter = replacementLetter || "■"; + let censorRegexps = regexpList.map((regexp) => { + return createWatchedWordRegExp(toWatchedWord(regexp)); + }); return function (text) { - text = text.replace(censorRegexp, (fullMatch, ...groupMatches) => { - const stringMatch = groupMatches.find((g) => typeof g === "string"); - return fullMatch.replace( - stringMatch, - new Array(stringMatch.length + 1).join(replacementLetter) - ); + censorRegexps.forEach((censorRegexp) => { + text = text.replace(censorRegexp, (fullMatch, ...groupMatches) => { + const stringMatch = groupMatches.find((g) => typeof g === "string"); + return fullMatch.replace( + stringMatch, + new Array(stringMatch.length + 1).join(replacementLetter) + ); + }); }); return text; diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js index 34ab6c6f595..0456641ab8e 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js @@ -28,11 +28,11 @@ function censorTree(state, censor) { export function setup(helper) { helper.registerPlugin((md) => { - const censoredRegexp = md.options.discourse.censoredRegexp; + const censoredRegexps = md.options.discourse.censoredRegexp; - if (censoredRegexp) { + if (Array.isArray(censoredRegexps) && censoredRegexps.length > 0) { const replacement = String.fromCharCode(9632); - const censor = censorFn(censoredRegexp, replacement); + const censor = censorFn(censoredRegexps, replacement); md.core.ruler.push("censored", (state) => censorTree(state, censor)); } }); diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js b/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js index 7c09bf1c35c..3b1face02d1 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js @@ -1,3 +1,8 @@ +import { + createWatchedWordRegExp, + toWatchedWord, +} from "discourse-common/utils/watched-words"; + const MAX_MATCHES = 100; function isLinkOpen(str) { @@ -47,10 +52,12 @@ export function setup(helper) { if (md.options.discourse.watchedWordsReplace) { Object.entries(md.options.discourse.watchedWordsReplace).map( - ([word, replacement]) => { + ([regexpString, options]) => { + const word = toWatchedWord({ [regexpString]: options }); + matchers.push({ - pattern: new RegExp(word, "gi"), - replacement, + pattern: createWatchedWordRegExp(word), + replacement: options.replacement, link: false, }); } @@ -59,10 +66,12 @@ export function setup(helper) { if (md.options.discourse.watchedWordsLink) { Object.entries(md.options.discourse.watchedWordsLink).map( - ([word, replacement]) => { + ([regexpString, options]) => { + const word = toWatchedWord({ [regexpString]: options }); + matchers.push({ - pattern: new RegExp(word, "gi"), - replacement, + pattern: createWatchedWordRegExp(word), + replacement: options.replacement, link: true, }); } diff --git a/app/controllers/admin/watched_words_controller.rb b/app/controllers/admin/watched_words_controller.rb index c2cdd25ea50..6b825faedbf 100644 --- a/app/controllers/admin/watched_words_controller.rb +++ b/app/controllers/admin/watched_words_controller.rb @@ -41,7 +41,8 @@ class Admin::WatchedWordsController < Admin::AdminController watched_word = WatchedWord.create_or_update_word( word: row[0], replacement: has_replacement ? row[1] : nil, - action_key: action_key + action_key: action_key, + case_sensitive: "true" == row[2]&.strip&.downcase ) if watched_word.valid? StaffActionLogger.new(current_user).log_watched_words_creation(watched_word) @@ -95,7 +96,6 @@ class Admin::WatchedWordsController < Admin::AdminController private def watched_words_params - params.permit(:id, :word, :replacement, :action_key) + params.permit(:id, :word, :replacement, :action_key, :case_sensitive) end - end diff --git a/app/models/admin_dashboard_data.rb b/app/models/admin_dashboard_data.rb index 831dc099dc8..ae6dfa2535e 100644 --- a/app/models/admin_dashboard_data.rb +++ b/app/models/admin_dashboard_data.rb @@ -360,7 +360,7 @@ class AdminDashboardData def watched_words_check WatchedWord.actions.keys.each do |action| begin - WordWatcher.word_matcher_regexp(action, raise_errors: true) + WordWatcher.word_matcher_regexp_list(action, raise_errors: true) rescue RegexpError => e translated_action = I18n.t("admin_js.admin.watched_words.actions.#{action}") I18n.t('dashboard.watched_word_regexp_error', base_path: Discourse.base_path, action: translated_action) diff --git a/app/models/watched_word.rb b/app/models/watched_word.rb index fafbdd388a6..66cee4588ad 100644 --- a/app/models/watched_word.rb +++ b/app/models/watched_word.rb @@ -65,6 +65,7 @@ class WatchedWord < ActiveRecord::Base w.replacement = params[:replacement] if params[:replacement] w.action_key = params[:action_key] if params[:action_key] w.action = params[:action] if params[:action] + w.case_sensitive = params[:case_sensitive] if !params[:case_sensitive].nil? w.save w end @@ -94,12 +95,13 @@ end # # Table name: watched_words # -# id :integer not null, primary key -# word :string not null -# action :integer not null -# created_at :datetime not null -# updated_at :datetime not null -# replacement :string +# id :integer not null, primary key +# word :string not null +# action :integer not null +# created_at :datetime not null +# updated_at :datetime not null +# replacement :string +# case_sensitive :boolean default(FALSE), not null # # Indexes # diff --git a/app/serializers/site_serializer.rb b/app/serializers/site_serializer.rb index 76aafb1e2d7..b7abf74c3f5 100644 --- a/app/serializers/site_serializer.rb +++ b/app/serializers/site_serializer.rb @@ -178,7 +178,7 @@ class SiteSerializer < ApplicationSerializer end def censored_regexp - WordWatcher.word_matcher_regexp(:censor)&.source + WordWatcher.serializable_word_matcher_regexp(:censor) end def custom_emoji_translation diff --git a/app/serializers/watched_word_list_serializer.rb b/app/serializers/watched_word_list_serializer.rb index 8c024d525d4..ec3cac72a90 100644 --- a/app/serializers/watched_word_list_serializer.rb +++ b/app/serializers/watched_word_list_serializer.rb @@ -17,7 +17,7 @@ class WatchedWordListSerializer < ApplicationSerializer def compiled_regular_expressions expressions = {} actions.each do |action| - expressions[action] = WordWatcher.word_matcher_regexp(action)&.source + expressions[action] = WordWatcher.serializable_word_matcher_regexp(action) end expressions end diff --git a/app/serializers/watched_word_serializer.rb b/app/serializers/watched_word_serializer.rb index 070da701fd2..a798091f282 100644 --- a/app/serializers/watched_word_serializer.rb +++ b/app/serializers/watched_word_serializer.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class WatchedWordSerializer < ApplicationSerializer - attributes :id, :word, :regexp, :replacement, :action + attributes :id, :word, :regexp, :replacement, :action, :case_sensitive def regexp WordWatcher.word_to_regexp(word, whole: true) diff --git a/app/services/word_watcher.rb b/app/services/word_watcher.rb index c61d0deff68..4f5b516b624 100644 --- a/app/services/word_watcher.rb +++ b/app/services/word_watcher.rb @@ -18,16 +18,13 @@ class WordWatcher end def self.words_for_action(action) - words = WatchedWord + WatchedWord .where(action: WatchedWord.actions[action.to_sym]) .limit(WatchedWord::MAX_WORDS_PER_ACTION) .order(:id) - - if WatchedWord.has_replacement?(action.to_sym) - words.pluck(:word, :replacement).to_h - else - words.pluck(:word) - end + .pluck(:word, :replacement, :case_sensitive) + .map { |w, r, c| [w, { replacement: r, case_sensitive: c }.compact] } + .to_h end def self.words_for_action_exists?(action) @@ -44,42 +41,55 @@ class WordWatcher end end + def self.serializable_word_matcher_regexp(action) + word_matcher_regexp_list(action) + .map { |r| { r.source => { case_sensitive: !r.casefold? } } } + end + # This regexp is run in miniracer, and the client JS app # Make sure it is compatible with major browsers when changing # hint: non-chrome browsers do not support 'lookbehind' - def self.word_matcher_regexp(action, raise_errors: false) + def self.word_matcher_regexp_list(action, raise_errors: false) words = get_cached_words(action) - if words - if WatchedWord.has_replacement?(action.to_sym) - words = words.keys - end - words = words.map do |w| - word = word_to_regexp(w) - word = "(#{word})" if SiteSetting.watched_words_regular_expressions? - word - end - regexp = words.join('|') - if !SiteSetting.watched_words_regular_expressions? - regexp = "(#{regexp})" - regexp = "(?:\\W|^)#{regexp}(?=\\W|$)" - end - Regexp.new(regexp, Regexp::IGNORECASE) + return [] if words.blank? + + grouped_words = { case_sensitive: [], case_insensitive: [] } + + words.each do |w, attrs| + word = word_to_regexp(w) + word = "(#{word})" if SiteSetting.watched_words_regular_expressions? + + group_key = attrs[:case_sensitive] ? :case_sensitive : :case_insensitive + grouped_words[group_key] << word end + + regexps = grouped_words + .select { |_, w| w.present? } + .transform_values { |w| w.join("|") } + + if !SiteSetting.watched_words_regular_expressions? + regexps.transform_values! do |regexp| + regexp = "(#{regexp})" + "(?:\\W|^)#{regexp}(?=\\W|$)" + end + end + + regexps + .map { |c, regexp| Regexp.new(regexp, c == :case_sensitive ? nil : Regexp::IGNORECASE) } rescue RegexpError raise if raise_errors - nil # Admin will be alerted via admin_dashboard_data.rb + [] # Admin will be alerted via admin_dashboard_data.rb end def self.word_matcher_regexps(action) if words = get_cached_words(action) - words.map { |w, r| [word_to_regexp(w, whole: true), r] }.to_h + words.map { |w, opts| [word_to_regexp(w, whole: true), opts] }.to_h end end def self.word_to_regexp(word, whole: false) if SiteSetting.watched_words_regular_expressions? - # Strip ruby regexp format if present, we're going to make the whole thing - # case insensitive anyway + # Strip ruby regexp format if present regexp = word.start_with?("(?-mix:") ? word[7..-2] : word regexp = "(#{regexp})" if whole return regexp @@ -99,32 +109,34 @@ class WordWatcher end def self.censor(html) - regexp = word_matcher_regexp(:censor) - return html if regexp.blank? + regexps = word_matcher_regexp_list(:censor) + return html if regexps.blank? doc = Nokogiri::HTML5::fragment(html) doc.traverse do |node| - node.content = censor_text_with_regexp(node.content, regexp) if node.text? + regexps.each do |regexp| + node.content = censor_text_with_regexp(node.content, regexp) if node.text? + end end + doc.to_s end def self.censor_text(text) - regexp = word_matcher_regexp(:censor) - return text if regexp.blank? + regexps = word_matcher_regexp_list(:censor) + return text if regexps.blank? - censor_text_with_regexp(text, regexp) + regexps.inject(text) { |txt, regexp| censor_text_with_regexp(txt, regexp) } end def self.apply_to_text(text) - if regexp = word_matcher_regexp(:censor) - text = censor_text_with_regexp(text, regexp) - end + text = censor_text(text) %i[replace link] .flat_map { |type| word_matcher_regexps(type).to_a } - .reduce(text) do |t, (word_regexp, replacement)| - t.gsub(Regexp.new(word_regexp)) { |match| "#{match[0]}#{replacement}" } + .reduce(text) do |t, (word_regexp, attrs)| + case_flag = attrs[:case_sensitive] ? nil : Regexp::IGNORECASE + replace_text_with_regexp(t, Regexp.new(word_regexp, case_flag), attrs[:replacement]) end end @@ -151,10 +163,19 @@ class WordWatcher end def word_matches_for_action?(action, all_matches: false) - regexp = self.class.word_matcher_regexp(action) - if regexp + regexps = self.class.word_matcher_regexp_list(action) + return if regexps.blank? + + match_list = [] + regexps.each do |regexp| match = regexp.match(@raw) - return match if !all_matches || !match + + if !all_matches + return match if match + next + end + + next if !match if SiteSetting.watched_words_regular_expressions? set = Set.new @@ -165,25 +186,44 @@ class WordWatcher set.add(m) end end + matches = set.to_a else matches = @raw.scan(regexp) matches.flatten! - matches.uniq! end - matches.compact! - matches.sort! - matches - else - false + + match_list.concat(matches) + end + + return if match_list.blank? + + match_list.compact! + match_list.uniq! + match_list.sort! + match_list + end + + def word_matches?(word, case_sensitive: false) + Regexp + .new(WordWatcher.word_to_regexp(word, whole: true), case_sensitive ? nil : Regexp::IGNORECASE) + .match?(@raw) + end + + def self.replace_text_with_regexp(text, regexp, replacement) + text.gsub(regexp) do |match| + prefix = "" + # match may be prefixed with a non-word character from the non-capturing group + # Ensure this isn't replaced if watched words regular expression is disabled. + if !SiteSetting.watched_words_regular_expressions? && (match[0] =~ /\W/) != nil + prefix = "#{match[0]}" + end + + "#{prefix}#{replacement}" end end - def word_matches?(word) - Regexp.new(WordWatcher.word_to_regexp(word, whole: true), Regexp::IGNORECASE).match?(@raw) - end - - private + private_class_method :replace_text_with_regexp def self.censor_text_with_regexp(text, regexp) text.gsub(regexp) do |match| @@ -196,4 +236,6 @@ class WordWatcher end end end + + private_class_method :censor_text_with_regexp end diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 67ce660c75f..3bf0e5b6375 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -5109,6 +5109,7 @@ en: show_words: one: "show %{count} word" other: "show %{count} words" + case_sensitive: "(case-sensitive)" download: Download clear_all: Clear All clear_all_confirm: "Are you sure you want to clear all watched words for the %{action} action?" @@ -5146,6 +5147,8 @@ en: exists: "Already exists" upload: "Add from file" upload_successful: "Upload successful. Words have been added." + case_sensitivity_label: "Is case-sensitive" + case_sensitivity_description: "Only words with matching character casing" test: button_label: "Test" modal_title: "%{action}: Test Watched Words" diff --git a/db/migrate/20220712040959_add_case_sensitive_to_watched_words.rb b/db/migrate/20220712040959_add_case_sensitive_to_watched_words.rb new file mode 100644 index 00000000000..d21de1eecc1 --- /dev/null +++ b/db/migrate/20220712040959_add_case_sensitive_to_watched_words.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddCaseSensitiveToWatchedWords < ActiveRecord::Migration[7.0] + def change + add_column :watched_words, :case_sensitive, :boolean, default: false, null: false + end +end diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb index 57c72ff2bd6..d80df5bbf8d 100644 --- a/lib/pretty_text.rb +++ b/lib/pretty_text.rb @@ -115,6 +115,7 @@ module PrettyText apply_es6_file(ctx, root_path, "discourse-common/addon/lib/object") apply_es6_file(ctx, root_path, "discourse-common/addon/lib/deprecated") apply_es6_file(ctx, root_path, "discourse-common/addon/lib/escape") + apply_es6_file(ctx, root_path, "discourse-common/addon/utils/watched-words") apply_es6_file(ctx, root_path, "discourse/app/lib/to-markdown") apply_es6_file(ctx, root_path, "discourse/app/lib/utilities") @@ -213,7 +214,7 @@ module PrettyText __optInput.customEmojiTranslation = #{Plugin::CustomEmoji.translations.to_json}; __optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer; __optInput.lookupUploadUrls = __lookupUploadUrls; - __optInput.censoredRegexp = #{WordWatcher.word_matcher_regexp(:censor)&.source.to_json}; + __optInput.censoredRegexp = #{WordWatcher.serializable_word_matcher_regexp(:censor).to_json }; __optInput.watchedWordsReplace = #{WordWatcher.word_matcher_regexps(:replace).to_json}; __optInput.watchedWordsLink = #{WordWatcher.word_matcher_regexps(:link).to_json}; __optInput.additionalOptions = #{Site.markdown_additional_options.to_json}; diff --git a/lib/topic_creator.rb b/lib/topic_creator.rb index c7e812f8160..ca98d6a71a7 100644 --- a/lib/topic_creator.rb +++ b/lib/topic_creator.rb @@ -180,8 +180,10 @@ class TopicCreator if watched_words.present? word_watcher = WordWatcher.new("#{@opts[:title]} #{@opts[:raw]}") word_watcher_tags = topic.tags.map(&:name) - watched_words.each do |word, tags| - word_watcher_tags += tags.split(",") if word_watcher.word_matches?(word) + watched_words.each do |word, opts| + if word_watcher.word_matches?(word, case_sensitive: opts[:case_sensitive]) + word_watcher_tags += opts[:replacement].split(",") + end end DiscourseTagging.tag_topic_by_names(topic, Discourse.system_user.guardian, word_watcher_tags) end diff --git a/lib/validators/censored_words_validator.rb b/lib/validators/censored_words_validator.rb index 0c72e51a311..eacc69f9e5d 100644 --- a/lib/validators/censored_words_validator.rb +++ b/lib/validators/censored_words_validator.rb @@ -2,10 +2,11 @@ class CensoredWordsValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - words_regexp = censored_words_regexp - if WordWatcher.words_for_action(:censor).present? && !words_regexp.nil? - censored_words = censor_words(value, words_regexp) + words_regexps = WordWatcher.word_matcher_regexp_list(:censor) + if WordWatcher.words_for_action_exists?(:censor).present? && words_regexps.present? + censored_words = censor_words(value, words_regexps) return if censored_words.blank? + record.errors.add( attribute, :contains_censored_words, @@ -16,8 +17,8 @@ class CensoredWordsValidator < ActiveModel::EachValidator private - def censor_words(value, regexp) - censored_words = value.scan(regexp) + def censor_words(value, regexps) + censored_words = regexps.map { |r| value.scan(r) } censored_words.flatten! censored_words.compact! censored_words.map!(&:strip) @@ -31,8 +32,4 @@ class CensoredWordsValidator < ActiveModel::EachValidator censored_words.uniq! censored_words.join(", ") end - - def censored_words_regexp - WordWatcher.word_matcher_regexp :censor - end end diff --git a/spec/fixtures/csv/words_case_sensitive.csv b/spec/fixtures/csv/words_case_sensitive.csv new file mode 100644 index 00000000000..18e4d05e407 --- /dev/null +++ b/spec/fixtures/csv/words_case_sensitive.csv @@ -0,0 +1,11 @@ +hello,"tag1,tag2",True + +UN,"tag1,tag3",true + + +world,"tag2,tag3",FALSE + + + +test,"tag1,tag3" + diff --git a/spec/lib/topic_creator_spec.rb b/spec/lib/topic_creator_spec.rb index f1fd99404de..cb90c464809 100644 --- a/spec/lib/topic_creator_spec.rb +++ b/spec/lib/topic_creator_spec.rb @@ -104,6 +104,25 @@ RSpec.describe TopicCreator do end end + context 'when assigned via matched watched words' do + fab!(:word1) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag1.name) } + fab!(:word2) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag2.name) } + fab!(:word3) { Fabricate(:watched_word, action: WatchedWord.actions[:tag], replacement: tag3.name, case_sensitive: true) } + + it 'adds watched words as tags' do + topic = TopicCreator.create( + user, + Guardian.new(user), + valid_attrs.merge( + title: "This is a #{word1.word} title", + raw: "#{word2.word.upcase} is not the same as #{word3.word.upcase}") + ) + + expect(topic).to be_valid + expect(topic.tags).to contain_exactly(tag1, tag2) + end + end + context 'staff-only tags' do before do create_staff_only_tags(['alpha']) diff --git a/spec/lib/validators/censored_words_validator_spec.rb b/spec/lib/validators/censored_words_validator_spec.rb index 597884092b7..9a64e7b455a 100644 --- a/spec/lib/validators/censored_words_validator_spec.rb +++ b/spec/lib/validators/censored_words_validator_spec.rb @@ -9,9 +9,9 @@ RSpec.describe CensoredWordsValidator do context "when there are censored words for action" do let!(:watched_word) { Fabricate(:watched_word, action: WatchedWord.actions[:censor], word: 'bad') } - context "when there is a nil word_matcher_regexp" do + context "when word_matcher_regexp_list is empty" do before do - WordWatcher.stubs(:word_matcher_regexp).returns(nil) + WordWatcher.stubs(:word_matcher_regexp_list).returns([]) end it "adds no errors to the record" do @@ -20,7 +20,7 @@ RSpec.describe CensoredWordsValidator do end end - context "when there is word_matcher_regexp" do + context "when word_matcher_regexp_list is not empty" do context "when the new value does not contain the watched word" do let(:value) { 'some new good text' } diff --git a/spec/models/watched_word_spec.rb b/spec/models/watched_word_spec.rb index 9a9192f3938..cf3ab22618d 100644 --- a/spec/models/watched_word_spec.rb +++ b/spec/models/watched_word_spec.rb @@ -23,6 +23,10 @@ RSpec.describe WatchedWord do expect(described_class.create(word: "a**les").word).to eq('a*les') end + it "is case-insensitive by default" do + expect(described_class.create(word: "Jest").case_sensitive?).to eq(false) + end + describe "action_key=" do let(:w) { WatchedWord.new(word: "troll") } @@ -105,5 +109,21 @@ RSpec.describe WatchedWord do word = Fabricate(:watched_word, action: described_class.actions[:link], word: "meta3", replacement: "/test") expect(word.replacement).to eq("http://test.localhost/test") end + + it "sets case-sensitivity of a word" do + word = described_class.create_or_update_word(word: 'joker', action_key: :block, case_sensitive: true) + expect(word.case_sensitive?).to eq(true) + + word = described_class.create_or_update_word(word: 'free', action_key: :block) + expect(word.case_sensitive?).to eq(false) + end + + it "updates case-sensitivity of a word" do + existing = Fabricate(:watched_word, action: described_class.actions[:block], case_sensitive: true) + updated = described_class.create_or_update_word(word: existing.word, action_key: :block, case_sensitive: false) + + expect(updated.case_sensitive?).to eq(false) + end + end end diff --git a/spec/requests/admin/watched_words_controller_spec.rb b/spec/requests/admin/watched_words_controller_spec.rb index 6def887c57a..d3037b9fae5 100644 --- a/spec/requests/admin/watched_words_controller_spec.rb +++ b/spec/requests/admin/watched_words_controller_spec.rb @@ -26,6 +26,37 @@ RSpec.describe Admin::WatchedWordsController do end end + describe '#create' do + context 'logged in as admin' do + before do + sign_in(admin) + end + + it 'creates a word with default case sensitivity' do + post '/admin/customize/watched_words.json', params: { + action_key: 'flag', + word: 'Deals' + } + + expect(response.status).to eq(200) + expect(WatchedWord.take.word).to eq('Deals') + end + + it 'creates a word with the given case sensitivity' do + post '/admin/customize/watched_words.json', params: { + action_key: 'flag', + word: 'PNG', + case_sensitive: true + } + + expect(response.status).to eq(200) + expect(WatchedWord.take.case_sensitive?).to eq(true) + expect(WatchedWord.take.word).to eq('PNG') + end + + end + end + describe '#upload' do context 'logged in as admin' do before do @@ -69,6 +100,21 @@ RSpec.describe Admin::WatchedWordsController do expect(WatchedWord.pluck(:action).uniq).to eq([WatchedWord.actions[:tag]]) expect(UserHistory.where(action: UserHistory.actions[:watched_word_create]).count).to eq(2) end + + it 'creates case-sensitive words from the file' do + post '/admin/customize/watched_words/upload.json', params: { + action_key: 'flag', + file: Rack::Test::UploadedFile.new(file_from_fixtures("words_case_sensitive.csv", "csv")) + } + + expect(response.status).to eq(200) + expect(WatchedWord.pluck(:word, :case_sensitive)).to contain_exactly( + ['hello', true], + ['UN', true], + ['world', false], + ['test', false] + ) + end end end diff --git a/spec/requests/api/schemas/json/site_response.json b/spec/requests/api/schemas/json/site_response.json index fd3ddc05290..a07f3c9afe1 100644 --- a/spec/requests/api/schemas/json/site_response.json +++ b/spec/requests/api/schemas/json/site_response.json @@ -444,10 +444,10 @@ ] }, "censored_regexp": { - "type": [ - "string", - "null" - ] + "type": "array", + "items": { + "type": "object" + } }, "custom_emoji_translation": { "type": "object", diff --git a/spec/services/word_watcher_spec.rb b/spec/services/word_watcher_spec.rb index 936e0ab8310..9b280547b29 100644 --- a/spec/services/word_watcher_spec.rb +++ b/spec/services/word_watcher_spec.rb @@ -1,27 +1,92 @@ # frozen_string_literal: true -RSpec.describe WordWatcher do - let(:raw) { "Do you like liquorice?\n\nI really like them. One could even say that I am *addicted* to liquorice. And if\nyou can mix it up with some anise, then I'm in heaven ;)" } +describe WordWatcher do + let(:raw) do + <<~RAW.strip + Do you like liquorice? + + + I really like them. One could even say that I am *addicted* to liquorice. And if + you can mix it up with some anise, then I'm in heaven ;) + RAW + end after do Discourse.redis.flushdb end - describe '.word_matcher_regexp' do + describe ".words_for_action" do + it "returns words with metadata including case sensitivity flag" do + Fabricate(:watched_word, action: WatchedWord.actions[:censor]) + word1 = Fabricate(:watched_word, action: WatchedWord.actions[:block]).word + word2 = Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word + + expect(described_class.words_for_action(:block)).to include( + word1 => { case_sensitive: false }, + word2 => { case_sensitive: true } + ) + end + + it "returns word with metadata including replacement if word has replacement" do + word = Fabricate( + :watched_word, + action: WatchedWord.actions[:link], + replacement: "http://test.localhost/" + ).word + + expect(described_class.words_for_action(:link)).to include( + word => { case_sensitive: false, replacement: "http://test.localhost/" } + ) + end + + it "returns an empty hash when no words are present" do + expect(described_class.words_for_action(:tag)).to eq({}) + end + end + + describe ".word_matcher_regexp_list" do let!(:word1) { Fabricate(:watched_word, action: WatchedWord.actions[:block]).word } let!(:word2) { Fabricate(:watched_word, action: WatchedWord.actions[:block]).word } + let!(:word3) { Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word } + let!(:word4) { Fabricate(:watched_word, action: WatchedWord.actions[:block], case_sensitive: true).word } - context 'format of the result regexp' do + context "format of the result regexp" do it "is correct when watched_words_regular_expressions = true" do SiteSetting.watched_words_regular_expressions = true - regexp = described_class.word_matcher_regexp(:block) - expect(regexp.inspect).to eq("/(#{word1})|(#{word2})/i") + regexps = described_class.word_matcher_regexp_list(:block) + + expect(regexps).to be_an(Array) + expect(regexps.map(&:inspect)).to contain_exactly("/(#{word1})|(#{word2})/i", "/(#{word3})|(#{word4})/") end it "is correct when watched_words_regular_expressions = false" do SiteSetting.watched_words_regular_expressions = false - regexp = described_class.word_matcher_regexp(:block) - expect(regexp.inspect).to eq("/(?:\\W|^)(#{word1}|#{word2})(?=\\W|$)/i") + regexps = described_class.word_matcher_regexp_list(:block) + + expect(regexps).to be_an(Array) + expect(regexps.map(&:inspect)).to contain_exactly("/(?:\\W|^)(#{word1}|#{word2})(?=\\W|$)/i", "/(?:\\W|^)(#{word3}|#{word4})(?=\\W|$)/") + end + + it "is empty for an action without watched words" do + regexps = described_class.word_matcher_regexp_list(:censor) + + expect(regexps).to be_an(Array) + expect(regexps).to be_empty + end + end + + context "when regular expression is invalid" do + before do + SiteSetting.watched_words_regular_expressions = true + Fabricate(:watched_word, word: "Test[\S*", action: WatchedWord.actions[:block]) + end + + it "does not raise an exception by default" do + expect { described_class.word_matcher_regexp_list(:block) }.not_to raise_error + end + + it "raises an exception with raise_errors set to true" do + expect { described_class.word_matcher_regexp_list(:block, raise_errors: true) }.to raise_error(RegexpError) end end end @@ -187,6 +252,41 @@ RSpec.describe WordWatcher do end end + context "when case sensitive words are present" do + before do + Fabricate( + :watched_word, + word: "Discourse", + action: WatchedWord.actions[:block], + case_sensitive: true + ) + end + + context "when watched_words_regular_expressions = true" do + it "respects case sensitivity flag in matching words" do + SiteSetting.watched_words_regular_expressions = true + Fabricate(:watched_word, word: "p(rivate|ublic)", action: WatchedWord.actions[:block]) + + matches = described_class + .new("PUBLIC: Discourse is great for public discourse") + .word_matches_for_action?(:block, all_matches: true) + expect(matches).to contain_exactly("PUBLIC", "Discourse", "public") + end + end + + context "when watched_words_regular_expressions = false" do + it "repects case sensitivity flag in matching" do + SiteSetting.watched_words_regular_expressions = false + Fabricate(:watched_word, word: "private", action: WatchedWord.actions[:block]) + + matches = described_class + .new("PRIVATE: Discourse is also great private discourse") + .word_matches_for_action?(:block, all_matches: true) + + expect(matches).to contain_exactly("PRIVATE", "Discourse", "private") + end + end + end end end @@ -200,5 +300,31 @@ RSpec.describe WordWatcher do expected = "hello #{described_class::REPLACEMENT_LETTER * 8} world replaced https://discourse.org" expect(described_class.apply_to_text(text)).to eq(expected) end + + context "when watched_words_regular_expressions = true" do + it "replaces captured non-word prefix" do + SiteSetting.watched_words_regular_expressions = true + Fabricate( + :watched_word, + word: "\\Wplaceholder", + replacement: "replacement", + action: WatchedWord.actions[:replace] + ) + + text = "is \tplaceholder in https://notdiscourse.org" + expected = "is replacement in https://discourse.org" + expect(described_class.apply_to_text(text)).to eq(expected) + end + end + + context "when watched_words_regular_expressions = false" do + it "maintains non-word character prefix" do + SiteSetting.watched_words_regular_expressions = false + + text = "to replace and\thttps://notdiscourse.org" + expected = "replaced and\thttps://discourse.org" + expect(described_class.apply_to_text(text)).to eq(expected) + end + end end end