mirror of https://github.com/apache/nifi.git
NIFI-3853:
- Filtering out certain control characters and unpaired Unicode surrogate codepoints prior to saving the flow.xml. This closes #1784. Signed-off-by: Andy LoPresto <alopresto@apache.org>
This commit is contained in:
Matt Gilman
Andy LoPresto
parent
bf15502e19
commit
6ffb78d404
|
|
@ -478,6 +478,10 @@ option but rather has a `View Configuration` option. Processor configuration can
|
|||
running. You must first stop the Processor and wait for all of its active tasks to complete before configuring
|
||||
the Processor again.
|
||||
|
||||
Note that entering certain control characters are not supported and will be automatically filtered out when entered. The following characters and any
|
||||
unpaired Unicode surrogate codepoints will not be retained in any configuration:
|
||||
|
||||
[#x0], [#x1], [#x2], [#x3], [#x4], [#x5], [#x6], [#x7], [#x8], [#xB], [#xC], [#xE], [#xF], [#x10], [#x11], [#x12], [#x13], [#x14], [#x15], [#x16], [#x17], [#x18], [#x19], [#x1A], [#x1B], [#x1C], [#x1D], [#x1E], [#x1F], [#xFFFE], [#xFFFF]
|
||||
|
||||
==== Settings Tab
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.apache.nifi.controller.service.ControllerServiceProvider;
|
|||
import org.apache.nifi.nar.ExtensionManager;
|
||||
import org.apache.nifi.nar.NarCloseable;
|
||||
import org.apache.nifi.registry.VariableRegistry;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
import org.apache.nifi.util.file.classloader.ClassLoaderUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -105,7 +106,7 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
|
|||
|
||||
@Override
|
||||
public void setName(final String name) {
|
||||
this.name.set(Objects.requireNonNull(name).intern());
|
||||
this.name.set(CharacterFilterUtils.filterInvalidXmlCharacters(Objects.requireNonNull(name).intern()));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -115,7 +116,7 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
|
|||
|
||||
@Override
|
||||
public void setAnnotationData(final String data) {
|
||||
annotationData.set(data);
|
||||
annotationData.set(CharacterFilterUtils.filterInvalidXmlCharacters(data));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -140,7 +141,7 @@ public abstract class AbstractConfiguredComponent implements ConfigurableCompone
|
|||
if (entry.getKey() != null && entry.getValue() == null) {
|
||||
removeProperty(entry.getKey());
|
||||
} else if (entry.getKey() != null) {
|
||||
setProperty(entry.getKey(), entry.getValue());
|
||||
setProperty(entry.getKey(), CharacterFilterUtils.filterInvalidXmlCharacters(entry.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,22 +16,6 @@
|
|||
*/
|
||||
package org.apache.nifi.controller;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringStyle;
|
||||
import org.apache.nifi.authorization.Resource;
|
||||
|
|
@ -50,8 +34,25 @@ import org.apache.nifi.processor.ProcessSession;
|
|||
import org.apache.nifi.processor.ProcessSessionFactory;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
import org.apache.nifi.util.FormatUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public abstract class AbstractPort implements Port {
|
||||
|
||||
public static final Relationship PORT_RELATIONSHIP = new Relationship.Builder()
|
||||
|
|
@ -179,7 +180,7 @@ public abstract class AbstractPort implements Port {
|
|||
|
||||
@Override
|
||||
public void setComments(final String comments) {
|
||||
this.comments.set(comments);
|
||||
this.comments.set(CharacterFilterUtils.filterInvalidXmlCharacters(comments));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.util;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.text.translate.AggregateTranslator;
|
||||
import org.apache.commons.lang3.text.translate.CharSequenceTranslator;
|
||||
import org.apache.commons.lang3.text.translate.LookupTranslator;
|
||||
import org.apache.commons.lang3.text.translate.UnicodeUnpairedSurrogateRemover;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class CharacterFilterUtils {
|
||||
|
||||
private static final List<String> INVALID_XML_CHARACTERS = Arrays.asList(
|
||||
"\u0000", "\u0001", "\u0002", "\u0003", "\u0004", "\u0005", "\u0006", "\u0007", "\u0008", "\u000b",
|
||||
"\u000c", "\u000e", "\u000f", "\u0010", "\u0011", "\u0012", "\u0013", "\u0014", "\u0015", "\u0016",
|
||||
"\u0017", "\u0018", "\u0019", "\u001a", "\u001b", "\u001c", "\u001d", "\u001e", "\u001f", "\ufffe",
|
||||
"\uffff");
|
||||
|
||||
private static final String[][] INVALID_XML_CHARACTER_MAPPING = INVALID_XML_CHARACTERS.stream()
|
||||
.map(invalidCharacter -> new String[] { invalidCharacter, StringUtils.EMPTY })
|
||||
.toArray(String[][]::new);
|
||||
|
||||
private static final CharSequenceTranslator INVALID_XML_CHARACTER_FILTER = new AggregateTranslator(
|
||||
new LookupTranslator(INVALID_XML_CHARACTER_MAPPING),
|
||||
new UnicodeUnpairedSurrogateRemover());
|
||||
|
||||
public static String filterInvalidXmlCharacters(final String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return INVALID_XML_CHARACTER_FILTER.translate(value);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.util;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CharacterFilterUtilsTest {
|
||||
|
||||
@Test
|
||||
public void filterInvalidCharacters() throws Exception {
|
||||
final String text = "This is an example with characters that need to be filtered \u0002 in it. " + Character.MIN_SURROGATE;
|
||||
final String filtered = CharacterFilterUtils.filterInvalidXmlCharacters(text);
|
||||
|
||||
final String expected = "This is an example with characters that need to be filtered in it. ";
|
||||
Assert.assertEquals(expected, filtered);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -59,6 +59,7 @@ import org.apache.nifi.processor.SimpleProcessLogger;
|
|||
import org.apache.nifi.registry.VariableRegistry;
|
||||
import org.apache.nifi.scheduling.ExecutionNode;
|
||||
import org.apache.nifi.scheduling.SchedulingStrategy;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
import org.apache.nifi.util.FormatUtils;
|
||||
import org.apache.nifi.util.NiFiProperties;
|
||||
import org.apache.nifi.util.ReflectionUtils;
|
||||
|
|
@ -258,7 +259,7 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
|
|||
if (isRunning()) {
|
||||
throw new IllegalStateException("Cannot modify Processor configuration while the Processor is running");
|
||||
}
|
||||
this.comments.set(comments);
|
||||
this.comments.set(CharacterFilterUtils.filterInvalidXmlCharacters(comments));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import org.apache.nifi.authorization.resource.ResourceType;
|
|||
import org.apache.nifi.connectable.Position;
|
||||
import org.apache.nifi.connectable.Size;
|
||||
import org.apache.nifi.groups.ProcessGroup;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -47,7 +48,7 @@ public class StandardLabel implements Label {
|
|||
this.position = new AtomicReference<>(position);
|
||||
this.style = new AtomicReference<>(Collections.unmodifiableMap(new HashMap<>(style)));
|
||||
this.size = new AtomicReference<>(new Size(150, 150));
|
||||
this.value = new AtomicReference<>(value);
|
||||
this.value = new AtomicReference<>(CharacterFilterUtils.filterInvalidXmlCharacters(value));
|
||||
this.processGroup = new AtomicReference<>(processGroup);
|
||||
}
|
||||
|
||||
|
|
@ -116,7 +117,7 @@ public class StandardLabel implements Label {
|
|||
}
|
||||
|
||||
public void setValue(final String value) {
|
||||
this.value.set(value);
|
||||
this.value.set(CharacterFilterUtils.filterInvalidXmlCharacters(value));
|
||||
}
|
||||
|
||||
public void setProcessGroup(final ProcessGroup group) {
|
||||
|
|
|
|||
|
|
@ -21,11 +21,11 @@ import org.apache.nifi.bundle.BundleCoordinate;
|
|||
import org.apache.nifi.components.ConfigurableComponent;
|
||||
import org.apache.nifi.components.ValidationResult;
|
||||
import org.apache.nifi.controller.AbstractConfiguredComponent;
|
||||
import org.apache.nifi.controller.ReloadComponent;
|
||||
import org.apache.nifi.controller.ConfigurationContext;
|
||||
import org.apache.nifi.controller.ControllerServiceLookup;
|
||||
import org.apache.nifi.controller.LoggableComponent;
|
||||
import org.apache.nifi.controller.ProcessScheduler;
|
||||
import org.apache.nifi.controller.ReloadComponent;
|
||||
import org.apache.nifi.controller.ReportingTaskNode;
|
||||
import org.apache.nifi.controller.ScheduledState;
|
||||
import org.apache.nifi.controller.ValidationContextFactory;
|
||||
|
|
@ -36,7 +36,11 @@ import org.apache.nifi.logging.ComponentLog;
|
|||
import org.apache.nifi.registry.VariableRegistry;
|
||||
import org.apache.nifi.reporting.ReportingTask;
|
||||
import org.apache.nifi.scheduling.SchedulingStrategy;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
import org.apache.nifi.util.FormatUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.core.annotation.AnnotationUtils;
|
||||
|
||||
import java.net.URL;
|
||||
import java.util.Collection;
|
||||
|
|
@ -46,10 +50,6 @@ import java.util.Set;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.core.annotation.AnnotationUtils;
|
||||
|
||||
public abstract class AbstractReportingTaskNode extends AbstractConfiguredComponent implements ReportingTaskNode {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AbstractReportingTaskNode.class);
|
||||
|
|
@ -208,7 +208,7 @@ public abstract class AbstractReportingTaskNode extends AbstractConfiguredCompon
|
|||
|
||||
@Override
|
||||
public void setComments(final String comment) {
|
||||
this.comment = comment;
|
||||
this.comment = CharacterFilterUtils.filterInvalidXmlCharacters(comment);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import org.apache.nifi.persistence.TemplateSerializer;
|
|||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.remote.RemoteGroupPort;
|
||||
import org.apache.nifi.remote.RootGroupPort;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
import org.apache.nifi.util.StringUtils;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
|
|
@ -524,7 +525,7 @@ public class StandardFlowSerializer implements FlowSerializer {
|
|||
private static void addTextElement(final Element element, final String name, final String value) {
|
||||
final Document doc = element.getOwnerDocument();
|
||||
final Element toAdd = doc.createElement(name);
|
||||
toAdd.setTextContent(value);
|
||||
toAdd.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value)); // value should already be filtered, but just in case ensure there are no invalid xml characters
|
||||
element.appendChild(toAdd);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import org.apache.nifi.logging.ComponentLog;
|
|||
import org.apache.nifi.nar.NarCloseable;
|
||||
import org.apache.nifi.processor.SimpleProcessLogger;
|
||||
import org.apache.nifi.registry.VariableRegistry;
|
||||
import org.apache.nifi.util.CharacterFilterUtils;
|
||||
import org.apache.nifi.util.ReflectionUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -353,7 +354,7 @@ public class StandardControllerServiceNode extends AbstractConfiguredComponent i
|
|||
public void setComments(final String comment) {
|
||||
writeLock.lock();
|
||||
try {
|
||||
this.comment = comment;
|
||||
this.comment = CharacterFilterUtils.filterInvalidXmlCharacters(comment);
|
||||
} finally {
|
||||
writeLock.unlock();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.controller.serialization;
|
||||
|
||||
import org.apache.nifi.admin.service.AuditService;
|
||||
import org.apache.nifi.authorization.AbstractPolicyBasedAuthorizer;
|
||||
import org.apache.nifi.authorization.MockPolicyBasedAuthorizer;
|
||||
import org.apache.nifi.bundle.Bundle;
|
||||
import org.apache.nifi.controller.DummyScheduledProcessor;
|
||||
import org.apache.nifi.controller.FlowController;
|
||||
import org.apache.nifi.controller.ProcessorNode;
|
||||
import org.apache.nifi.controller.TestFlowController;
|
||||
import org.apache.nifi.controller.repository.FlowFileEventRepository;
|
||||
import org.apache.nifi.encrypt.StringEncryptor;
|
||||
import org.apache.nifi.nar.ExtensionManager;
|
||||
import org.apache.nifi.nar.SystemBundle;
|
||||
import org.apache.nifi.provenance.MockProvenanceRepository;
|
||||
import org.apache.nifi.registry.VariableRegistry;
|
||||
import org.apache.nifi.reporting.BulletinRepository;
|
||||
import org.apache.nifi.util.FileBasedVariableRegistry;
|
||||
import org.apache.nifi.util.NiFiProperties;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class StandardFlowSerializerTest {
|
||||
|
||||
private static final String RAW_COMMENTS =
|
||||
"<tagName> \"This\" is an ' example with many characters that need to be filtered and escaped \u0002 in it. \u007f \u0086 " + Character.MIN_SURROGATE;
|
||||
private static final String SERIALIZED_COMMENTS =
|
||||
"<tagName> \"This\" is an ' example with many characters that need to be filtered and escaped in it.  † ";
|
||||
|
||||
private FlowController controller;
|
||||
private Bundle systemBundle;
|
||||
private StandardFlowSerializer serializer;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestFlowController.class.getResource("/nifi.properties").getFile());
|
||||
|
||||
final FlowFileEventRepository flowFileEventRepo = Mockito.mock(FlowFileEventRepository.class);
|
||||
final AuditService auditService = Mockito.mock(AuditService.class);
|
||||
final Map<String, String> otherProps = new HashMap<>();
|
||||
otherProps.put(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, MockProvenanceRepository.class.getName());
|
||||
otherProps.put("nifi.remote.input.socket.port", "");
|
||||
otherProps.put("nifi.remote.input.secure", "");
|
||||
final NiFiProperties nifiProperties = NiFiProperties.createBasicNiFiProperties(null, otherProps);
|
||||
final StringEncryptor encryptor = StringEncryptor.createEncryptor(nifiProperties);
|
||||
|
||||
// use the system bundle
|
||||
systemBundle = SystemBundle.create(nifiProperties);
|
||||
ExtensionManager.discoverExtensions(systemBundle, Collections.emptySet());
|
||||
|
||||
final AbstractPolicyBasedAuthorizer authorizer = new MockPolicyBasedAuthorizer();
|
||||
final VariableRegistry variableRegistry = new FileBasedVariableRegistry(nifiProperties.getVariableRegistryPropertiesPaths());
|
||||
|
||||
final BulletinRepository bulletinRepo = Mockito.mock(BulletinRepository.class);
|
||||
controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, auditService, encryptor, bulletinRepo, variableRegistry);
|
||||
|
||||
serializer = new StandardFlowSerializer(encryptor);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSerializationEscapingAndFiltering() throws Exception {
|
||||
final ProcessorNode dummy = controller.createProcessor(DummyScheduledProcessor.class.getName(), UUID.randomUUID().toString(), systemBundle.getBundleDetails().getCoordinate());
|
||||
dummy.setComments(RAW_COMMENTS);
|
||||
controller.getRootGroup().addProcessor(dummy);
|
||||
|
||||
// serialize the controller
|
||||
final ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
serializer.serialize(controller, os);
|
||||
|
||||
// verify the results contain the serialized string
|
||||
final String serializedFlow = os.toString(StandardCharsets.UTF_8.name());
|
||||
assertTrue(serializedFlow.contains(SERIALIZED_COMMENTS));
|
||||
assertFalse(serializedFlow.contains(RAW_COMMENTS));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue