From 6ed8a18131e10db90b9decd44efa95fa24bec94e Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Mon, 4 Nov 2024 18:36:10 +0100 Subject: [PATCH] NIFI-13955 - Filter out .directories in Git Registry clients (#9477) * NIFI-13955 - Filter out .directories in Git Registry clients * Added a filter property and added a unit test --- .../flow/git/AbstractGitFlowRegistryClient.java | 14 ++++++++++++++ .../nifi/github/GitHubFlowRegistryClientTest.java | 13 ++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java b/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java index ac36714786..ad2c25cc61 100644 --- a/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java +++ b/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java @@ -60,6 +60,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -83,6 +84,15 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry .required(false) .build(); + public static final PropertyDescriptor DIRECTORY_FILTER_EXCLUDE = new PropertyDescriptor.Builder() + .name("Directory Filter Exclusion") + .description("Directories whose names match the given regular expression will be ignored " + + "when listing buckets.") + .defaultValue("[.].*") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .build(); + static final String DEFAULT_BUCKET_NAME = "default"; static final String DEFAULT_BUCKET_KEEP_FILE_PATH = DEFAULT_BUCKET_NAME + "/.keep"; static final String DEFAULT_BUCKET_KEEP_FILE_CONTENT = "Do Not Delete"; @@ -98,6 +108,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry private volatile FlowSnapshotSerializer flowSnapshotSerializer; private volatile GitRepositoryClient repositoryClient; + private volatile Pattern directoryExclusionPattern; private final AtomicBoolean clientInitialized = new AtomicBoolean(false); private volatile List propertyDescriptors; @@ -109,6 +120,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry final List combinedPropertyDescriptors = new ArrayList<>(createPropertyDescriptors()); combinedPropertyDescriptors.add(REPOSITORY_BRANCH); combinedPropertyDescriptors.add(REPOSITORY_PATH); + combinedPropertyDescriptors.add(DIRECTORY_FILTER_EXCLUDE); propertyDescriptors = Collections.unmodifiableList(combinedPropertyDescriptors); flowSnapshotSerializer = createFlowSnapshotSerializer(); @@ -174,6 +186,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry verifyReadPermissions(repositoryClient); final Set buckets = repositoryClient.getTopLevelDirectoryNames(branch).stream() + .filter(bucketName -> !directoryExclusionPattern.matcher(bucketName).matches()) .map(bucketName -> createFlowRegistryBucket(repositoryClient, bucketName)) .collect(Collectors.toSet()); @@ -568,6 +581,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry repositoryClient = createRepositoryClient(context); clientInitialized.set(true); initializeDefaultBucket(context); + directoryExclusionPattern = Pattern.compile(context.getProperty(DIRECTORY_FILTER_EXCLUDE).getValue()); } return repositoryClient; } diff --git a/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java b/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java index 32a624ff08..79958c9458 100644 --- a/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java +++ b/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java @@ -52,6 +52,7 @@ public class GitHubFlowRegistryClientTest { static final String DEFAULT_REPO_PATH = "some-path"; static final String DEFAULT_REPO_BRANCH = "some-branch"; + static final String DEFAULT_FILTER = "[.].*"; private GitHubRepositoryClient repositoryClient; private FlowSnapshotSerializer flowSnapshotSerializer; @@ -74,7 +75,14 @@ public class GitHubFlowRegistryClientTest { when(repositoryClient.hasReadPermission()).thenReturn(true); when(repositoryClient.hasWritePermission()).thenReturn(true); - when(repositoryClient.getTopLevelDirectoryNames(anyString())).thenReturn(Set.of("existing-bucket")); + when(repositoryClient.getTopLevelDirectoryNames(anyString())).thenReturn(Set.of("existing-bucket", ".github")); + } + + @Test + public void testDirExclusion() throws IOException, FlowRegistryException { + setupClientConfigurationContextWithDefaults(); + final Set buckets = flowRegistryClient.getBuckets(clientConfigurationContext, DEFAULT_REPO_BRANCH); + assertEquals(buckets.stream().filter(b -> b.getName().equals(".github")).count(), 0); } @Test @@ -169,6 +177,9 @@ public class GitHubFlowRegistryClientTest { final PropertyValue branchPropertyValue = createMockPropertyValue(branch); when(clientConfigurationContext.getProperty(GitHubFlowRegistryClient.REPOSITORY_BRANCH)).thenReturn(branchPropertyValue); + + final PropertyValue filterPropertyValue = createMockPropertyValue(DEFAULT_FILTER); + when(clientConfigurationContext.getProperty(GitHubFlowRegistryClient.DIRECTORY_FILTER_EXCLUDE)).thenReturn(filterPropertyValue); } private void setupClientConfigurationContextWithDefaults() {