Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-13955 - Filter out .directories in Git Registry clients (#9477) 

* NIFI-13955 - Filter out .directories in Git Registry clients
* Added a filter property and added a unit test
This commit is contained in:
Pierre Villard 2024-11-04 18:36:10 +01:00 committed by GitHub
parent d428970efb
commit 6ed8a18131
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 26 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java
View File

@ -60,6 +60,7 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
@ -83,6 +84,15 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry
.required(false)
.build();
public static final PropertyDescriptor DIRECTORY_FILTER_EXCLUDE = new PropertyDescriptor.Builder()
.name("Directory Filter Exclusion")
.description("Directories whose names match the given regular expression will be ignored "
+ "when listing buckets.")
.defaultValue("[.].*")
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
.required(true)
.build();
static final String DEFAULT_BUCKET_NAME = "default";
static final String DEFAULT_BUCKET_KEEP_FILE_PATH = DEFAULT_BUCKET_NAME + "/.keep";
static final String DEFAULT_BUCKET_KEEP_FILE_CONTENT = "Do Not Delete";
@ -98,6 +108,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry
private volatile FlowSnapshotSerializer flowSnapshotSerializer;
private volatile GitRepositoryClient repositoryClient;
private volatile Pattern directoryExclusionPattern;
private final AtomicBoolean clientInitialized = new AtomicBoolean(false);
private volatile List<PropertyDescriptor> propertyDescriptors;
@ -109,6 +120,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry
final List<PropertyDescriptor> combinedPropertyDescriptors = new ArrayList<>(createPropertyDescriptors());
combinedPropertyDescriptors.add(REPOSITORY_BRANCH);
combinedPropertyDescriptors.add(REPOSITORY_PATH);
combinedPropertyDescriptors.add(DIRECTORY_FILTER_EXCLUDE);
propertyDescriptors = Collections.unmodifiableList(combinedPropertyDescriptors);
flowSnapshotSerializer = createFlowSnapshotSerializer();
@ -174,6 +186,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry
verifyReadPermissions(repositoryClient);
final Set<FlowRegistryBucket> buckets = repositoryClient.getTopLevelDirectoryNames(branch).stream()
.filter(bucketName -> !directoryExclusionPattern.matcher(bucketName).matches())
.map(bucketName -> createFlowRegistryBucket(repositoryClient, bucketName))
.collect(Collectors.toSet());
@ -568,6 +581,7 @@ public abstract class AbstractGitFlowRegistryClient extends AbstractFlowRegistry
repositoryClient = createRepositoryClient(context);
clientInitialized.set(true);
initializeDefaultBucket(context);
directoryExclusionPattern = Pattern.compile(context.getProperty(DIRECTORY_FILTER_EXCLUDE).getValue());
}
return repositoryClient;
}

13
nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java
View File

@ -52,6 +52,7 @@ public class GitHubFlowRegistryClientTest {
static final String DEFAULT_REPO_PATH = "some-path";
static final String DEFAULT_REPO_BRANCH = "some-branch";
static final String DEFAULT_FILTER = "[.].*";
private GitHubRepositoryClient repositoryClient;
private FlowSnapshotSerializer flowSnapshotSerializer;
@ -74,7 +75,14 @@ public class GitHubFlowRegistryClientTest {
when(repositoryClient.hasReadPermission()).thenReturn(true);
when(repositoryClient.hasWritePermission()).thenReturn(true);
when(repositoryClient.getTopLevelDirectoryNames(anyString())).thenReturn(Set.of("existing-bucket"));
when(repositoryClient.getTopLevelDirectoryNames(anyString())).thenReturn(Set.of("existing-bucket", ".github"));
}
@Test
public void testDirExclusion() throws IOException, FlowRegistryException {
setupClientConfigurationContextWithDefaults();
final Set<FlowRegistryBucket> buckets = flowRegistryClient.getBuckets(clientConfigurationContext, DEFAULT_REPO_BRANCH);
assertEquals(buckets.stream().filter(b -> b.getName().equals(".github")).count(), 0);
}
@Test
@ -169,6 +177,9 @@ public class GitHubFlowRegistryClientTest {
final PropertyValue branchPropertyValue = createMockPropertyValue(branch);
when(clientConfigurationContext.getProperty(GitHubFlowRegistryClient.REPOSITORY_BRANCH)).thenReturn(branchPropertyValue);
final PropertyValue filterPropertyValue = createMockPropertyValue(DEFAULT_FILTER);
when(clientConfigurationContext.getProperty(GitHubFlowRegistryClient.DIRECTORY_FILTER_EXCLUDE)).thenReturn(filterPropertyValue);
}
private void setupClientConfigurationContextWithDefaults() {