Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-13581 Remove Parameter Contexts and Providers in destroyFlow of NiFiSystemIT (#9110) 

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Bryan Bende 2024-07-24 19:51:49 -04:00 committed by GitHub
parent 2dd7cf7f92
commit 94f8688545
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 66 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/system-tests.yml vendored
View File

@ -26,7 +26,7 @@ on:
- 'nifi-system-tests/**'
- 'nifi-api/**'
- 'nifi-framework-api/**'
- 'nifi-nifi-framework-bundle/**'
- 'nifi-framework-bundle/**'
- 'nifi-extension-bundles/nifi-py4j-bundle/**'
- 'nifi-stateless/**'
pull_request:
@ -36,7 +36,7 @@ on:
- 'nifi-system-tests/**'
- 'nifi-api/**'
- 'nifi-framework-api/**'
- 'nifi-nifi-framework-bundle/**'
- 'nifi-framework-bundle/**'
- 'nifi-extension-bundles/nifi-py4j-bundle/**'
- 'nifi-stateless/**'

36
nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java
View File

@ -89,6 +89,7 @@ import org.apache.nifi.web.api.entity.NodeEntity;
import org.apache.nifi.web.api.entity.ParameterContextEntity;
import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity;
import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity;
import org.apache.nifi.web.api.entity.ParameterContextsEntity;
import org.apache.nifi.web.api.entity.ParameterEntity;
import org.apache.nifi.web.api.entity.ParameterGroupConfigurationEntity;
import org.apache.nifi.web.api.entity.ParameterProviderApplyParametersRequestEntity;
@ -96,6 +97,7 @@ import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity;
import org.apache.nifi.web.api.entity.ParameterProviderEntity;
import org.apache.nifi.web.api.entity.ParameterProviderParameterApplicationEntity;
import org.apache.nifi.web.api.entity.ParameterProviderParameterFetchEntity;
import org.apache.nifi.web.api.entity.ParameterProvidersEntity;
import org.apache.nifi.web.api.entity.PortEntity;
import org.apache.nifi.web.api.entity.ProcessGroupEntity;
import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity;
@ -126,11 +128,13 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
public class NiFiClientUtil {
@ -472,6 +476,38 @@ public class NiFiClientUtil {
}
}
public void deleteParameterContexts() throws NiFiClientException, IOException {
final ParameterContextsEntity parameterContextsEntity = nifiClient.getParamContextClient().getParamContexts();
final Map<String, ParameterContextEntity> parameterContextMap = parameterContextsEntity.getParameterContexts().stream()
.collect(Collectors.toMap(ParameterContextEntity::getId, Function.identity()));
// If parameter context have inherited contexts then they needed to be deleted from the bottom up, so we just keep iterating
// over the set of ids and retrying deletes until all have been deleted, knowing that some delete calls will fail the first time
final Set<String> parameterContextIds = new HashSet<>(parameterContextMap.keySet());
while (!parameterContextIds.isEmpty()) {
final Iterator<String> parameterContextIdIterator = parameterContextIds.iterator();
while (parameterContextIdIterator.hasNext()) {
final String parameterContextId = parameterContextIdIterator.next();
final ParameterContextEntity parameterContextEntity = parameterContextMap.get(parameterContextId);
try {
final String version = String.valueOf(parameterContextEntity.getRevision().getVersion());
nifiClient.getParamContextClient().deleteParamContext(parameterContextId, version, true);
parameterContextIdIterator.remove();
} catch (final Exception e) {
logger.warn("Failed to delete parameter context [{}] due to: {}", parameterContextId, e.getMessage());
}
}
}
}
public void deleteParameterProviders() throws NiFiClientException, IOException {
final ParameterProvidersEntity parameterProvidersEntity = nifiClient.getFlowClient().getParamProviders();
for (final ParameterProviderEntity parameterProviderEntity : parameterProvidersEntity.getParameterProviders()) {
final String version = String.valueOf(parameterProviderEntity.getRevision().getVersion());
nifiClient.getParamProviderClient().deleteParamProvider(parameterProviderEntity.getId(), version, true);
}
}
public void waitForFlowAnalysisRuleState(final String desiredState, final Collection<String> ruleIdsOfInterest) throws NiFiClientException, IOException {
final long maxTimestamp = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2L);

2
nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java
View File

@ -249,6 +249,8 @@ public abstract class NiFiSystemIT implements NiFiInstanceProvider {
getClientUtil().deleteControllerLevelServices();
getClientUtil().deleteReportingTasks();
getClientUtil().deleteFlowAnalysisRules();
getClientUtil().deleteParameterContexts();
getClientUtil().deleteParameterProviders();
logger.info("Finished destroyFlow");
}

2
nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java
View File

@ -32,6 +32,8 @@ public interface ParamContextClient {
ParameterContextEntity deleteParamContext(String id, String version) throws NiFiClientException, IOException;
ParameterContextEntity deleteParamContext(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException;
ParameterContextUpdateRequestEntity updateParamContext(ParameterContextEntity paramContext) throws NiFiClientException, IOException;
ParameterContextUpdateRequestEntity getParamContextUpdateRequest(String contextId, String updateRequestId) throws NiFiClientException, IOException;

2
nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java
View File

@ -32,6 +32,8 @@ public interface ParamProviderClient {
ParameterProviderEntity deleteParamProvider(String id, String version) throws NiFiClientException, IOException;
ParameterProviderEntity deleteParamProvider(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException;
ParameterProviderEntity fetchParameters(ParameterProviderParameterFetchEntity parameterFetchEntity) throws NiFiClientException, IOException;
ParameterProviderApplyParametersRequestEntity applyParameters(ParameterProviderParameterApplicationEntity parameterApplicationEntity) throws NiFiClientException, IOException;

12
nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java
View File

@ -83,6 +83,11 @@ public class JerseyParamContextClient extends AbstractJerseyClient implements Pa
@Override
public ParameterContextEntity deleteParamContext(final String id, final String version) throws NiFiClientException, IOException {
return deleteParamContext(id, version, false);
}
@Override
public ParameterContextEntity deleteParamContext(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException {
if (StringUtils.isBlank(id)) {
throw new IllegalArgumentException("Parameter context id cannot be null or blank");
}
@ -92,9 +97,14 @@ public class JerseyParamContextClient extends AbstractJerseyClient implements Pa
}
return executeAction("Error deleting parameter context", () -> {
final WebTarget target = paramContextTarget.path("{id}")
WebTarget target = paramContextTarget.path("{id}")
.resolveTemplate("id", id)
.queryParam("version", version);
if (disconnectedNodeAcknowledged) {
target = target.queryParam("disconnectedNodeAcknowledged", "true");
}
return getRequestBuilder(target).delete(ParameterContextEntity.class);
});
}

12
nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java
View File

@ -75,6 +75,11 @@ public class JerseyParamProviderClient extends AbstractJerseyClient implements P
@Override
public ParameterProviderEntity deleteParamProvider(final String id, final String version) throws NiFiClientException, IOException {
return deleteParamProvider(id, version, false);
}
@Override
public ParameterProviderEntity deleteParamProvider(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException {
if (StringUtils.isBlank(id)) {
throw new IllegalArgumentException("Parameter provider id cannot be null or blank");
}
@ -84,9 +89,14 @@ public class JerseyParamProviderClient extends AbstractJerseyClient implements P
}
return executeAction("Error deleting parameter provider", () -> {
final WebTarget target = paramProviderTarget.path("{id}")
WebTarget target = paramProviderTarget.path("{id}")
.resolveTemplate("id", id)
.queryParam("version", version);
if (disconnectedNodeAcknowledged) {
target = target.queryParam("disconnectedNodeAcknowledged", "true");
}
return getRequestBuilder(target).delete(ParameterProviderEntity.class);
});
}