From 1823a52e3610fd207957365dfc41ddcfa9d777f1 Mon Sep 17 00:00:00 2001 From: Mark Bathori <57758037+mark-bathori@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:00:36 +0200 Subject: [PATCH] NIFI-13884 Removed File IO Implementation Property from PutIceberg (#9403) - Removed pending further testing and evaluation of runtime behavior Signed-off-by: David Handermann --- .../nifi-iceberg-common/pom.xml | 15 ---- .../catalog/IcebergCatalogFactory.java | 3 +- .../iceberg/IcebergCatalogProperty.java | 3 +- .../iceberg/FileIOImplementation.java | 68 ------------------- .../services/iceberg/JdbcCatalogService.java | 14 +--- 5 files changed, 3 insertions(+), 100 deletions(-) delete mode 100644 nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java diff --git a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml index 7ef3aa0065..232068391b 100644 --- a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml +++ b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml @@ -89,21 +89,6 @@ iceberg-orc ${iceberg.version} - - org.apache.iceberg - iceberg-aws - ${iceberg.version} - - - org.apache.iceberg - iceberg-azure - ${iceberg.version} - - - org.apache.iceberg - iceberg-gcp - ${iceberg.version} - org.apache.hadoop hadoop-client diff --git a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java index 19da644afb..eac1981735 100644 --- a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java +++ b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java @@ -36,7 +36,6 @@ import java.util.function.Function; import static org.apache.nifi.processors.iceberg.IcebergUtils.getConfigurationFromFiles; import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CATALOG_NAME; -import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.FILE_IO_IMPLEMENTATION; import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE; import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.METASTORE_URI; import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION; @@ -101,7 +100,7 @@ public class IcebergCatalogFactory { final DBCPService dbcpService = (DBCPService) catalogProperties.get(CLIENT_POOL_SERVICE); final Function, JdbcClientPool> clientPoolBuilder = props -> new IcebergJdbcClientPool(props, dbcpService); - final Function, FileIO> ioBuilder = props -> CatalogUtil.loadFileIO((String) catalogProperties.get(FILE_IO_IMPLEMENTATION), props, configuration); + final Function, FileIO> ioBuilder = props -> CatalogUtil.loadFileIO("org.apache.iceberg.hadoop.HadoopFileIO", props, configuration); JdbcCatalog catalog = new JdbcCatalog(ioBuilder, clientPoolBuilder, false); catalog.setConf(configuration); diff --git a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java index 1e4987a0a5..0874f731ad 100644 --- a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java +++ b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java @@ -22,8 +22,7 @@ public enum IcebergCatalogProperty { CATALOG_NAME, METASTORE_URI("hive.metastore.uris"), WAREHOUSE_LOCATION("hive.metastore.warehouse.dir"), - CLIENT_POOL_SERVICE, - FILE_IO_IMPLEMENTATION; + CLIENT_POOL_SERVICE; private static final String EMPTY_STRING = ""; diff --git a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java deleted file mode 100644 index 45c3d3cfaa..0000000000 --- a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.services.iceberg; - -import org.apache.nifi.components.DescribedValue; - -import java.util.HashMap; -import java.util.Map; - -public enum FileIOImplementation implements DescribedValue { - HADOOP( "org.apache.iceberg.hadoop.HadoopFileIO", "Hadoop File IO"), - RESOLVING("org.apache.iceberg.io.ResolvingFileIO", "Resolving File IO"), - S3( "org.apache.iceberg.aws.s3.S3FileIO", "S3 File IO"), - GCS( "org.apache.iceberg.gcp.gcs.GCSFileIO", "GCS File IO"), - ADLS( "org.apache.iceberg.azure.adlsv2.ADLSFileIO", "ADLS File IO"); - - private static final Map ENUM_MAP = new HashMap<>(); - - static { - for (FileIOImplementation strategy : FileIOImplementation.values()) { - ENUM_MAP.put(strategy.getValue(), strategy); - } - } - - private final String value; - private final String displayName; - private final String description; - - FileIOImplementation(String value, String displayName) { - this(value, displayName, null); - } - - FileIOImplementation(String value, String displayName, String description) { - this.value = value; - this.displayName = displayName; - this.description = description; - } - - @Override - public String getValue() { - return value; - } - - @Override - public String getDisplayName() { - return displayName; - } - - @Override - public String getDescription() { - return description; - } -} diff --git a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java index 9a596aa06f..7f8be77e06 100644 --- a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java +++ b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java @@ -28,8 +28,6 @@ import org.apache.nifi.processor.util.StandardValidators; import java.util.List; -import static org.apache.nifi.services.iceberg.FileIOImplementation.HADOOP; -import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.FILE_IO_IMPLEMENTATION; import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE; import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION; @@ -53,17 +51,8 @@ public class JdbcCatalogService extends AbstractCatalogService { .required(true) .build(); - public static final PropertyDescriptor FILE_IO_IMPL = new PropertyDescriptor.Builder() - .name("File IO Implementation") - .description("Specifies the implementation of FileIO interface to be used. " + - "The provided implementation have to include the class and full package name.") - .required(true) - .defaultValue(HADOOP.getValue()) - .allowableValues(FileIOImplementation.class) - .build(); - private static final List PROPERTIES = List.of( - CATALOG_NAME, CONNECTION_POOL, FILE_IO_IMPL, WAREHOUSE_PATH, HADOOP_CONFIGURATION_RESOURCES); + CATALOG_NAME, CONNECTION_POOL, WAREHOUSE_PATH, HADOOP_CONFIGURATION_RESOURCES); @Override protected List getSupportedPropertyDescriptors() { @@ -78,7 +67,6 @@ public class JdbcCatalogService extends AbstractCatalogService { catalogProperties.put(IcebergCatalogProperty.CATALOG_NAME, context.getProperty(CATALOG_NAME).evaluateAttributeExpressions().getValue()); catalogProperties.put(CLIENT_POOL_SERVICE, context.getProperty(CONNECTION_POOL).asControllerService(DBCPService.class)); - catalogProperties.put(FILE_IO_IMPLEMENTATION, context.getProperty(FILE_IO_IMPL).evaluateAttributeExpressions().getValue()); catalogProperties.put(WAREHOUSE_LOCATION, context.getProperty(WAREHOUSE_PATH).evaluateAttributeExpressions().getValue()); }