Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-13884 Removed File IO Implementation Property from PutIceberg (#9403) 

- Removed pending further testing and evaluation of runtime behavior

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Mark Bathori 2024-10-17 16:00:36 +02:00 committed by GitHub
parent 94d0ee9460
commit 1823a52e36
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 3 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml
View File

@ -89,21 +89,6 @@
<artifactId>iceberg-orc</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-aws</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-azure</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-gcp</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>

3
nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java
View File

@ -36,7 +36,6 @@ import java.util.function.Function;
import static org.apache.nifi.processors.iceberg.IcebergUtils.getConfigurationFromFiles;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CATALOG_NAME;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.FILE_IO_IMPLEMENTATION;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.METASTORE_URI;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION;
@ -101,7 +100,7 @@ public class IcebergCatalogFactory {
final DBCPService dbcpService = (DBCPService) catalogProperties.get(CLIENT_POOL_SERVICE);
final Function<Map<String, String>, JdbcClientPool> clientPoolBuilder = props -> new IcebergJdbcClientPool(props, dbcpService);
final Function<Map<String, String>, FileIO> ioBuilder = props -> CatalogUtil.loadFileIO((String) catalogProperties.get(FILE_IO_IMPLEMENTATION), props, configuration);
final Function<Map<String, String>, FileIO> ioBuilder = props -> CatalogUtil.loadFileIO("org.apache.iceberg.hadoop.HadoopFileIO", props, configuration);
JdbcCatalog catalog = new JdbcCatalog(ioBuilder, clientPoolBuilder, false);
catalog.setConf(configuration);

3
nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java
View File

@ -22,8 +22,7 @@ public enum IcebergCatalogProperty {
CATALOG_NAME,
METASTORE_URI("hive.metastore.uris"),
WAREHOUSE_LOCATION("hive.metastore.warehouse.dir"),
CLIENT_POOL_SERVICE,
FILE_IO_IMPLEMENTATION;
CLIENT_POOL_SERVICE;
private static final String EMPTY_STRING = "";

68
nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java
View File

@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.services.iceberg;
import org.apache.nifi.components.DescribedValue;
import java.util.HashMap;
import java.util.Map;
public enum FileIOImplementation implements DescribedValue {
HADOOP( "org.apache.iceberg.hadoop.HadoopFileIO", "Hadoop File IO"),
RESOLVING("org.apache.iceberg.io.ResolvingFileIO", "Resolving File IO"),
S3( "org.apache.iceberg.aws.s3.S3FileIO", "S3 File IO"),
GCS( "org.apache.iceberg.gcp.gcs.GCSFileIO", "GCS File IO"),
ADLS( "org.apache.iceberg.azure.adlsv2.ADLSFileIO", "ADLS File IO");
private static final Map<String, FileIOImplementation> ENUM_MAP = new HashMap<>();
static {
for (FileIOImplementation strategy : FileIOImplementation.values()) {
ENUM_MAP.put(strategy.getValue(), strategy);
}
}
private final String value;
private final String displayName;
private final String description;
FileIOImplementation(String value, String displayName) {
this(value, displayName, null);
}
FileIOImplementation(String value, String displayName, String description) {
this.value = value;
this.displayName = displayName;
this.description = description;
}
@Override
public String getValue() {
return value;
}
@Override
public String getDisplayName() {
return displayName;
}
@Override
public String getDescription() {
return description;
}
}

14
nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java
View File

@ -28,8 +28,6 @@ import org.apache.nifi.processor.util.StandardValidators;
import java.util.List;
import static org.apache.nifi.services.iceberg.FileIOImplementation.HADOOP;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.FILE_IO_IMPLEMENTATION;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION;
@ -53,17 +51,8 @@ public class JdbcCatalogService extends AbstractCatalogService {
.required(true)
.build();
public static final PropertyDescriptor FILE_IO_IMPL = new PropertyDescriptor.Builder()
.name("File IO Implementation")
.description("Specifies the implementation of FileIO interface to be used. " +
"The provided implementation have to include the class and full package name.")
.required(true)
.defaultValue(HADOOP.getValue())
.allowableValues(FileIOImplementation.class)
.build();
private static final List<PropertyDescriptor> PROPERTIES = List.of(
CATALOG_NAME, CONNECTION_POOL, FILE_IO_IMPL, WAREHOUSE_PATH, HADOOP_CONFIGURATION_RESOURCES);
CATALOG_NAME, CONNECTION_POOL, WAREHOUSE_PATH, HADOOP_CONFIGURATION_RESOURCES);
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
@ -78,7 +67,6 @@ public class JdbcCatalogService extends AbstractCatalogService {
catalogProperties.put(IcebergCatalogProperty.CATALOG_NAME, context.getProperty(CATALOG_NAME).evaluateAttributeExpressions().getValue());
catalogProperties.put(CLIENT_POOL_SERVICE, context.getProperty(CONNECTION_POOL).asControllerService(DBCPService.class));
catalogProperties.put(FILE_IO_IMPLEMENTATION, context.getProperty(FILE_IO_IMPL).evaluateAttributeExpressions().getValue());
catalogProperties.put(WAREHOUSE_LOCATION, context.getProperty(WAREHOUSE_PATH).evaluateAttributeExpressions().getValue());
}