mirror of https://github.com/apache/nifi.git
NIFI-271
This commit is contained in:
parent
6a706458d0
commit
9a3b6bed62
|
@ -42,7 +42,7 @@ import org.apache.nifi.stream.io.DataOutputStream;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
@SeeAlso(classNames={"org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer", "org.apache.nifi.ssl.StandardSSLContextService"})
|
@SeeAlso(classNames = {"org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer", "org.apache.nifi.ssl.StandardSSLContextService"})
|
||||||
@CapabilityDescription("Provides the ability to communicate with a DistributedMapCacheServer. This can be used in order to share a Map "
|
@CapabilityDescription("Provides the ability to communicate with a DistributedMapCacheServer. This can be used in order to share a Map "
|
||||||
+ "between nodes in a NiFi cluster")
|
+ "between nodes in a NiFi cluster")
|
||||||
public class DistributedMapCacheClientService extends AbstractControllerService implements DistributedMapCacheClient {
|
public class DistributedMapCacheClientService extends AbstractControllerService implements DistributedMapCacheClient {
|
||||||
|
@ -65,14 +65,14 @@ public class DistributedMapCacheClientService extends AbstractControllerService
|
||||||
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||||
.name("SSL Context Service")
|
.name("SSL Context Service")
|
||||||
.description("If specified, indicates the SSL Context Service that is used to communicate with the "
|
.description("If specified, indicates the SSL Context Service that is used to communicate with the "
|
||||||
+ "remote server. If not specified, communications will not be encrypted")
|
+ "remote server. If not specified, communications will not be encrypted")
|
||||||
.required(false)
|
.required(false)
|
||||||
.identifiesControllerService(SSLContextService.class)
|
.identifiesControllerService(SSLContextService.class)
|
||||||
.build();
|
.build();
|
||||||
public static final PropertyDescriptor COMMUNICATIONS_TIMEOUT = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor COMMUNICATIONS_TIMEOUT = new PropertyDescriptor.Builder()
|
||||||
.name("Communications Timeout")
|
.name("Communications Timeout")
|
||||||
.description("Specifies how long to wait when communicating with the remote server before determining that "
|
.description("Specifies how long to wait when communicating with the remote server before determining that "
|
||||||
+ "there is a communications failure if data cannot be sent or received")
|
+ "there is a communications failure if data cannot be sent or received")
|
||||||
.required(true)
|
.required(true)
|
||||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||||
.defaultValue("30 secs")
|
.defaultValue("30 secs")
|
||||||
|
@ -299,6 +299,7 @@ public class DistributedMapCacheClientService extends AbstractControllerService
|
||||||
}
|
}
|
||||||
|
|
||||||
private static interface CommsAction<T> {
|
private static interface CommsAction<T> {
|
||||||
|
|
||||||
T execute(CommsSession commsSession) throws IOException;
|
T execute(CommsSession commsSession) throws IOException;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -42,7 +42,7 @@ import org.apache.nifi.stream.io.DataOutputStream;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
@SeeAlso(classNames={"org.apache.nifi.distributed.cache.server.DistributedSetCacheServer", "org.apache.nifi.ssl.StandardSSLContextService"})
|
@SeeAlso(classNames = {"org.apache.nifi.distributed.cache.server.DistributedSetCacheServer", "org.apache.nifi.ssl.StandardSSLContextService"})
|
||||||
@CapabilityDescription("Provides the ability to communicate with a DistributedSetCacheServer. This can be used in order to share a Set "
|
@CapabilityDescription("Provides the ability to communicate with a DistributedSetCacheServer. This can be used in order to share a Set "
|
||||||
+ "between nodes in a NiFi cluster")
|
+ "between nodes in a NiFi cluster")
|
||||||
public class DistributedSetCacheClientService extends AbstractControllerService implements DistributedSetCacheClient {
|
public class DistributedSetCacheClientService extends AbstractControllerService implements DistributedSetCacheClient {
|
||||||
|
@ -65,14 +65,14 @@ public class DistributedSetCacheClientService extends AbstractControllerService
|
||||||
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||||
.name("SSL Context Service")
|
.name("SSL Context Service")
|
||||||
.description("If specified, indicates the SSL Context Service that is used to communicate with the "
|
.description("If specified, indicates the SSL Context Service that is used to communicate with the "
|
||||||
+ "remote server. If not specified, communications will not be encrypted")
|
+ "remote server. If not specified, communications will not be encrypted")
|
||||||
.required(false)
|
.required(false)
|
||||||
.identifiesControllerService(SSLContextService.class)
|
.identifiesControllerService(SSLContextService.class)
|
||||||
.build();
|
.build();
|
||||||
public static final PropertyDescriptor COMMUNICATIONS_TIMEOUT = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor COMMUNICATIONS_TIMEOUT = new PropertyDescriptor.Builder()
|
||||||
.name("Communications Timeout")
|
.name("Communications Timeout")
|
||||||
.description("Specifices how long to wait when communicating with the remote server before determining "
|
.description("Specifices how long to wait when communicating with the remote server before determining "
|
||||||
+ "that there is a communications failure if data cannot be sent or received")
|
+ "that there is a communications failure if data cannot be sent or received")
|
||||||
.required(true)
|
.required(true)
|
||||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||||
.defaultValue("30 secs")
|
.defaultValue("30 secs")
|
||||||
|
|
|
@ -30,6 +30,7 @@ import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelInputStream;
|
||||||
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelOutputStream;
|
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelOutputStream;
|
||||||
|
|
||||||
public class SSLCommsSession implements CommsSession {
|
public class SSLCommsSession implements CommsSession {
|
||||||
|
|
||||||
private final SSLSocketChannel sslSocketChannel;
|
private final SSLSocketChannel sslSocketChannel;
|
||||||
private final SSLContext sslContext;
|
private final SSLContext sslContext;
|
||||||
private final String hostname;
|
private final String hostname;
|
||||||
|
@ -94,10 +95,12 @@ public class SSLCommsSession implements CommsSession {
|
||||||
public int getPort() {
|
public int getPort() {
|
||||||
return port;
|
return port;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SSLContext getSSLContext() {
|
public SSLContext getSSLContext() {
|
||||||
return sslContext;
|
return sslContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getTimeout(final TimeUnit timeUnit) {
|
public long getTimeout(final TimeUnit timeUnit) {
|
||||||
return timeUnit.convert(sslSocketChannel.getTimeout(), TimeUnit.MILLISECONDS);
|
return timeUnit.convert(sslSocketChannel.getTimeout(), TimeUnit.MILLISECONDS);
|
||||||
|
|
|
@ -33,6 +33,7 @@ import org.apache.nifi.remote.io.socket.SocketChannelInputStream;
|
||||||
import org.apache.nifi.remote.io.socket.SocketChannelOutputStream;
|
import org.apache.nifi.remote.io.socket.SocketChannelOutputStream;
|
||||||
|
|
||||||
public class StandardCommsSession implements CommsSession {
|
public class StandardCommsSession implements CommsSession {
|
||||||
|
|
||||||
private final SocketChannel socketChannel;
|
private final SocketChannel socketChannel;
|
||||||
private final String hostname;
|
private final String hostname;
|
||||||
private final int port;
|
private final int port;
|
||||||
|
|
|
@ -1,35 +1,35 @@
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<!--
|
<!--
|
||||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
contributor license agreements. See the NOTICE file distributed with
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
this work for additional information regarding copyright ownership.
|
this work for additional information regarding copyright ownership.
|
||||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
(the "License"); you may not use this file except in compliance with
|
(the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
Unless required by applicable law or agreed to in writing, software
|
Unless required by applicable law or agreed to in writing, software
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
See the License for the specific language governing permissions and
|
See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
-->
|
-->
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<title>Distributed Map Cache Client Service</title>
|
<title>Distributed Map Cache Client Service</title>
|
||||||
<link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
|
<link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
<p>
|
<p>
|
||||||
Below is an example of how to create a client connection to your distributed map cache server.
|
Below is an example of how to create a client connection to your distributed map cache server.
|
||||||
Note that the identifier in this example is <code>cache-client</code>. If you are using this template
|
Note that the identifier in this example is <code>cache-client</code>. If you are using this template
|
||||||
to create your own MapCacheClient service, replace the values in this template with values that are
|
to create your own MapCacheClient service, replace the values in this template with values that are
|
||||||
suitable for your system. Possible options for <code>Server Hostname</code>, <code>Server Port</code>,
|
suitable for your system. Possible options for <code>Server Hostname</code>, <code>Server Port</code>,
|
||||||
<code>Communications Timeout</code>, and <span style="font-style: italic;">SSL Context Service</span>.
|
<code>Communications Timeout</code>, and <span style="font-style: italic;">SSL Context Service</span>.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<pre>
|
<pre>
|
||||||
<?xml version="1.0" encoding="UTF-8" ?>
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
<services>
|
<services>
|
||||||
<service>
|
<service>
|
||||||
|
@ -40,6 +40,6 @@
|
||||||
<property name="Communications Timeout">30 secs</property>
|
<property name="Communications Timeout">30 secs</property>
|
||||||
</service>
|
</service>
|
||||||
</services>
|
</services>
|
||||||
</pre>
|
</pre>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
@ -51,7 +51,8 @@ public abstract class AbstractCacheServer implements CacheServer {
|
||||||
private final int port;
|
private final int port;
|
||||||
private final SSLContext sslContext;
|
private final SSLContext sslContext;
|
||||||
protected volatile boolean stopped = false;
|
protected volatile boolean stopped = false;
|
||||||
private final Set<Thread> processInputThreads = new CopyOnWriteArraySet<>();;
|
private final Set<Thread> processInputThreads = new CopyOnWriteArraySet<>();
|
||||||
|
;
|
||||||
|
|
||||||
private volatile ServerSocketChannel serverSocketChannel;
|
private volatile ServerSocketChannel serverSocketChannel;
|
||||||
|
|
||||||
|
@ -75,7 +76,7 @@ public abstract class AbstractCacheServer implements CacheServer {
|
||||||
final SocketChannel socketChannel;
|
final SocketChannel socketChannel;
|
||||||
try {
|
try {
|
||||||
socketChannel = serverSocketChannel.accept();
|
socketChannel = serverSocketChannel.accept();
|
||||||
logger.debug("Connected to {}", new Object[] { socketChannel });
|
logger.debug("Connected to {}", new Object[]{socketChannel});
|
||||||
} catch (final IOException e) {
|
} catch (final IOException e) {
|
||||||
if (!stopped) {
|
if (!stopped) {
|
||||||
logger.error("{} unable to accept connection from remote peer due to {}", this, e.toString());
|
logger.error("{} unable to accept connection from remote peer due to {}", this, e.toString());
|
||||||
|
@ -104,7 +105,7 @@ public abstract class AbstractCacheServer implements CacheServer {
|
||||||
rawOutputStream = new SSLSocketChannelOutputStream(sslSocketChannel);
|
rawOutputStream = new SSLSocketChannelOutputStream(sslSocketChannel);
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("Cannot create input and/or output streams for {}", new Object[] { identifier }, e);
|
logger.error("Cannot create input and/or output streams for {}", new Object[]{identifier}, e);
|
||||||
if (logger.isDebugEnabled()) {
|
if (logger.isDebugEnabled()) {
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
}
|
}
|
||||||
|
@ -127,12 +128,12 @@ public abstract class AbstractCacheServer implements CacheServer {
|
||||||
continueComms = listen(in, out, versionNegotiator.getVersion());
|
continueComms = listen(in, out, versionNegotiator.getVersion());
|
||||||
}
|
}
|
||||||
// client has issued 'close'
|
// client has issued 'close'
|
||||||
logger.debug("Client issued close on {}", new Object[] { socketChannel });
|
logger.debug("Client issued close on {}", new Object[]{socketChannel});
|
||||||
} catch (final SocketTimeoutException e) {
|
} catch (final SocketTimeoutException e) {
|
||||||
logger.debug("30 sec timeout reached", e);
|
logger.debug("30 sec timeout reached", e);
|
||||||
} catch (final IOException | HandshakeException e) {
|
} catch (final IOException | HandshakeException e) {
|
||||||
if (!stopped) {
|
if (!stopped) {
|
||||||
logger.error("{} unable to communicate with remote peer {} due to {}", new Object[] { this, peer, e.toString() });
|
logger.error("{} unable to communicate with remote peer {} due to {}", new Object[]{this, peer, e.toString()});
|
||||||
if (logger.isDebugEnabled()) {
|
if (logger.isDebugEnabled()) {
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
}
|
}
|
||||||
|
@ -161,7 +162,7 @@ public abstract class AbstractCacheServer implements CacheServer {
|
||||||
@Override
|
@Override
|
||||||
public void stop() throws IOException {
|
public void stop() throws IOException {
|
||||||
stopped = true;
|
stopped = true;
|
||||||
logger.info("Stopping CacheServer {}", new Object[] { this.identifier });
|
logger.info("Stopping CacheServer {}", new Object[]{this.identifier});
|
||||||
|
|
||||||
if (serverSocketChannel != null && serverSocketChannel.isOpen()) {
|
if (serverSocketChannel != null && serverSocketChannel.isOpen()) {
|
||||||
serverSocketChannel.close();
|
serverSocketChannel.close();
|
||||||
|
@ -189,11 +190,11 @@ public abstract class AbstractCacheServer implements CacheServer {
|
||||||
/**
|
/**
|
||||||
* Listens for incoming data and communicates with remote peer
|
* Listens for incoming data and communicates with remote peer
|
||||||
*
|
*
|
||||||
* @param in
|
* @param in in
|
||||||
* @param out
|
* @param out out
|
||||||
* @param version
|
* @param version version
|
||||||
* @return <code>true</code> if communications should continue, <code>false</code> otherwise
|
* @return <code>true</code> if communications should continue, <code>false</code> otherwise
|
||||||
* @throws IOException
|
* @throws IOException ex
|
||||||
*/
|
*/
|
||||||
protected abstract boolean listen(InputStream in, OutputStream out, int version) throws IOException;
|
protected abstract boolean listen(InputStream in, OutputStream out, int version) throws IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.io.IOException;
|
||||||
public interface CacheServer {
|
public interface CacheServer {
|
||||||
|
|
||||||
void start() throws IOException;
|
void start() throws IOException;
|
||||||
|
|
||||||
void stop() throws IOException;
|
void stop() throws IOException;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.nifi.processor.util.StandardValidators;
|
||||||
import org.apache.nifi.ssl.SSLContextService;
|
import org.apache.nifi.ssl.SSLContextService;
|
||||||
|
|
||||||
public abstract class DistributedCacheServer extends AbstractControllerService {
|
public abstract class DistributedCacheServer extends AbstractControllerService {
|
||||||
|
|
||||||
public static final String EVICTION_STRATEGY_LFU = "Least Frequently Used";
|
public static final String EVICTION_STRATEGY_LFU = "Least Frequently Used";
|
||||||
public static final String EVICTION_STRATEGY_LRU = "Least Recently Used";
|
public static final String EVICTION_STRATEGY_LRU = "Least Recently Used";
|
||||||
public static final String EVICTION_STRATEGY_FIFO = "First In, First Out";
|
public static final String EVICTION_STRATEGY_FIFO = "First In, First Out";
|
||||||
|
@ -43,7 +44,7 @@ public abstract class DistributedCacheServer extends AbstractControllerService {
|
||||||
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||||
.name("SSL Context Service")
|
.name("SSL Context Service")
|
||||||
.description("If specified, this service will be used to create an SSL Context that will be used "
|
.description("If specified, this service will be used to create an SSL Context that will be used "
|
||||||
+ "to secure communications; if not specified, communications will not be secure")
|
+ "to secure communications; if not specified, communications will not be secure")
|
||||||
.required(false)
|
.required(false)
|
||||||
.identifiesControllerService(SSLContextService.class)
|
.identifiesControllerService(SSLContextService.class)
|
||||||
.build();
|
.build();
|
||||||
|
|
|
@ -25,6 +25,7 @@ import org.apache.nifi.annotation.documentation.Tags;
|
||||||
import org.apache.nifi.controller.ConfigurationContext;
|
import org.apache.nifi.controller.ConfigurationContext;
|
||||||
import org.apache.nifi.ssl.SSLContextService;
|
import org.apache.nifi.ssl.SSLContextService;
|
||||||
import org.apache.nifi.ssl.SSLContextService.ClientAuth;
|
import org.apache.nifi.ssl.SSLContextService.ClientAuth;
|
||||||
|
|
||||||
@Tags({"distributed", "set", "distinct", "cache", "server"})
|
@Tags({"distributed", "set", "distinct", "cache", "server"})
|
||||||
@CapabilityDescription("Provides a set (collection of unique values) cache that can be accessed over a socket. "
|
@CapabilityDescription("Provides a set (collection of unique values) cache that can be accessed over a socket. "
|
||||||
+ "Interaction with this service is typically accomplished via a DistributedSetCacheClient service.")
|
+ "Interaction with this service is typically accomplished via a DistributedSetCacheClient service.")
|
||||||
|
@ -39,7 +40,7 @@ public class DistributedSetCacheServer extends DistributedCacheServer {
|
||||||
final String evictionPolicyName = context.getProperty(EVICTION_POLICY).getValue();
|
final String evictionPolicyName = context.getProperty(EVICTION_POLICY).getValue();
|
||||||
|
|
||||||
final SSLContext sslContext;
|
final SSLContext sslContext;
|
||||||
if ( sslContextService == null ) {
|
if (sslContextService == null) {
|
||||||
sslContext = null;
|
sslContext = null;
|
||||||
} else {
|
} else {
|
||||||
sslContext = sslContextService.createSSLContext(ClientAuth.REQUIRED);
|
sslContext = sslContextService.createSSLContext(ClientAuth.REQUIRED);
|
||||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.nifi.distributed.cache.server;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
|
||||||
public enum EvictionPolicy {
|
public enum EvictionPolicy {
|
||||||
|
|
||||||
LFU(new LFUComparator()),
|
LFU(new LFUComparator()),
|
||||||
LRU(new LRUComparator()),
|
LRU(new LRUComparator()),
|
||||||
FIFO(new FIFOComparator());
|
FIFO(new FIFOComparator());
|
||||||
|
@ -34,9 +35,10 @@ public enum EvictionPolicy {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class LFUComparator implements Comparator<CacheRecord> {
|
public static class LFUComparator implements Comparator<CacheRecord> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
||||||
if ( o1.equals(o2) ) {
|
if (o1.equals(o2)) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -47,9 +49,10 @@ public enum EvictionPolicy {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class LRUComparator implements Comparator<CacheRecord> {
|
public static class LRUComparator implements Comparator<CacheRecord> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
||||||
if ( o1.equals(o2) ) {
|
if (o1.equals(o2)) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,9 +62,10 @@ public enum EvictionPolicy {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class FIFOComparator implements Comparator<CacheRecord> {
|
public static class FIFOComparator implements Comparator<CacheRecord> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
||||||
if ( o1.equals(o2) ) {
|
if (o1.equals(o2)) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -67,17 +67,17 @@ public class SetCacheServer extends AbstractCacheServer {
|
||||||
|
|
||||||
final SetCacheResult response;
|
final SetCacheResult response;
|
||||||
switch (action) {
|
switch (action) {
|
||||||
case "addIfAbsent":
|
case "addIfAbsent":
|
||||||
response = cache.addIfAbsent(valueBuffer);
|
response = cache.addIfAbsent(valueBuffer);
|
||||||
break;
|
break;
|
||||||
case "contains":
|
case "contains":
|
||||||
response = cache.contains(valueBuffer);
|
response = cache.contains(valueBuffer);
|
||||||
break;
|
break;
|
||||||
case "remove":
|
case "remove":
|
||||||
response = cache.remove(valueBuffer);
|
response = cache.remove(valueBuffer);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new IOException("IllegalRequest");
|
throw new IOException("IllegalRequest");
|
||||||
}
|
}
|
||||||
|
|
||||||
dos.writeBoolean(response.getResult());
|
dos.writeBoolean(response.getResult());
|
||||||
|
@ -97,8 +97,9 @@ public class SetCacheServer extends AbstractCacheServer {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void finalize() throws Throwable {
|
protected void finalize() throws Throwable {
|
||||||
if (!stopped)
|
if (!stopped) {
|
||||||
stop();
|
stop();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,7 +33,7 @@ import org.apache.nifi.ssl.SSLContextService.ClientAuth;
|
||||||
@Tags({"distributed", "cluster", "map", "cache", "server", "key/value"})
|
@Tags({"distributed", "cluster", "map", "cache", "server", "key/value"})
|
||||||
@CapabilityDescription("Provides a map (key/value) cache that can be accessed over a socket. Interaction with this service"
|
@CapabilityDescription("Provides a map (key/value) cache that can be accessed over a socket. Interaction with this service"
|
||||||
+ " is typically accomplished via a DistributedMapCacheClient service.")
|
+ " is typically accomplished via a DistributedMapCacheClient service.")
|
||||||
@SeeAlso(classNames={"org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService", "org.apache.nifi.ssl.StandardSSLContextService"})
|
@SeeAlso(classNames = {"org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService", "org.apache.nifi.ssl.StandardSSLContextService"})
|
||||||
public class DistributedMapCacheServer extends DistributedCacheServer {
|
public class DistributedMapCacheServer extends DistributedCacheServer {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -45,7 +45,7 @@ public class DistributedMapCacheServer extends DistributedCacheServer {
|
||||||
final String evictionPolicyName = context.getProperty(EVICTION_POLICY).getValue();
|
final String evictionPolicyName = context.getProperty(EVICTION_POLICY).getValue();
|
||||||
|
|
||||||
final SSLContext sslContext;
|
final SSLContext sslContext;
|
||||||
if ( sslContextService == null ) {
|
if (sslContextService == null) {
|
||||||
sslContext = null;
|
sslContext = null;
|
||||||
} else {
|
} else {
|
||||||
sslContext = sslContextService.createSSLContext(ClientAuth.REQUIRED);
|
sslContext = sslContextService.createSSLContext(ClientAuth.REQUIRED);
|
||||||
|
|
|
@ -22,8 +22,12 @@ import java.nio.ByteBuffer;
|
||||||
public interface MapCache {
|
public interface MapCache {
|
||||||
|
|
||||||
MapPutResult putIfAbsent(ByteBuffer key, ByteBuffer value) throws IOException;
|
MapPutResult putIfAbsent(ByteBuffer key, ByteBuffer value) throws IOException;
|
||||||
|
|
||||||
boolean containsKey(ByteBuffer key) throws IOException;
|
boolean containsKey(ByteBuffer key) throws IOException;
|
||||||
|
|
||||||
ByteBuffer get(ByteBuffer key) throws IOException;
|
ByteBuffer get(ByteBuffer key) throws IOException;
|
||||||
|
|
||||||
ByteBuffer remove(ByteBuffer key) throws IOException;
|
ByteBuffer remove(ByteBuffer key) throws IOException;
|
||||||
|
|
||||||
void shutdown() throws IOException;
|
void shutdown() throws IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
|
||||||
import org.apache.nifi.distributed.cache.server.CacheRecord;
|
import org.apache.nifi.distributed.cache.server.CacheRecord;
|
||||||
|
|
||||||
public class MapCacheRecord extends CacheRecord {
|
public class MapCacheRecord extends CacheRecord {
|
||||||
|
|
||||||
private final ByteBuffer key;
|
private final ByteBuffer key;
|
||||||
private final ByteBuffer value;
|
private final ByteBuffer value;
|
||||||
|
|
||||||
|
@ -44,11 +45,11 @@ public class MapCacheRecord extends CacheRecord {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(final Object obj) {
|
public boolean equals(final Object obj) {
|
||||||
if ( obj == this ) {
|
if (obj == this) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( obj instanceof MapCacheRecord ) {
|
if (obj instanceof MapCacheRecord) {
|
||||||
final MapCacheRecord that = ((MapCacheRecord) obj);
|
final MapCacheRecord that = ((MapCacheRecord) obj);
|
||||||
return key.equals(that.key) && value.equals(that.value);
|
return key.equals(that.key) && value.equals(that.value);
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,63 +55,63 @@ public class MapCacheServer extends AbstractCacheServer {
|
||||||
final String action = dis.readUTF();
|
final String action = dis.readUTF();
|
||||||
try {
|
try {
|
||||||
switch (action) {
|
switch (action) {
|
||||||
case "close": {
|
case "close": {
|
||||||
return false;
|
return false;
|
||||||
}
|
|
||||||
case "putIfAbsent": {
|
|
||||||
final byte[] key = readValue(dis);
|
|
||||||
final byte[] value = readValue(dis);
|
|
||||||
final MapPutResult putResult = cache.putIfAbsent(ByteBuffer.wrap(key), ByteBuffer.wrap(value));
|
|
||||||
dos.writeBoolean(putResult.isSuccessful());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case "containsKey": {
|
|
||||||
final byte[] key = readValue(dis);
|
|
||||||
final boolean contains = cache.containsKey(ByteBuffer.wrap(key));
|
|
||||||
dos.writeBoolean(contains);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case "getAndPutIfAbsent": {
|
|
||||||
final byte[] key = readValue(dis);
|
|
||||||
final byte[] value = readValue(dis);
|
|
||||||
|
|
||||||
final MapPutResult putResult = cache.putIfAbsent(ByteBuffer.wrap(key), ByteBuffer.wrap(value));
|
|
||||||
if (putResult.isSuccessful()) {
|
|
||||||
// Put was successful. There was no old value to get.
|
|
||||||
dos.writeInt(0);
|
|
||||||
} else {
|
|
||||||
// we didn't put. Write back the previous value
|
|
||||||
final byte[] byteArray = putResult.getExistingValue().array();
|
|
||||||
dos.writeInt(byteArray.length);
|
|
||||||
dos.write(byteArray);
|
|
||||||
}
|
}
|
||||||
|
case "putIfAbsent": {
|
||||||
break;
|
final byte[] key = readValue(dis);
|
||||||
}
|
final byte[] value = readValue(dis);
|
||||||
case "get": {
|
final MapPutResult putResult = cache.putIfAbsent(ByteBuffer.wrap(key), ByteBuffer.wrap(value));
|
||||||
final byte[] key = readValue(dis);
|
dos.writeBoolean(putResult.isSuccessful());
|
||||||
final ByteBuffer existingValue = cache.get(ByteBuffer.wrap(key));
|
break;
|
||||||
if (existingValue == null) {
|
|
||||||
// there was no existing value; we did a "put".
|
|
||||||
dos.writeInt(0);
|
|
||||||
} else {
|
|
||||||
// a value already existed. we did not update the map
|
|
||||||
final byte[] byteArray = existingValue.array();
|
|
||||||
dos.writeInt(byteArray.length);
|
|
||||||
dos.write(byteArray);
|
|
||||||
}
|
}
|
||||||
|
case "containsKey": {
|
||||||
|
final byte[] key = readValue(dis);
|
||||||
|
final boolean contains = cache.containsKey(ByteBuffer.wrap(key));
|
||||||
|
dos.writeBoolean(contains);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "getAndPutIfAbsent": {
|
||||||
|
final byte[] key = readValue(dis);
|
||||||
|
final byte[] value = readValue(dis);
|
||||||
|
|
||||||
break;
|
final MapPutResult putResult = cache.putIfAbsent(ByteBuffer.wrap(key), ByteBuffer.wrap(value));
|
||||||
}
|
if (putResult.isSuccessful()) {
|
||||||
case "remove": {
|
// Put was successful. There was no old value to get.
|
||||||
final byte[] key = readValue(dis);
|
dos.writeInt(0);
|
||||||
final boolean removed = cache.remove(ByteBuffer.wrap(key)) != null;
|
} else {
|
||||||
dos.writeBoolean(removed);
|
// we didn't put. Write back the previous value
|
||||||
break;
|
final byte[] byteArray = putResult.getExistingValue().array();
|
||||||
}
|
dos.writeInt(byteArray.length);
|
||||||
default: {
|
dos.write(byteArray);
|
||||||
throw new IOException("Illegal Request");
|
}
|
||||||
}
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "get": {
|
||||||
|
final byte[] key = readValue(dis);
|
||||||
|
final ByteBuffer existingValue = cache.get(ByteBuffer.wrap(key));
|
||||||
|
if (existingValue == null) {
|
||||||
|
// there was no existing value; we did a "put".
|
||||||
|
dos.writeInt(0);
|
||||||
|
} else {
|
||||||
|
// a value already existed. we did not update the map
|
||||||
|
final byte[] byteArray = existingValue.array();
|
||||||
|
dos.writeInt(byteArray.length);
|
||||||
|
dos.write(byteArray);
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "remove": {
|
||||||
|
final byte[] key = readValue(dis);
|
||||||
|
final boolean removed = cache.remove(ByteBuffer.wrap(key)) != null;
|
||||||
|
dos.writeBoolean(removed);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default: {
|
||||||
|
throw new IOException("Illegal Request");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
dos.flush();
|
dos.flush();
|
||||||
|
@ -131,8 +131,9 @@ public class MapCacheServer extends AbstractCacheServer {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void finalize() throws Throwable {
|
protected void finalize() throws Throwable {
|
||||||
if (!stopped)
|
if (!stopped) {
|
||||||
stop();
|
stop();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private byte[] readValue(final DataInputStream dis) throws IOException {
|
private byte[] readValue(final DataInputStream dis) throws IOException {
|
||||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.nifi.distributed.cache.server.map;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
public class MapPutResult {
|
public class MapPutResult {
|
||||||
|
|
||||||
private final boolean successful;
|
private final boolean successful;
|
||||||
private final ByteBuffer key, value;
|
private final ByteBuffer key, value;
|
||||||
private final ByteBuffer existingValue;
|
private final ByteBuffer existingValue;
|
||||||
|
|
|
@ -48,8 +48,8 @@ public class PersistentMapCache implements MapCache {
|
||||||
|
|
||||||
synchronized void restore() throws IOException {
|
synchronized void restore() throws IOException {
|
||||||
final Collection<MapWaliRecord> recovered = wali.recoverRecords();
|
final Collection<MapWaliRecord> recovered = wali.recoverRecords();
|
||||||
for ( final MapWaliRecord record : recovered ) {
|
for (final MapWaliRecord record : recovered) {
|
||||||
if ( record.getUpdateType() == UpdateType.CREATE ) {
|
if (record.getUpdateType() == UpdateType.CREATE) {
|
||||||
wrapped.putIfAbsent(record.getKey(), record.getValue());
|
wrapped.putIfAbsent(record.getKey(), record.getValue());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -58,20 +58,20 @@ public class PersistentMapCache implements MapCache {
|
||||||
@Override
|
@Override
|
||||||
public MapPutResult putIfAbsent(final ByteBuffer key, final ByteBuffer value) throws IOException {
|
public MapPutResult putIfAbsent(final ByteBuffer key, final ByteBuffer value) throws IOException {
|
||||||
final MapPutResult putResult = wrapped.putIfAbsent(key, value);
|
final MapPutResult putResult = wrapped.putIfAbsent(key, value);
|
||||||
if ( putResult.isSuccessful() ) {
|
if (putResult.isSuccessful()) {
|
||||||
// The put was successful.
|
// The put was successful.
|
||||||
final MapWaliRecord record = new MapWaliRecord(UpdateType.CREATE, key, value);
|
final MapWaliRecord record = new MapWaliRecord(UpdateType.CREATE, key, value);
|
||||||
final List<MapWaliRecord> records = new ArrayList<>();
|
final List<MapWaliRecord> records = new ArrayList<>();
|
||||||
records.add(record);
|
records.add(record);
|
||||||
|
|
||||||
if ( putResult.getEvictedKey() != null ) {
|
if (putResult.getEvictedKey() != null) {
|
||||||
records.add(new MapWaliRecord(UpdateType.DELETE, putResult.getEvictedKey(), putResult.getEvictedValue()));
|
records.add(new MapWaliRecord(UpdateType.DELETE, putResult.getEvictedKey(), putResult.getEvictedValue()));
|
||||||
}
|
}
|
||||||
|
|
||||||
wali.update(Collections.singletonList(record), false);
|
wali.update(Collections.singletonList(record), false);
|
||||||
|
|
||||||
final long modCount = modifications.getAndIncrement();
|
final long modCount = modifications.getAndIncrement();
|
||||||
if ( modCount > 0 && modCount % 100000 == 0 ) {
|
if (modCount > 0 && modCount % 100000 == 0) {
|
||||||
wali.checkpoint();
|
wali.checkpoint();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -92,28 +92,27 @@ public class PersistentMapCache implements MapCache {
|
||||||
@Override
|
@Override
|
||||||
public ByteBuffer remove(ByteBuffer key) throws IOException {
|
public ByteBuffer remove(ByteBuffer key) throws IOException {
|
||||||
final ByteBuffer removeResult = wrapped.remove(key);
|
final ByteBuffer removeResult = wrapped.remove(key);
|
||||||
if ( removeResult != null ) {
|
if (removeResult != null) {
|
||||||
final MapWaliRecord record = new MapWaliRecord(UpdateType.DELETE, key, removeResult);
|
final MapWaliRecord record = new MapWaliRecord(UpdateType.DELETE, key, removeResult);
|
||||||
final List<MapWaliRecord> records = new ArrayList<>(1);
|
final List<MapWaliRecord> records = new ArrayList<>(1);
|
||||||
records.add(record);
|
records.add(record);
|
||||||
wali.update(records, false);
|
wali.update(records, false);
|
||||||
|
|
||||||
final long modCount = modifications.getAndIncrement();
|
final long modCount = modifications.getAndIncrement();
|
||||||
if ( modCount > 0 && modCount % 1000 == 0 ) {
|
if (modCount > 0 && modCount % 1000 == 0) {
|
||||||
wali.checkpoint();
|
wali.checkpoint();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return removeResult;
|
return removeResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void shutdown() throws IOException {
|
public void shutdown() throws IOException {
|
||||||
wali.shutdown();
|
wali.shutdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static class MapWaliRecord {
|
private static class MapWaliRecord {
|
||||||
|
|
||||||
private final UpdateType updateType;
|
private final UpdateType updateType;
|
||||||
private final ByteBuffer key;
|
private final ByteBuffer key;
|
||||||
private final ByteBuffer value;
|
private final ByteBuffer value;
|
||||||
|
@ -142,7 +141,7 @@ public class PersistentMapCache implements MapCache {
|
||||||
@Override
|
@Override
|
||||||
public void serializeEdit(MapWaliRecord previousRecordState, MapWaliRecord newRecordState, java.io.DataOutputStream out) throws IOException {
|
public void serializeEdit(MapWaliRecord previousRecordState, MapWaliRecord newRecordState, java.io.DataOutputStream out) throws IOException {
|
||||||
final UpdateType updateType = newRecordState.getUpdateType();
|
final UpdateType updateType = newRecordState.getUpdateType();
|
||||||
if ( updateType == UpdateType.DELETE ) {
|
if (updateType == UpdateType.DELETE) {
|
||||||
out.write(0);
|
out.write(0);
|
||||||
} else {
|
} else {
|
||||||
out.write(1);
|
out.write(1);
|
||||||
|
@ -165,7 +164,7 @@ public class PersistentMapCache implements MapCache {
|
||||||
@Override
|
@Override
|
||||||
public MapWaliRecord deserializeEdit(final DataInputStream in, final Map<Object, MapWaliRecord> currentRecordStates, final int version) throws IOException {
|
public MapWaliRecord deserializeEdit(final DataInputStream in, final Map<Object, MapWaliRecord> currentRecordStates, final int version) throws IOException {
|
||||||
final int updateTypeValue = in.read();
|
final int updateTypeValue = in.read();
|
||||||
if ( updateTypeValue < 0 ) {
|
if (updateTypeValue < 0) {
|
||||||
throw new EOFException();
|
throw new EOFException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -33,6 +33,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class SimpleMapCache implements MapCache {
|
public class SimpleMapCache implements MapCache {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(SimpleMapCache.class);
|
private static final Logger logger = LoggerFactory.getLogger(SimpleMapCache.class);
|
||||||
|
|
||||||
private final Map<ByteBuffer, MapCacheRecord> cache = new HashMap<>();
|
private final Map<ByteBuffer, MapCacheRecord> cache = new HashMap<>();
|
||||||
|
@ -61,7 +62,7 @@ public class SimpleMapCache implements MapCache {
|
||||||
// don't need synchronized because this method is only called when the writeLock is held, and all
|
// don't need synchronized because this method is only called when the writeLock is held, and all
|
||||||
// public methods obtain either the read or write lock
|
// public methods obtain either the read or write lock
|
||||||
private MapCacheRecord evict() {
|
private MapCacheRecord evict() {
|
||||||
if ( cache.size() < maxSize ) {
|
if (cache.size() < maxSize) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -69,7 +70,7 @@ public class SimpleMapCache implements MapCache {
|
||||||
final ByteBuffer valueToEvict = inverseCacheMap.remove(recordToEvict);
|
final ByteBuffer valueToEvict = inverseCacheMap.remove(recordToEvict);
|
||||||
cache.remove(valueToEvict);
|
cache.remove(valueToEvict);
|
||||||
|
|
||||||
if ( logger.isDebugEnabled() ) {
|
if (logger.isDebugEnabled()) {
|
||||||
logger.debug("Evicting value {} from cache", new String(valueToEvict.array(), StandardCharsets.UTF_8));
|
logger.debug("Evicting value {} from cache", new String(valueToEvict.array(), StandardCharsets.UTF_8));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,14 +82,14 @@ public class SimpleMapCache implements MapCache {
|
||||||
writeLock.lock();
|
writeLock.lock();
|
||||||
try {
|
try {
|
||||||
final MapCacheRecord record = cache.get(key);
|
final MapCacheRecord record = cache.get(key);
|
||||||
if ( record == null ) {
|
if (record == null) {
|
||||||
// Record is null. We will add.
|
// Record is null. We will add.
|
||||||
final MapCacheRecord evicted = evict();
|
final MapCacheRecord evicted = evict();
|
||||||
final MapCacheRecord newRecord = new MapCacheRecord(key, value);
|
final MapCacheRecord newRecord = new MapCacheRecord(key, value);
|
||||||
cache.put(key, newRecord);
|
cache.put(key, newRecord);
|
||||||
inverseCacheMap.put(newRecord, key);
|
inverseCacheMap.put(newRecord, key);
|
||||||
|
|
||||||
if ( evicted == null ) {
|
if (evicted == null) {
|
||||||
return new MapPutResult(true, key, value, null, null, null);
|
return new MapPutResult(true, key, value, null, null, null);
|
||||||
} else {
|
} else {
|
||||||
return new MapPutResult(true, key, value, null, evicted.getKey(), evicted.getValue());
|
return new MapPutResult(true, key, value, null, evicted.getKey(), evicted.getValue());
|
||||||
|
@ -111,7 +112,7 @@ public class SimpleMapCache implements MapCache {
|
||||||
readLock.lock();
|
readLock.lock();
|
||||||
try {
|
try {
|
||||||
final MapCacheRecord record = cache.get(key);
|
final MapCacheRecord record = cache.get(key);
|
||||||
if ( record == null ) {
|
if (record == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -130,7 +131,7 @@ public class SimpleMapCache implements MapCache {
|
||||||
readLock.lock();
|
readLock.lock();
|
||||||
try {
|
try {
|
||||||
final MapCacheRecord record = cache.get(key);
|
final MapCacheRecord record = cache.get(key);
|
||||||
if ( record == null ) {
|
if (record == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -48,8 +48,8 @@ public class PersistentSetCache implements SetCache {
|
||||||
|
|
||||||
public synchronized void restore() throws IOException {
|
public synchronized void restore() throws IOException {
|
||||||
final Collection<SetRecord> recovered = wali.recoverRecords();
|
final Collection<SetRecord> recovered = wali.recoverRecords();
|
||||||
for ( final SetRecord record : recovered ) {
|
for (final SetRecord record : recovered) {
|
||||||
if ( record.getUpdateType() == UpdateType.CREATE ) {
|
if (record.getUpdateType() == UpdateType.CREATE) {
|
||||||
addIfAbsent(record.getBuffer());
|
addIfAbsent(record.getBuffer());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -58,14 +58,14 @@ public class PersistentSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public synchronized SetCacheResult remove(final ByteBuffer value) throws IOException {
|
public synchronized SetCacheResult remove(final ByteBuffer value) throws IOException {
|
||||||
final SetCacheResult removeResult = wrapped.remove(value);
|
final SetCacheResult removeResult = wrapped.remove(value);
|
||||||
if ( removeResult.getResult() ) {
|
if (removeResult.getResult()) {
|
||||||
final SetRecord record = new SetRecord(UpdateType.DELETE, value);
|
final SetRecord record = new SetRecord(UpdateType.DELETE, value);
|
||||||
final List<SetRecord> records = new ArrayList<>();
|
final List<SetRecord> records = new ArrayList<>();
|
||||||
records.add(record);
|
records.add(record);
|
||||||
wali.update(records, false);
|
wali.update(records, false);
|
||||||
|
|
||||||
final long modCount = modifications.getAndIncrement();
|
final long modCount = modifications.getAndIncrement();
|
||||||
if ( modCount > 0 && modCount % 1000 == 0 ) {
|
if (modCount > 0 && modCount % 1000 == 0) {
|
||||||
wali.checkpoint();
|
wali.checkpoint();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -76,20 +76,20 @@ public class PersistentSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public synchronized SetCacheResult addIfAbsent(final ByteBuffer value) throws IOException {
|
public synchronized SetCacheResult addIfAbsent(final ByteBuffer value) throws IOException {
|
||||||
final SetCacheResult addResult = wrapped.addIfAbsent(value);
|
final SetCacheResult addResult = wrapped.addIfAbsent(value);
|
||||||
if ( addResult.getResult() ) {
|
if (addResult.getResult()) {
|
||||||
final SetRecord record = new SetRecord(UpdateType.CREATE, value);
|
final SetRecord record = new SetRecord(UpdateType.CREATE, value);
|
||||||
final List<SetRecord> records = new ArrayList<>();
|
final List<SetRecord> records = new ArrayList<>();
|
||||||
records.add(record);
|
records.add(record);
|
||||||
|
|
||||||
final SetCacheRecord evictedRecord = addResult.getEvictedRecord();
|
final SetCacheRecord evictedRecord = addResult.getEvictedRecord();
|
||||||
if ( evictedRecord != null ) {
|
if (evictedRecord != null) {
|
||||||
records.add(new SetRecord(UpdateType.DELETE, evictedRecord.getValue()));
|
records.add(new SetRecord(UpdateType.DELETE, evictedRecord.getValue()));
|
||||||
}
|
}
|
||||||
|
|
||||||
wali.update(records, false);
|
wali.update(records, false);
|
||||||
|
|
||||||
final long modCount = modifications.getAndIncrement();
|
final long modCount = modifications.getAndIncrement();
|
||||||
if ( modCount > 0 && modCount % 1000 == 0 ) {
|
if (modCount > 0 && modCount % 1000 == 0) {
|
||||||
wali.checkpoint();
|
wali.checkpoint();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -108,6 +108,7 @@ public class PersistentSetCache implements SetCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class SetRecord {
|
private static class SetRecord {
|
||||||
|
|
||||||
private final UpdateType updateType;
|
private final UpdateType updateType;
|
||||||
private final ByteBuffer value;
|
private final ByteBuffer value;
|
||||||
|
|
||||||
|
@ -134,7 +135,7 @@ public class PersistentSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public void serializeEdit(final SetRecord previousRecordState, final SetRecord newRecordState, final DataOutputStream out) throws IOException {
|
public void serializeEdit(final SetRecord previousRecordState, final SetRecord newRecordState, final DataOutputStream out) throws IOException {
|
||||||
final UpdateType updateType = newRecordState.getUpdateType();
|
final UpdateType updateType = newRecordState.getUpdateType();
|
||||||
if ( updateType == UpdateType.DELETE ) {
|
if (updateType == UpdateType.DELETE) {
|
||||||
out.write(0);
|
out.write(0);
|
||||||
} else {
|
} else {
|
||||||
out.write(1);
|
out.write(1);
|
||||||
|
@ -153,7 +154,7 @@ public class PersistentSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public SetRecord deserializeEdit(final DataInputStream in, final Map<Object, SetRecord> currentRecordStates, final int version) throws IOException {
|
public SetRecord deserializeEdit(final DataInputStream in, final Map<Object, SetRecord> currentRecordStates, final int version) throws IOException {
|
||||||
final int value = in.read();
|
final int value = in.read();
|
||||||
if ( value < 0 ) {
|
if (value < 0) {
|
||||||
throw new EOFException();
|
throw new EOFException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -22,8 +22,11 @@ import java.nio.ByteBuffer;
|
||||||
public interface SetCache {
|
public interface SetCache {
|
||||||
|
|
||||||
SetCacheResult remove(ByteBuffer value) throws IOException;
|
SetCacheResult remove(ByteBuffer value) throws IOException;
|
||||||
|
|
||||||
SetCacheResult addIfAbsent(ByteBuffer value) throws IOException;
|
SetCacheResult addIfAbsent(ByteBuffer value) throws IOException;
|
||||||
|
|
||||||
SetCacheResult contains(ByteBuffer value) throws IOException;
|
SetCacheResult contains(ByteBuffer value) throws IOException;
|
||||||
|
|
||||||
void shutdown() throws IOException;
|
void shutdown() throws IOException;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
|
||||||
import org.apache.nifi.distributed.cache.server.CacheRecord;
|
import org.apache.nifi.distributed.cache.server.CacheRecord;
|
||||||
|
|
||||||
public class SetCacheRecord extends CacheRecord {
|
public class SetCacheRecord extends CacheRecord {
|
||||||
|
|
||||||
private final ByteBuffer value;
|
private final ByteBuffer value;
|
||||||
|
|
||||||
public SetCacheRecord(final ByteBuffer value) {
|
public SetCacheRecord(final ByteBuffer value) {
|
||||||
|
@ -38,7 +39,7 @@ public class SetCacheRecord extends CacheRecord {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(final Object obj) {
|
public boolean equals(final Object obj) {
|
||||||
if ( this == obj ) {
|
if (this == obj) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,9 +16,8 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.nifi.distributed.cache.server.set;
|
package org.apache.nifi.distributed.cache.server.set;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public class SetCacheResult {
|
public class SetCacheResult {
|
||||||
|
|
||||||
private final boolean result;
|
private final boolean result;
|
||||||
private final SetCacheRecord stats;
|
private final SetCacheRecord stats;
|
||||||
private final SetCacheRecord evictedRecord;
|
private final SetCacheRecord evictedRecord;
|
||||||
|
|
|
@ -30,6 +30,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class SimpleSetCache implements SetCache {
|
public class SimpleSetCache implements SetCache {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(SimpleSetCache.class);
|
private static final Logger logger = LoggerFactory.getLogger(SimpleSetCache.class);
|
||||||
|
|
||||||
private final Map<ByteBuffer, SetCacheRecord> cache = new HashMap<>();
|
private final Map<ByteBuffer, SetCacheRecord> cache = new HashMap<>();
|
||||||
|
@ -46,7 +47,7 @@ public class SimpleSetCache implements SetCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
private synchronized SetCacheRecord evict() {
|
private synchronized SetCacheRecord evict() {
|
||||||
if ( cache.size() < maxSize ) {
|
if (cache.size() < maxSize) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -54,7 +55,7 @@ public class SimpleSetCache implements SetCache {
|
||||||
final ByteBuffer valueToEvict = inverseCacheMap.remove(recordToEvict);
|
final ByteBuffer valueToEvict = inverseCacheMap.remove(recordToEvict);
|
||||||
cache.remove(valueToEvict);
|
cache.remove(valueToEvict);
|
||||||
|
|
||||||
if ( logger.isDebugEnabled() ) {
|
if (logger.isDebugEnabled()) {
|
||||||
logger.debug("Evicting value {} from cache", new String(valueToEvict.array(), StandardCharsets.UTF_8));
|
logger.debug("Evicting value {} from cache", new String(valueToEvict.array(), StandardCharsets.UTF_8));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -64,7 +65,7 @@ public class SimpleSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public synchronized SetCacheResult addIfAbsent(final ByteBuffer value) {
|
public synchronized SetCacheResult addIfAbsent(final ByteBuffer value) {
|
||||||
final SetCacheRecord record = cache.get(value);
|
final SetCacheRecord record = cache.get(value);
|
||||||
if ( record == null ) {
|
if (record == null) {
|
||||||
final SetCacheRecord evicted = evict();
|
final SetCacheRecord evicted = evict();
|
||||||
final SetCacheRecord newRecord = new SetCacheRecord(value);
|
final SetCacheRecord newRecord = new SetCacheRecord(value);
|
||||||
cache.put(value, newRecord);
|
cache.put(value, newRecord);
|
||||||
|
@ -83,7 +84,7 @@ public class SimpleSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public synchronized SetCacheResult contains(final ByteBuffer value) {
|
public synchronized SetCacheResult contains(final ByteBuffer value) {
|
||||||
final SetCacheRecord record = cache.get(value);
|
final SetCacheRecord record = cache.get(value);
|
||||||
if ( record == null ) {
|
if (record == null) {
|
||||||
return new SetCacheResult(false, null, null);
|
return new SetCacheResult(false, null, null);
|
||||||
} else {
|
} else {
|
||||||
// We have to remove the record and add it again in order to cause the Map to stay sorted
|
// We have to remove the record and add it again in order to cause the Map to stay sorted
|
||||||
|
@ -98,7 +99,7 @@ public class SimpleSetCache implements SetCache {
|
||||||
@Override
|
@Override
|
||||||
public synchronized SetCacheResult remove(final ByteBuffer value) {
|
public synchronized SetCacheResult remove(final ByteBuffer value) {
|
||||||
final SetCacheRecord record = cache.remove(value);
|
final SetCacheRecord record = cache.remove(value);
|
||||||
if ( record == null ) {
|
if (record == null) {
|
||||||
return new SetCacheResult(false, null, null);
|
return new SetCacheResult(false, null, null);
|
||||||
} else {
|
} else {
|
||||||
inverseCacheMap.remove(record);
|
inverseCacheMap.remove(record);
|
||||||
|
|
|
@ -1,36 +1,36 @@
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<!--
|
<!--
|
||||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
contributor license agreements. See the NOTICE file distributed with
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
this work for additional information regarding copyright ownership.
|
this work for additional information regarding copyright ownership.
|
||||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
(the "License"); you may not use this file except in compliance with
|
(the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
Unless required by applicable law or agreed to in writing, software
|
Unless required by applicable law or agreed to in writing, software
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
See the License for the specific language governing permissions and
|
See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
-->
|
-->
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<title>Distributed Map Cache Client Service</title>
|
<title>Distributed Map Cache Client Service</title>
|
||||||
<link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
|
<link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
<p>
|
<p>
|
||||||
Below is an example of how to create a distributed map cache server for clients to connect to.
|
Below is an example of how to create a distributed map cache server for clients to connect to.
|
||||||
Note that the identifier in this example is <code>cache-server</code>. If you are using this template
|
Note that the identifier in this example is <code>cache-server</code>. If you are using this template
|
||||||
to create your own DistributedMapCache server, replace the values in this template with values that are
|
to create your own DistributedMapCache server, replace the values in this template with values that are
|
||||||
suitable for your system. Possible options for <code>Port</code>, <code>Maximum Cache Entries</code>,
|
suitable for your system. Possible options for <code>Port</code>, <code>Maximum Cache Entries</code>,
|
||||||
<code>Eviction Strategy</code>, <span style="font-style: italic;">SSL Context Service</span>, and
|
<code>Eviction Strategy</code>, <span style="font-style: italic;">SSL Context Service</span>, and
|
||||||
<span style="font-style: italic;">Persistence Directory</span>
|
<span style="font-style: italic;">Persistence Directory</span>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<pre>
|
<pre>
|
||||||
<?xml version="1.0" encoding="UTF-8" ?>
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
<services>
|
<services>
|
||||||
<service>
|
<service>
|
||||||
|
@ -41,6 +41,6 @@
|
||||||
<property name="Eviction Strategy">Least Recently Used</property>
|
<property name="Eviction Strategy">Least Recently Used</property>
|
||||||
</service>
|
</service>
|
||||||
</services>
|
</services>
|
||||||
</pre>
|
</pre>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
@ -374,8 +374,7 @@ public class TestServerAndClient {
|
||||||
public void testClientTermination() throws InitializationException, IOException, InterruptedException {
|
public void testClientTermination() throws InitializationException, IOException, InterruptedException {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug
|
* This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug See: https://issues.apache.org/jira/browse/NIFI-437
|
||||||
* See: https://issues.apache.org/jira/browse/NIFI-437
|
|
||||||
*/
|
*/
|
||||||
Assume.assumeFalse("testClientTermination is skipped due to build environment being OS X with JDK 1.8. See https://issues.apache.org/jira/browse/NIFI-437",
|
Assume.assumeFalse("testClientTermination is skipped due to build environment being OS X with JDK 1.8. See https://issues.apache.org/jira/browse/NIFI-437",
|
||||||
SystemUtils.IS_OS_MAC && SystemUtils.IS_JAVA_1_8);
|
SystemUtils.IS_OS_MAC && SystemUtils.IS_JAVA_1_8);
|
||||||
|
@ -509,6 +508,7 @@ public class TestServerAndClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class StringSerializer implements Serializer<String> {
|
private static class StringSerializer implements Serializer<String> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void serialize(final String value, final OutputStream output) throws SerializationException, IOException {
|
public void serialize(final String value, final OutputStream output) throws SerializationException, IOException {
|
||||||
output.write(value.getBytes(StandardCharsets.UTF_8));
|
output.write(value.getBytes(StandardCharsets.UTF_8));
|
||||||
|
@ -516,6 +516,7 @@ public class TestServerAndClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class StringDeserializer implements Deserializer<String> {
|
private static class StringDeserializer implements Deserializer<String> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String deserialize(final byte[] input) throws DeserializationException, IOException {
|
public String deserialize(final byte[] input) throws DeserializationException, IOException {
|
||||||
return (input.length == 0) ? null : new String(input, StandardCharsets.UTF_8);
|
return (input.length == 0) ? null : new String(input, StandardCharsets.UTF_8);
|
||||||
|
|
|
@ -14,24 +14,24 @@
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
-->
|
-->
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-standard-services</artifactId>
|
<artifactId>nifi-standard-services</artifactId>
|
||||||
<version>0.1.0-incubating-SNAPSHOT</version>
|
<version>0.1.0-incubating-SNAPSHOT</version>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<artifactId>nifi-http-context-map-api</artifactId>
|
<artifactId>nifi-http-context-map-api</artifactId>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-api</artifactId>
|
<artifactId>nifi-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>javax.servlet</groupId>
|
<groupId>javax.servlet</groupId>
|
||||||
<artifactId>javax.servlet-api</artifactId>
|
<artifactId>javax.servlet-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -22,20 +22,15 @@ import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
import org.apache.nifi.controller.ControllerService;
|
import org.apache.nifi.controller.ControllerService;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>
|
* <p>
|
||||||
* An interface that provides the capability of receiving an HTTP servlet request in one component
|
* An interface that provides the capability of receiving an HTTP servlet request in one component and responding to that request in another component.
|
||||||
* and responding to that request in another component.
|
|
||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* <p>
|
* <p>
|
||||||
* The intended flow is for the component receiving the HTTP request to register the request, response,
|
* The intended flow is for the component receiving the HTTP request to register the request, response, and AsyncContext with a particular identifier via the
|
||||||
* and AsyncContext with a particular identifier via the
|
* {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)} method. Another component is then able to obtain the response by providing that identifier to the
|
||||||
* {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)}
|
* {@link #getResponse(String)} method. After writing to the HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
|
||||||
* method. Another component is then able to obtain the response
|
|
||||||
* by providing that identifier to the {@link #getResponse(String)} method. After writing to the
|
|
||||||
* HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
|
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
public interface HttpContextMap extends ControllerService {
|
public interface HttpContextMap extends ControllerService {
|
||||||
|
@ -43,12 +38,12 @@ public interface HttpContextMap extends ControllerService {
|
||||||
/**
|
/**
|
||||||
* Registers an HttpServletRequest, HttpServletResponse, and the AsyncContext for a given identifier
|
* Registers an HttpServletRequest, HttpServletResponse, and the AsyncContext for a given identifier
|
||||||
*
|
*
|
||||||
* @param identifier
|
* @param identifier identifier
|
||||||
* @param request
|
* @param request request
|
||||||
* @param response
|
* @param response response
|
||||||
* @param context
|
* @param context context
|
||||||
*
|
*
|
||||||
* @return true if register is successful, false if the context map is too full because too many requests have already been received and not processed
|
* @return true if register is successful, false if the context map is too full because too many requests have already been received and not processed
|
||||||
*
|
*
|
||||||
* @throws IllegalStateException if the identifier is already registered
|
* @throws IllegalStateException if the identifier is already registered
|
||||||
*/
|
*/
|
||||||
|
@ -56,14 +51,16 @@ public interface HttpContextMap extends ControllerService {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Retrieves the HttpServletResponse for the given identifier, if it exists
|
* Retrieves the HttpServletResponse for the given identifier, if it exists
|
||||||
* @param identifier
|
*
|
||||||
|
* @param identifier identifier
|
||||||
* @return the HttpServletResponse for the given identifier, or {@code null} if it does not exist
|
* @return the HttpServletResponse for the given identifier, or {@code null} if it does not exist
|
||||||
*/
|
*/
|
||||||
HttpServletResponse getResponse(String identifier);
|
HttpServletResponse getResponse(String identifier);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Marks the HTTP request/response for the given identifier as complete
|
* Marks the HTTP request/response for the given identifier as complete
|
||||||
* @param identifier
|
*
|
||||||
|
* @param identifier identifier
|
||||||
*
|
*
|
||||||
* @throws IllegalStateException if the identifier is not registered to a valid AsyncContext
|
* @throws IllegalStateException if the identifier is not registered to a valid AsyncContext
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -24,21 +24,21 @@
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-api</artifactId>
|
<artifactId>nifi-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-processor-utils</artifactId>
|
<artifactId>nifi-processor-utils</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-http-context-map-api</artifactId>
|
<artifactId>nifi-http-context-map-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>javax.servlet</groupId>
|
<groupId>javax.servlet</groupId>
|
||||||
<artifactId>javax.servlet-api</artifactId>
|
<artifactId>javax.servlet-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -42,27 +42,28 @@ import org.apache.nifi.controller.ConfigurationContext;
|
||||||
import org.apache.nifi.processor.util.StandardValidators;
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
|
||||||
@Tags({"http", "request", "response"})
|
@Tags({"http", "request", "response"})
|
||||||
@SeeAlso(classNames={
|
@SeeAlso(classNames = {
|
||||||
"org.apache.nifi.processors.standard.HandleHttpRequest",
|
"org.apache.nifi.processors.standard.HandleHttpRequest",
|
||||||
"org.apache.nifi.processors.standard.HandleHttpResponse"})
|
"org.apache.nifi.processors.standard.HandleHttpResponse"})
|
||||||
@CapabilityDescription("Provides the ability to store and retrieve HTTP requests and responses external to a Processor, so that "
|
@CapabilityDescription("Provides the ability to store and retrieve HTTP requests and responses external to a Processor, so that "
|
||||||
+ "multiple Processors can interact with the same HTTP request.")
|
+ "multiple Processors can interact with the same HTTP request.")
|
||||||
public class StandardHttpContextMap extends AbstractControllerService implements HttpContextMap {
|
public class StandardHttpContextMap extends AbstractControllerService implements HttpContextMap {
|
||||||
|
|
||||||
public static final PropertyDescriptor MAX_OUTSTANDING_REQUESTS = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor MAX_OUTSTANDING_REQUESTS = new PropertyDescriptor.Builder()
|
||||||
.name("Maximum Outstanding Requests")
|
.name("Maximum Outstanding Requests")
|
||||||
.description("The maximum number of HTTP requests that can be outstanding at any one time. Any attempt to register an additional HTTP Request will cause an error")
|
.description("The maximum number of HTTP requests that can be outstanding at any one time. Any attempt to register an additional HTTP Request will cause an error")
|
||||||
.required(true)
|
.required(true)
|
||||||
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
|
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
|
||||||
.defaultValue("5000")
|
.defaultValue("5000")
|
||||||
.build();
|
.build();
|
||||||
public static final PropertyDescriptor REQUEST_EXPIRATION = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor REQUEST_EXPIRATION = new PropertyDescriptor.Builder()
|
||||||
.name("Request Expiration")
|
.name("Request Expiration")
|
||||||
.description("Specifies how long an HTTP Request should be left unanswered before being evicted from the cache and being responded to with a Service Unavailable status code")
|
.description("Specifies how long an HTTP Request should be left unanswered before being evicted from the cache and being responded to with a Service Unavailable status code")
|
||||||
.required(true)
|
.required(true)
|
||||||
.expressionLanguageSupported(false)
|
.expressionLanguageSupported(false)
|
||||||
.defaultValue("1 min")
|
.defaultValue("1 min")
|
||||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
private final ConcurrentMap<String, Wrapper> wrapperMap = new ConcurrentHashMap<>();
|
private final ConcurrentMap<String, Wrapper> wrapperMap = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
@ -90,7 +91,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
||||||
|
|
||||||
@OnDisabled
|
@OnDisabled
|
||||||
public void cleanup() {
|
public void cleanup() {
|
||||||
if ( executor != null ) {
|
if (executor != null) {
|
||||||
executor.shutdown();
|
executor.shutdown();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -99,22 +100,22 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
||||||
public boolean register(final String identifier, final HttpServletRequest request, final HttpServletResponse response, final AsyncContext context) {
|
public boolean register(final String identifier, final HttpServletRequest request, final HttpServletResponse response, final AsyncContext context) {
|
||||||
// fail if there are too many already. Maybe add a configuration property for how many
|
// fail if there are too many already. Maybe add a configuration property for how many
|
||||||
// outstanding, with a default of say 5000
|
// outstanding, with a default of say 5000
|
||||||
if ( wrapperMap.size() >= maxSize ) {
|
if (wrapperMap.size() >= maxSize) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
final Wrapper wrapper = new Wrapper(request, response, context);
|
final Wrapper wrapper = new Wrapper(request, response, context);
|
||||||
final Wrapper existing = wrapperMap.putIfAbsent(identifier, wrapper);
|
final Wrapper existing = wrapperMap.putIfAbsent(identifier, wrapper);
|
||||||
if ( existing != null ) {
|
if (existing != null) {
|
||||||
throw new IllegalStateException("HTTP Request already registered with identifier " + identifier);
|
throw new IllegalStateException("HTTP Request already registered with identifier " + identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public HttpServletResponse getResponse(final String identifier) {
|
public HttpServletResponse getResponse(final String identifier) {
|
||||||
final Wrapper wrapper = wrapperMap.get(identifier);
|
final Wrapper wrapper = wrapperMap.get(identifier);
|
||||||
if ( wrapper == null ) {
|
if (wrapper == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -124,7 +125,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
||||||
@Override
|
@Override
|
||||||
public void complete(final String identifier) {
|
public void complete(final String identifier) {
|
||||||
final Wrapper wrapper = wrapperMap.remove(identifier);
|
final Wrapper wrapper = wrapperMap.remove(identifier);
|
||||||
if ( wrapper == null ) {
|
if (wrapper == null) {
|
||||||
throw new IllegalStateException("No HTTP Request registered with identifier " + identifier);
|
throw new IllegalStateException("No HTTP Request registered with identifier " + identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -132,6 +133,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class Wrapper {
|
private static class Wrapper {
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
private final HttpServletRequest request;
|
private final HttpServletRequest request;
|
||||||
private final HttpServletResponse response;
|
private final HttpServletResponse response;
|
||||||
|
@ -158,15 +160,16 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
||||||
}
|
}
|
||||||
|
|
||||||
private class CleanupExpiredRequests implements Runnable {
|
private class CleanupExpiredRequests implements Runnable {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
final long now = System.nanoTime();
|
final long now = System.nanoTime();
|
||||||
final long threshold = now - maxRequestNanos;
|
final long threshold = now - maxRequestNanos;
|
||||||
|
|
||||||
final Iterator<Map.Entry<String, Wrapper>> itr = wrapperMap.entrySet().iterator();
|
final Iterator<Map.Entry<String, Wrapper>> itr = wrapperMap.entrySet().iterator();
|
||||||
while ( itr.hasNext() ) {
|
while (itr.hasNext()) {
|
||||||
final Map.Entry<String, Wrapper> entry = itr.next();
|
final Map.Entry<String, Wrapper> entry = itr.next();
|
||||||
if ( entry.getValue().getNanoTimeAdded() < threshold ) {
|
if (entry.getValue().getNanoTimeAdded() < threshold) {
|
||||||
itr.remove();
|
itr.remove();
|
||||||
|
|
||||||
// send SERVICE_UNAVAILABLE
|
// send SERVICE_UNAVAILABLE
|
||||||
|
|
|
@ -22,15 +22,15 @@
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
<h2>Description:</h2>
|
<h2>Description:</h2>
|
||||||
<p>
|
<p>
|
||||||
This is the standard implementation of the SSL Context Map. This service is used to provide
|
This is the standard implementation of the SSL Context Map. This service is used to provide
|
||||||
coordination between
|
coordination between
|
||||||
<a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a>
|
<a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a>
|
||||||
and
|
and
|
||||||
<a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a>
|
<a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a>
|
||||||
Processors.
|
Processors.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<!-- Service Documentation ================================================== -->
|
<!-- Service Documentation ================================================== -->
|
||||||
<h2>Configuring the HTTP Context Map:</h2>
|
<h2>Configuring the HTTP Context Map:</h2>
|
||||||
|
@ -40,9 +40,9 @@
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p>
|
<p>
|
||||||
This controller service exposes a single property named <code>Maximum Outstanding Requests</code>.
|
This controller service exposes a single property named <code>Maximum Outstanding Requests</code>.
|
||||||
This property determines the maximum number of HTTP requests that can be outstanding at any one time.
|
This property determines the maximum number of HTTP requests that can be outstanding at any one time.
|
||||||
Any attempt to register an additional HTTP Request will cause an error. The default value is 5000.
|
Any attempt to register an additional HTTP Request will cause an error. The default value is 5000.
|
||||||
Below is an example of the template for a StandardHttpContextMap controller service.
|
Below is an example of the template for a StandardHttpContextMap controller service.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
|
@ -58,10 +58,10 @@
|
||||||
</pre>
|
</pre>
|
||||||
|
|
||||||
<p>
|
<p>
|
||||||
<strong>See Also:</strong><br />
|
<strong>See Also:</strong><br />
|
||||||
<a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a><br />
|
<a href="../org.apache.nifi.processors.standard.HandleHttpRequest/index.html">HandleHttpRequest</a><br />
|
||||||
<a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a><br />
|
<a href="../org.apache.nifi.processors.standard.HandleHttpResponse/index.html">HandleHttpResponse</a><br />
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
@ -238,7 +238,6 @@ public class StandardSSLContextService extends AbstractControllerService impleme
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SSLContext createSSLContext(final ClientAuth clientAuth) throws ProcessException {
|
public SSLContext createSSLContext(final ClientAuth clientAuth) throws ProcessException {
|
||||||
try {
|
try {
|
||||||
|
|
Loading…
Reference in New Issue