mirror of https://github.com/apache/nifi.git
NIFI-271
This commit is contained in:
parent
6a706458d0
commit
9a3b6bed62
|
@ -299,6 +299,7 @@ public class DistributedMapCacheClientService extends AbstractControllerService
|
|||
}
|
||||
|
||||
private static interface CommsAction<T> {
|
||||
|
||||
T execute(CommsSession commsSession) throws IOException;
|
||||
}
|
||||
|
||||
|
|
|
@ -30,6 +30,7 @@ import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelInputStream;
|
|||
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelOutputStream;
|
||||
|
||||
public class SSLCommsSession implements CommsSession {
|
||||
|
||||
private final SSLSocketChannel sslSocketChannel;
|
||||
private final SSLContext sslContext;
|
||||
private final String hostname;
|
||||
|
@ -94,10 +95,12 @@ public class SSLCommsSession implements CommsSession {
|
|||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SSLContext getSSLContext() {
|
||||
return sslContext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTimeout(final TimeUnit timeUnit) {
|
||||
return timeUnit.convert(sslSocketChannel.getTimeout(), TimeUnit.MILLISECONDS);
|
||||
|
|
|
@ -33,6 +33,7 @@ import org.apache.nifi.remote.io.socket.SocketChannelInputStream;
|
|||
import org.apache.nifi.remote.io.socket.SocketChannelOutputStream;
|
||||
|
||||
public class StandardCommsSession implements CommsSession {
|
||||
|
||||
private final SocketChannel socketChannel;
|
||||
private final String hostname;
|
||||
private final int port;
|
||||
|
|
|
@ -51,7 +51,8 @@ public abstract class AbstractCacheServer implements CacheServer {
|
|||
private final int port;
|
||||
private final SSLContext sslContext;
|
||||
protected volatile boolean stopped = false;
|
||||
private final Set<Thread> processInputThreads = new CopyOnWriteArraySet<>();;
|
||||
private final Set<Thread> processInputThreads = new CopyOnWriteArraySet<>();
|
||||
;
|
||||
|
||||
private volatile ServerSocketChannel serverSocketChannel;
|
||||
|
||||
|
@ -189,11 +190,11 @@ public abstract class AbstractCacheServer implements CacheServer {
|
|||
/**
|
||||
* Listens for incoming data and communicates with remote peer
|
||||
*
|
||||
* @param in
|
||||
* @param out
|
||||
* @param version
|
||||
* @param in in
|
||||
* @param out out
|
||||
* @param version version
|
||||
* @return <code>true</code> if communications should continue, <code>false</code> otherwise
|
||||
* @throws IOException
|
||||
* @throws IOException ex
|
||||
*/
|
||||
protected abstract boolean listen(InputStream in, OutputStream out, int version) throws IOException;
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ import java.io.IOException;
|
|||
public interface CacheServer {
|
||||
|
||||
void start() throws IOException;
|
||||
|
||||
void stop() throws IOException;
|
||||
|
||||
}
|
||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.nifi.processor.util.StandardValidators;
|
|||
import org.apache.nifi.ssl.SSLContextService;
|
||||
|
||||
public abstract class DistributedCacheServer extends AbstractControllerService {
|
||||
|
||||
public static final String EVICTION_STRATEGY_LFU = "Least Frequently Used";
|
||||
public static final String EVICTION_STRATEGY_LRU = "Least Recently Used";
|
||||
public static final String EVICTION_STRATEGY_FIFO = "First In, First Out";
|
||||
|
|
|
@ -25,6 +25,7 @@ import org.apache.nifi.annotation.documentation.Tags;
|
|||
import org.apache.nifi.controller.ConfigurationContext;
|
||||
import org.apache.nifi.ssl.SSLContextService;
|
||||
import org.apache.nifi.ssl.SSLContextService.ClientAuth;
|
||||
|
||||
@Tags({"distributed", "set", "distinct", "cache", "server"})
|
||||
@CapabilityDescription("Provides a set (collection of unique values) cache that can be accessed over a socket. "
|
||||
+ "Interaction with this service is typically accomplished via a DistributedSetCacheClient service.")
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.nifi.distributed.cache.server;
|
|||
import java.util.Comparator;
|
||||
|
||||
public enum EvictionPolicy {
|
||||
|
||||
LFU(new LFUComparator()),
|
||||
LRU(new LRUComparator()),
|
||||
FIFO(new FIFOComparator());
|
||||
|
@ -34,6 +35,7 @@ public enum EvictionPolicy {
|
|||
}
|
||||
|
||||
public static class LFUComparator implements Comparator<CacheRecord> {
|
||||
|
||||
@Override
|
||||
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
||||
if (o1.equals(o2)) {
|
||||
|
@ -47,6 +49,7 @@ public enum EvictionPolicy {
|
|||
}
|
||||
|
||||
public static class LRUComparator implements Comparator<CacheRecord> {
|
||||
|
||||
@Override
|
||||
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
||||
if (o1.equals(o2)) {
|
||||
|
@ -59,6 +62,7 @@ public enum EvictionPolicy {
|
|||
}
|
||||
|
||||
public static class FIFOComparator implements Comparator<CacheRecord> {
|
||||
|
||||
@Override
|
||||
public int compare(final CacheRecord o1, final CacheRecord o2) {
|
||||
if (o1.equals(o2)) {
|
||||
|
|
|
@ -97,8 +97,9 @@ public class SetCacheServer extends AbstractCacheServer {
|
|||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
if (!stopped)
|
||||
if (!stopped) {
|
||||
stop();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -22,8 +22,12 @@ import java.nio.ByteBuffer;
|
|||
public interface MapCache {
|
||||
|
||||
MapPutResult putIfAbsent(ByteBuffer key, ByteBuffer value) throws IOException;
|
||||
|
||||
boolean containsKey(ByteBuffer key) throws IOException;
|
||||
|
||||
ByteBuffer get(ByteBuffer key) throws IOException;
|
||||
|
||||
ByteBuffer remove(ByteBuffer key) throws IOException;
|
||||
|
||||
void shutdown() throws IOException;
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
|
|||
import org.apache.nifi.distributed.cache.server.CacheRecord;
|
||||
|
||||
public class MapCacheRecord extends CacheRecord {
|
||||
|
||||
private final ByteBuffer key;
|
||||
private final ByteBuffer value;
|
||||
|
||||
|
|
|
@ -131,9 +131,10 @@ public class MapCacheServer extends AbstractCacheServer {
|
|||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
if (!stopped)
|
||||
if (!stopped) {
|
||||
stop();
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] readValue(final DataInputStream dis) throws IOException {
|
||||
final int numBytes = dis.readInt();
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.nifi.distributed.cache.server.map;
|
|||
import java.nio.ByteBuffer;
|
||||
|
||||
public class MapPutResult {
|
||||
|
||||
private final boolean successful;
|
||||
private final ByteBuffer key, value;
|
||||
private final ByteBuffer existingValue;
|
||||
|
|
|
@ -106,14 +106,13 @@ public class PersistentMapCache implements MapCache {
|
|||
return removeResult;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void shutdown() throws IOException {
|
||||
wali.shutdown();
|
||||
}
|
||||
|
||||
|
||||
private static class MapWaliRecord {
|
||||
|
||||
private final UpdateType updateType;
|
||||
private final ByteBuffer key;
|
||||
private final ByteBuffer value;
|
||||
|
|
|
@ -33,6 +33,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class SimpleMapCache implements MapCache {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SimpleMapCache.class);
|
||||
|
||||
private final Map<ByteBuffer, MapCacheRecord> cache = new HashMap<>();
|
||||
|
|
|
@ -108,6 +108,7 @@ public class PersistentSetCache implements SetCache {
|
|||
}
|
||||
|
||||
private static class SetRecord {
|
||||
|
||||
private final UpdateType updateType;
|
||||
private final ByteBuffer value;
|
||||
|
||||
|
|
|
@ -22,8 +22,11 @@ import java.nio.ByteBuffer;
|
|||
public interface SetCache {
|
||||
|
||||
SetCacheResult remove(ByteBuffer value) throws IOException;
|
||||
|
||||
SetCacheResult addIfAbsent(ByteBuffer value) throws IOException;
|
||||
|
||||
SetCacheResult contains(ByteBuffer value) throws IOException;
|
||||
|
||||
void shutdown() throws IOException;
|
||||
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
|
|||
import org.apache.nifi.distributed.cache.server.CacheRecord;
|
||||
|
||||
public class SetCacheRecord extends CacheRecord {
|
||||
|
||||
private final ByteBuffer value;
|
||||
|
||||
public SetCacheRecord(final ByteBuffer value) {
|
||||
|
|
|
@ -16,9 +16,8 @@
|
|||
*/
|
||||
package org.apache.nifi.distributed.cache.server.set;
|
||||
|
||||
|
||||
|
||||
public class SetCacheResult {
|
||||
|
||||
private final boolean result;
|
||||
private final SetCacheRecord stats;
|
||||
private final SetCacheRecord evictedRecord;
|
||||
|
|
|
@ -30,6 +30,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class SimpleSetCache implements SetCache {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SimpleSetCache.class);
|
||||
|
||||
private final Map<ByteBuffer, SetCacheRecord> cache = new HashMap<>();
|
||||
|
|
|
@ -374,8 +374,7 @@ public class TestServerAndClient {
|
|||
public void testClientTermination() throws InitializationException, IOException, InterruptedException {
|
||||
|
||||
/**
|
||||
* This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug
|
||||
* See: https://issues.apache.org/jira/browse/NIFI-437
|
||||
* This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug See: https://issues.apache.org/jira/browse/NIFI-437
|
||||
*/
|
||||
Assume.assumeFalse("testClientTermination is skipped due to build environment being OS X with JDK 1.8. See https://issues.apache.org/jira/browse/NIFI-437",
|
||||
SystemUtils.IS_OS_MAC && SystemUtils.IS_JAVA_1_8);
|
||||
|
@ -509,6 +508,7 @@ public class TestServerAndClient {
|
|||
}
|
||||
|
||||
private static class StringSerializer implements Serializer<String> {
|
||||
|
||||
@Override
|
||||
public void serialize(final String value, final OutputStream output) throws SerializationException, IOException {
|
||||
output.write(value.getBytes(StandardCharsets.UTF_8));
|
||||
|
@ -516,6 +516,7 @@ public class TestServerAndClient {
|
|||
}
|
||||
|
||||
private static class StringDeserializer implements Deserializer<String> {
|
||||
|
||||
@Override
|
||||
public String deserialize(final byte[] input) throws DeserializationException, IOException {
|
||||
return (input.length == 0) ? null : new String(input, StandardCharsets.UTF_8);
|
||||
|
|
|
@ -22,20 +22,15 @@ import javax.servlet.http.HttpServletResponse;
|
|||
|
||||
import org.apache.nifi.controller.ControllerService;
|
||||
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* An interface that provides the capability of receiving an HTTP servlet request in one component
|
||||
* and responding to that request in another component.
|
||||
* An interface that provides the capability of receiving an HTTP servlet request in one component and responding to that request in another component.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The intended flow is for the component receiving the HTTP request to register the request, response,
|
||||
* and AsyncContext with a particular identifier via the
|
||||
* {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)}
|
||||
* method. Another component is then able to obtain the response
|
||||
* by providing that identifier to the {@link #getResponse(String)} method. After writing to the
|
||||
* HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
|
||||
* The intended flow is for the component receiving the HTTP request to register the request, response, and AsyncContext with a particular identifier via the
|
||||
* {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)} method. Another component is then able to obtain the response by providing that identifier to the
|
||||
* {@link #getResponse(String)} method. After writing to the HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
|
||||
* </p>
|
||||
*/
|
||||
public interface HttpContextMap extends ControllerService {
|
||||
|
@ -43,10 +38,10 @@ public interface HttpContextMap extends ControllerService {
|
|||
/**
|
||||
* Registers an HttpServletRequest, HttpServletResponse, and the AsyncContext for a given identifier
|
||||
*
|
||||
* @param identifier
|
||||
* @param request
|
||||
* @param response
|
||||
* @param context
|
||||
* @param identifier identifier
|
||||
* @param request request
|
||||
* @param response response
|
||||
* @param context context
|
||||
*
|
||||
* @return true if register is successful, false if the context map is too full because too many requests have already been received and not processed
|
||||
*
|
||||
|
@ -56,14 +51,16 @@ public interface HttpContextMap extends ControllerService {
|
|||
|
||||
/**
|
||||
* Retrieves the HttpServletResponse for the given identifier, if it exists
|
||||
* @param identifier
|
||||
*
|
||||
* @param identifier identifier
|
||||
* @return the HttpServletResponse for the given identifier, or {@code null} if it does not exist
|
||||
*/
|
||||
HttpServletResponse getResponse(String identifier);
|
||||
|
||||
/**
|
||||
* Marks the HTTP request/response for the given identifier as complete
|
||||
* @param identifier
|
||||
*
|
||||
* @param identifier identifier
|
||||
*
|
||||
* @throws IllegalStateException if the identifier is not registered to a valid AsyncContext
|
||||
*/
|
||||
|
|
|
@ -48,6 +48,7 @@ import org.apache.nifi.processor.util.StandardValidators;
|
|||
@CapabilityDescription("Provides the ability to store and retrieve HTTP requests and responses external to a Processor, so that "
|
||||
+ "multiple Processors can interact with the same HTTP request.")
|
||||
public class StandardHttpContextMap extends AbstractControllerService implements HttpContextMap {
|
||||
|
||||
public static final PropertyDescriptor MAX_OUTSTANDING_REQUESTS = new PropertyDescriptor.Builder()
|
||||
.name("Maximum Outstanding Requests")
|
||||
.description("The maximum number of HTTP requests that can be outstanding at any one time. Any attempt to register an additional HTTP Request will cause an error")
|
||||
|
@ -132,6 +133,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
|||
}
|
||||
|
||||
private static class Wrapper {
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private final HttpServletRequest request;
|
||||
private final HttpServletResponse response;
|
||||
|
@ -158,6 +160,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
|
|||
}
|
||||
|
||||
private class CleanupExpiredRequests implements Runnable {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
final long now = System.nanoTime();
|
||||
|
|
|
@ -238,7 +238,6 @@ public class StandardSSLContextService extends AbstractControllerService impleme
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public SSLContext createSSLContext(final ClientAuth clientAuth) throws ProcessException {
|
||||
try {
|
||||
|
|
Loading…
Reference in New Issue