This commit is contained in:
joewitt 2015-04-27 13:43:35 -04:00
parent 6a706458d0
commit 9a3b6bed62
34 changed files with 460 additions and 434 deletions

View File

@ -299,6 +299,7 @@ public class DistributedMapCacheClientService extends AbstractControllerService
}
private static interface CommsAction<T> {
T execute(CommsSession commsSession) throws IOException;
}

View File

@ -30,6 +30,7 @@ import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelInputStream;
import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelOutputStream;
public class SSLCommsSession implements CommsSession {
private final SSLSocketChannel sslSocketChannel;
private final SSLContext sslContext;
private final String hostname;
@ -94,10 +95,12 @@ public class SSLCommsSession implements CommsSession {
public int getPort() {
return port;
}
@Override
public SSLContext getSSLContext() {
return sslContext;
}
@Override
public long getTimeout(final TimeUnit timeUnit) {
return timeUnit.convert(sslSocketChannel.getTimeout(), TimeUnit.MILLISECONDS);

View File

@ -33,6 +33,7 @@ import org.apache.nifi.remote.io.socket.SocketChannelInputStream;
import org.apache.nifi.remote.io.socket.SocketChannelOutputStream;
public class StandardCommsSession implements CommsSession {
private final SocketChannel socketChannel;
private final String hostname;
private final int port;

View File

@ -51,7 +51,8 @@ public abstract class AbstractCacheServer implements CacheServer {
private final int port;
private final SSLContext sslContext;
protected volatile boolean stopped = false;
private final Set<Thread> processInputThreads = new CopyOnWriteArraySet<>();;
private final Set<Thread> processInputThreads = new CopyOnWriteArraySet<>();
;
private volatile ServerSocketChannel serverSocketChannel;
@ -189,11 +190,11 @@ public abstract class AbstractCacheServer implements CacheServer {
/**
* Listens for incoming data and communicates with remote peer
*
* @param in
* @param out
* @param version
* @param in in
* @param out out
* @param version version
* @return <code>true</code> if communications should continue, <code>false</code> otherwise
* @throws IOException
* @throws IOException ex
*/
protected abstract boolean listen(InputStream in, OutputStream out, int version) throws IOException;
}

View File

@ -21,6 +21,7 @@ import java.io.IOException;
public interface CacheServer {
void start() throws IOException;
void stop() throws IOException;
}

View File

@ -29,6 +29,7 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.ssl.SSLContextService;
public abstract class DistributedCacheServer extends AbstractControllerService {
public static final String EVICTION_STRATEGY_LFU = "Least Frequently Used";
public static final String EVICTION_STRATEGY_LRU = "Least Recently Used";
public static final String EVICTION_STRATEGY_FIFO = "First In, First Out";

View File

@ -25,6 +25,7 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.ssl.SSLContextService.ClientAuth;
@Tags({"distributed", "set", "distinct", "cache", "server"})
@CapabilityDescription("Provides a set (collection of unique values) cache that can be accessed over a socket. "
+ "Interaction with this service is typically accomplished via a DistributedSetCacheClient service.")

View File

@ -19,6 +19,7 @@ package org.apache.nifi.distributed.cache.server;
import java.util.Comparator;
public enum EvictionPolicy {
LFU(new LFUComparator()),
LRU(new LRUComparator()),
FIFO(new FIFOComparator());
@ -34,6 +35,7 @@ public enum EvictionPolicy {
}
public static class LFUComparator implements Comparator<CacheRecord> {
@Override
public int compare(final CacheRecord o1, final CacheRecord o2) {
if (o1.equals(o2)) {
@ -47,6 +49,7 @@ public enum EvictionPolicy {
}
public static class LRUComparator implements Comparator<CacheRecord> {
@Override
public int compare(final CacheRecord o1, final CacheRecord o2) {
if (o1.equals(o2)) {
@ -59,6 +62,7 @@ public enum EvictionPolicy {
}
public static class FIFOComparator implements Comparator<CacheRecord> {
@Override
public int compare(final CacheRecord o1, final CacheRecord o2) {
if (o1.equals(o2)) {

View File

@ -97,8 +97,9 @@ public class SetCacheServer extends AbstractCacheServer {
@Override
protected void finalize() throws Throwable {
if (!stopped)
if (!stopped) {
stop();
}
}
}

View File

@ -22,8 +22,12 @@ import java.nio.ByteBuffer;
public interface MapCache {
MapPutResult putIfAbsent(ByteBuffer key, ByteBuffer value) throws IOException;
boolean containsKey(ByteBuffer key) throws IOException;
ByteBuffer get(ByteBuffer key) throws IOException;
ByteBuffer remove(ByteBuffer key) throws IOException;
void shutdown() throws IOException;
}

View File

@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import org.apache.nifi.distributed.cache.server.CacheRecord;
public class MapCacheRecord extends CacheRecord {
private final ByteBuffer key;
private final ByteBuffer value;

View File

@ -131,9 +131,10 @@ public class MapCacheServer extends AbstractCacheServer {
@Override
protected void finalize() throws Throwable {
if (!stopped)
if (!stopped) {
stop();
}
}
private byte[] readValue(final DataInputStream dis) throws IOException {
final int numBytes = dis.readInt();

View File

@ -19,6 +19,7 @@ package org.apache.nifi.distributed.cache.server.map;
import java.nio.ByteBuffer;
public class MapPutResult {
private final boolean successful;
private final ByteBuffer key, value;
private final ByteBuffer existingValue;

View File

@ -106,14 +106,13 @@ public class PersistentMapCache implements MapCache {
return removeResult;
}
@Override
public void shutdown() throws IOException {
wali.shutdown();
}
private static class MapWaliRecord {
private final UpdateType updateType;
private final ByteBuffer key;
private final ByteBuffer value;

View File

@ -33,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SimpleMapCache implements MapCache {
private static final Logger logger = LoggerFactory.getLogger(SimpleMapCache.class);
private final Map<ByteBuffer, MapCacheRecord> cache = new HashMap<>();

View File

@ -108,6 +108,7 @@ public class PersistentSetCache implements SetCache {
}
private static class SetRecord {
private final UpdateType updateType;
private final ByteBuffer value;

View File

@ -22,8 +22,11 @@ import java.nio.ByteBuffer;
public interface SetCache {
SetCacheResult remove(ByteBuffer value) throws IOException;
SetCacheResult addIfAbsent(ByteBuffer value) throws IOException;
SetCacheResult contains(ByteBuffer value) throws IOException;
void shutdown() throws IOException;
}

View File

@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import org.apache.nifi.distributed.cache.server.CacheRecord;
public class SetCacheRecord extends CacheRecord {
private final ByteBuffer value;
public SetCacheRecord(final ByteBuffer value) {

View File

@ -16,9 +16,8 @@
*/
package org.apache.nifi.distributed.cache.server.set;
public class SetCacheResult {
private final boolean result;
private final SetCacheRecord stats;
private final SetCacheRecord evictedRecord;

View File

@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SimpleSetCache implements SetCache {
private static final Logger logger = LoggerFactory.getLogger(SimpleSetCache.class);
private final Map<ByteBuffer, SetCacheRecord> cache = new HashMap<>();

View File

@ -374,8 +374,7 @@ public class TestServerAndClient {
public void testClientTermination() throws InitializationException, IOException, InterruptedException {
/**
* This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug
* See: https://issues.apache.org/jira/browse/NIFI-437
* This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug See: https://issues.apache.org/jira/browse/NIFI-437
*/
Assume.assumeFalse("testClientTermination is skipped due to build environment being OS X with JDK 1.8. See https://issues.apache.org/jira/browse/NIFI-437",
SystemUtils.IS_OS_MAC && SystemUtils.IS_JAVA_1_8);
@ -509,6 +508,7 @@ public class TestServerAndClient {
}
private static class StringSerializer implements Serializer<String> {
@Override
public void serialize(final String value, final OutputStream output) throws SerializationException, IOException {
output.write(value.getBytes(StandardCharsets.UTF_8));
@ -516,6 +516,7 @@ public class TestServerAndClient {
}
private static class StringDeserializer implements Deserializer<String> {
@Override
public String deserialize(final byte[] input) throws DeserializationException, IOException {
return (input.length == 0) ? null : new String(input, StandardCharsets.UTF_8);

View File

@ -22,20 +22,15 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.nifi.controller.ControllerService;
/**
* <p>
* An interface that provides the capability of receiving an HTTP servlet request in one component
* and responding to that request in another component.
* An interface that provides the capability of receiving an HTTP servlet request in one component and responding to that request in another component.
* </p>
*
* <p>
* The intended flow is for the component receiving the HTTP request to register the request, response,
* and AsyncContext with a particular identifier via the
* {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)}
* method. Another component is then able to obtain the response
* by providing that identifier to the {@link #getResponse(String)} method. After writing to the
* HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
* The intended flow is for the component receiving the HTTP request to register the request, response, and AsyncContext with a particular identifier via the
* {@link #register(String, HttpServletRequest, HttpServletResponse, AsyncContext)} method. Another component is then able to obtain the response by providing that identifier to the
* {@link #getResponse(String)} method. After writing to the HttpServletResponse, the transaction is to then be completed via the {@link #complete(String)} method.
* </p>
*/
public interface HttpContextMap extends ControllerService {
@ -43,10 +38,10 @@ public interface HttpContextMap extends ControllerService {
/**
* Registers an HttpServletRequest, HttpServletResponse, and the AsyncContext for a given identifier
*
* @param identifier
* @param request
* @param response
* @param context
* @param identifier identifier
* @param request request
* @param response response
* @param context context
*
* @return true if register is successful, false if the context map is too full because too many requests have already been received and not processed
*
@ -56,14 +51,16 @@ public interface HttpContextMap extends ControllerService {
/**
* Retrieves the HttpServletResponse for the given identifier, if it exists
* @param identifier
*
* @param identifier identifier
* @return the HttpServletResponse for the given identifier, or {@code null} if it does not exist
*/
HttpServletResponse getResponse(String identifier);
/**
* Marks the HTTP request/response for the given identifier as complete
* @param identifier
*
* @param identifier identifier
*
* @throws IllegalStateException if the identifier is not registered to a valid AsyncContext
*/

View File

@ -48,6 +48,7 @@ import org.apache.nifi.processor.util.StandardValidators;
@CapabilityDescription("Provides the ability to store and retrieve HTTP requests and responses external to a Processor, so that "
+ "multiple Processors can interact with the same HTTP request.")
public class StandardHttpContextMap extends AbstractControllerService implements HttpContextMap {
public static final PropertyDescriptor MAX_OUTSTANDING_REQUESTS = new PropertyDescriptor.Builder()
.name("Maximum Outstanding Requests")
.description("The maximum number of HTTP requests that can be outstanding at any one time. Any attempt to register an additional HTTP Request will cause an error")
@ -132,6 +133,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
}
private static class Wrapper {
@SuppressWarnings("unused")
private final HttpServletRequest request;
private final HttpServletResponse response;
@ -158,6 +160,7 @@ public class StandardHttpContextMap extends AbstractControllerService implements
}
private class CleanupExpiredRequests implements Runnable {
@Override
public void run() {
final long now = System.nanoTime();

View File

@ -238,7 +238,6 @@ public class StandardSSLContextService extends AbstractControllerService impleme
}
}
@Override
public SSLContext createSSLContext(final ClientAuth clientAuth) throws ProcessException {
try {