From d4f0c1d048f0539a226b4b5ce33b9d172402f50e Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Fri, 19 May 2017 09:37:34 -0400 Subject: [PATCH] NIFI-3945 This closes #1829. Adding documentaion about security protocols to Kafka 0.10 processors Signed-off-by: joewitt --- .../additionalDetails.html | 142 +++++++++++++++++ .../additionalDetails.html | 109 +++++++++++++ .../additionalDetails.html | 143 ++++++++++++++++++ .../additionalDetails.html | 108 +++++++++++++ 4 files changed, 502 insertions(+) create mode 100644 nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_0_10/additionalDetails.html create mode 100644 nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_0_10/additionalDetails.html diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_0_10/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_0_10/additionalDetails.html new file mode 100644 index 0000000000..03a7482b2a --- /dev/null +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_0_10/additionalDetails.html @@ -0,0 +1,142 @@ + + + + + + ConsumeKafka + + + + + +

Description:

+

+ This Processor polls Apache Kafka + for data using KafkaConsumer API available with Kafka 0.10.x. When a message is received + from Kafka, the message will be deserialized using the configured Record Reader, and then + written to a FlowFile by serializing the message with the configured Record Writer. +

+

Security Configuration:

+

+ The Security Protocol property allows the user to specify the protocol for communicating + with the Kafka broker. The following sections describe each of the protocols in further detail. +

+

PLAINTEXT

+

+ This option provides an unsecured connection to the broker, with no client authentication and no encryption. + In order to use this option the broker must be configured with a listener of the form: + 

+    PLAINTEXT://host.name:port
+
+

+

SSL

+

+ This option provides an encrypted connection to the broker, with optional client authentication. In order + to use this option the broker must be configured with a listener of the form: + 

+    SSL://host.name:port
+
+ In addition, the processor must have an SSL Context Service selected. +

+

+ If the broker specifies ssl.client.auth=none, or does not specify ssl.client.auth, then the client will + not be required to present a certificate. In this case, the SSL Context Service selected may specify only + a truststore containing the public key of the certificate authority used to sign the broker's key. +

+

+ If the broker specifies ssl.client.auth=required then the client will be required to present a certificate. + In this case, the SSL Context Service must also specify a keystore containing a client key, in addition to + a truststore as described above. +

+

SASL_PLAINTEXT

+

+ This option uses SASL with a PLAINTEXT transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_PLAINTEXT://host.name:port
+
+ In addition, the Kerberos Service Name must be specified in the processor. +

+

SASL_PLAINTEXT - GSSAPI

+

+ If the SASL mechanism is GSSAPI, then the client must provide a JAAS configuration to authenticate. The + JAAS configuration can be provided by specifying the java.security.auth.login.config system property in + NiFi's bootstrap.conf, such as: + 

+    java.arg.16=-Djava.security.auth.login.config=/path/to/kafka_client_jaas.conf
+
+

+

+ An example of the JAAS config file would be the following: + 

+    KafkaClient {
+        com.sun.security.auth.module.Krb5LoginModule required
+        useKeyTab=true
+        storeKey=true
+        keyTab="/path/to/nifi.keytab"
+        serviceName="kafka"
+        principal="nifi@YOURREALM.COM";
+    };
+
+ NOTE: The serviceName in the JAAS file must match the Kerberos Service Name in the processor. +

+

+ Alternatively, starting with Apache NiFi 1.2.0 which uses the Kafka 0.10.2 client, the JAAS + configuration when using GSSAPI can be provided by specifying the Kerberos Principal and Kerberos Keytab + directly in the processor properties. This will dynamically create a JAAS configuration like above, and + will take precedence over the java.security.auth.login.config system property. +

+

SASL_PLAINTEXT - PLAIN

+

+ If the SASL mechanism is PLAIN, then client must provide a JAAS configuration to authenticate, but + the JAAS configuration must use Kafka's PlainLoginModule. An example of the JAAS config file would + be the following: + 

+    KafkaClient {
+      org.apache.kafka.common.security.plain.PlainLoginModule required
+      username="nifi"
+      password="nifi-password";
+    };
+
+

+

+ NOTE: It is not recommended to use a SASL mechanism of PLAIN with SASL_PLAINTEXT, as it would transmit + the username and password unencrypted. +

+

+ NOTE: Using the PlainLoginModule will cause it be registered in the JVM's static list of Providers, making + it visible to components in other NARs that may access the providers. There is currently a known issue + where Kafka processors using the PlainLoginModule will cause HDFS processors with Keberos to no longer work. +

+

SASL_SSL

+

+ This option uses SASL with an SSL/TLS transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_SSL://host.name:port
+
+

+

+ See the SASL_PLAINTEXT section for a description of how to provide the proper JAAS configuration + depending on the SASL mechanism (GSSAPI or PLAIN). +

+

+ See the SSL section for a description of how to configure the SSL Context Service based on the + ssl.client.auth property. +

+ + + diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_0_10/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_0_10/additionalDetails.html index 4c590de874..a8a86eb835 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_0_10/additionalDetails.html +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_0_10/additionalDetails.html @@ -29,5 +29,114 @@ from Kafka, this Processor emits a FlowFile where the content of the FlowFile is the value of the Kafka message.

+

Security Configuration:

+

+ The Security Protocol property allows the user to specify the protocol for communicating + with the Kafka broker. The following sections describe each of the protocols in further detail. +

+

PLAINTEXT

+

+ This option provides an unsecured connection to the broker, with no client authentication and no encryption. + In order to use this option the broker must be configured with a listener of the form: + 

+    PLAINTEXT://host.name:port
+
+

+

SSL

+

+ This option provides an encrypted connection to the broker, with optional client authentication. In order + to use this option the broker must be configured with a listener of the form: + 

+    SSL://host.name:port
+
+ In addition, the processor must have an SSL Context Service selected. +

+

+ If the broker specifies ssl.client.auth=none, or does not specify ssl.client.auth, then the client will + not be required to present a certificate. In this case, the SSL Context Service selected may specify only + a truststore containing the public key of the certificate authority used to sign the broker's key. +

+

+ If the broker specifies ssl.client.auth=required then the client will be required to present a certificate. + In this case, the SSL Context Service must also specify a keystore containing a client key, in addition to + a truststore as described above. +

+

SASL_PLAINTEXT

+

+ This option uses SASL with a PLAINTEXT transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_PLAINTEXT://host.name:port
+
+ In addition, the Kerberos Service Name must be specified in the processor. +

+

SASL_PLAINTEXT - GSSAPI

+

+ If the SASL mechanism is GSSAPI, then the client must provide a JAAS configuration to authenticate. The + JAAS configuration can be provided by specifying the java.security.auth.login.config system property in + NiFi's bootstrap.conf, such as: + 

+    java.arg.16=-Djava.security.auth.login.config=/path/to/kafka_client_jaas.conf
+
+

+

+ An example of the JAAS config file would be the following: + 

+    KafkaClient {
+        com.sun.security.auth.module.Krb5LoginModule required
+        useKeyTab=true
+        storeKey=true
+        keyTab="/path/to/nifi.keytab"
+        serviceName="kafka"
+        principal="nifi@YOURREALM.COM";
+    };
+
+ NOTE: The serviceName in the JAAS file must match the Kerberos Service Name in the processor. +

+

+ Alternatively, starting with Apache NiFi 1.2.0 which uses the Kafka 0.10.2 client, the JAAS + configuration when using GSSAPI can be provided by specifying the Kerberos Principal and Kerberos Keytab + directly in the processor properties. This will dynamically create a JAAS configuration like above, and + will take precedence over the java.security.auth.login.config system property. +

+

SASL_PLAINTEXT - PLAIN

+

+ If the SASL mechanism is PLAIN, then client must provide a JAAS configuration to authenticate, but + the JAAS configuration must use Kafka's PlainLoginModule. An example of the JAAS config file would + be the following: + 

+    KafkaClient {
+      org.apache.kafka.common.security.plain.PlainLoginModule required
+      username="nifi"
+      password="nifi-password";
+    };
+
+

+

+ NOTE: It is not recommended to use a SASL mechanism of PLAIN with SASL_PLAINTEXT, as it would transmit + the username and password unencrypted. +

+

+ NOTE: Using the PlainLoginModule will cause it be registered in the JVM's static list of Providers, making + it visible to components in other NARs that may access the providers. There is currently a known issue + where Kafka processors using the PlainLoginModule will cause HDFS processors with Keberos to no longer work. +

+

SASL_SSL

+

+ This option uses SASL with an SSL/TLS transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_SSL://host.name:port
+
+

+

+ See the SASL_PLAINTEXT section for a description of how to provide the proper JAAS configuration + depending on the SASL mechanism (GSSAPI or PLAIN). +

+

+ See the SSL section for a description of how to configure the SSL Context Service based on the + ssl.client.auth property. +

+ diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_0_10/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_0_10/additionalDetails.html new file mode 100644 index 0000000000..f997a49a76 --- /dev/null +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_0_10/additionalDetails.html @@ -0,0 +1,143 @@ + + + + + + PublishKafka + + + + + +

Description:

+

+ This Processor puts the contents of a FlowFile to a Topic in + Apache Kafka using KafkaProducer API available + with Kafka 0.10.x API. The contents of the incoming FlowFile will be read using the + configured Record Reader. Each record will then be serialized using the configured + Record Writer, and this serialized form will be the content of a Kafka message. + This message is optionally assigned a key by using the <Kafka Key> Property. +

+

Security Configuration:

+

+ The Security Protocol property allows the user to specify the protocol for communicating + with the Kafka broker. The following sections describe each of the protocols in further detail. +

+

PLAINTEXT

+

+ This option provides an unsecured connection to the broker, with no client authentication and no encryption. + In order to use this option the broker must be configured with a listener of the form: + 

+    PLAINTEXT://host.name:port
+
+

+

SSL

+

+ This option provides an encrypted connection to the broker, with optional client authentication. In order + to use this option the broker must be configured with a listener of the form: + 

+    SSL://host.name:port
+
+ In addition, the processor must have an SSL Context Service selected. +

+

+ If the broker specifies ssl.client.auth=none, or does not specify ssl.client.auth, then the client will + not be required to present a certificate. In this case, the SSL Context Service selected may specify only + a truststore containing the public key of the certificate authority used to sign the broker's key. +

+

+ If the broker specifies ssl.client.auth=required then the client will be required to present a certificate. + In this case, the SSL Context Service must also specify a keystore containing a client key, in addition to + a truststore as described above. +

+

SASL_PLAINTEXT

+

+ This option uses SASL with a PLAINTEXT transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_PLAINTEXT://host.name:port
+
+ In addition, the Kerberos Service Name must be specified in the processor. +

+

SASL_PLAINTEXT - GSSAPI

+

+ If the SASL mechanism is GSSAPI, then the client must provide a JAAS configuration to authenticate. The + JAAS configuration can be provided by specifying the java.security.auth.login.config system property in + NiFi's bootstrap.conf, such as: + 

+    java.arg.16=-Djava.security.auth.login.config=/path/to/kafka_client_jaas.conf
+
+

+

+ An example of the JAAS config file would be the following: + 

+    KafkaClient {
+        com.sun.security.auth.module.Krb5LoginModule required
+        useKeyTab=true
+        storeKey=true
+        keyTab="/path/to/nifi.keytab"
+        serviceName="kafka"
+        principal="nifi@YOURREALM.COM";
+    };
+
+ NOTE: The serviceName in the JAAS file must match the Kerberos Service Name in the processor. +

+

+ Alternatively, starting with Apache NiFi 1.2.0 which uses the Kafka 0.10.2 client, the JAAS + configuration when using GSSAPI can be provided by specifying the Kerberos Principal and Kerberos Keytab + directly in the processor properties. This will dynamically create a JAAS configuration like above, and + will take precedence over the java.security.auth.login.config system property. +

+

SASL_PLAINTEXT - PLAIN

+

+ If the SASL mechanism is PLAIN, then client must provide a JAAS configuration to authenticate, but + the JAAS configuration must use Kafka's PlainLoginModule. An example of the JAAS config file would + be the following: + 

+    KafkaClient {
+      org.apache.kafka.common.security.plain.PlainLoginModule required
+      username="nifi"
+      password="nifi-password";
+    };
+
+

+

+ NOTE: It is not recommended to use a SASL mechanism of PLAIN with SASL_PLAINTEXT, as it would transmit + the username and password unencrypted. +

+

+ NOTE: Using the PlainLoginModule will cause it be registered in the JVM's static list of Providers, making + it visible to components in other NARs that may access the providers. There is currently a known issue + where Kafka processors using the PlainLoginModule will cause HDFS processors with Keberos to no longer work. +

+

SASL_SSL

+

+ This option uses SASL with an SSL/TLS transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_SSL://host.name:port
+
+

+

+ See the SASL_PLAINTEXT section for a description of how to provide the proper JAAS configuration + depending on the SASL mechanism (GSSAPI or PLAIN). +

+

+ See the SSL section for a description of how to configure the SSL Context Service based on the + ssl.client.auth property. +

+ + diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_0_10/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_0_10/additionalDetails.html index 95c34c783c..119eb93ecf 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_0_10/additionalDetails.html +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_0_10/additionalDetails.html @@ -43,5 +43,113 @@ that was successfully ACKed by Kafka. (if no demarcator is used the value of this index will be -1). This will allow PublishKafka to only re-send un-ACKed messages on the next re-try.

+

Security Configuration:

+

+ The Security Protocol property allows the user to specify the protocol for communicating + with the Kafka broker. The following sections describe each of the protocols in further detail. +

+

PLAINTEXT

+

+ This option provides an unsecured connection to the broker, with no client authentication and no encryption. + In order to use this option the broker must be configured with a listener of the form: + 

+    PLAINTEXT://host.name:port
+
+

+

SSL

+

+ This option provides an encrypted connection to the broker, with optional client authentication. In order + to use this option the broker must be configured with a listener of the form: + 

+    SSL://host.name:port
+
+ In addition, the processor must have an SSL Context Service selected. +

+

+ If the broker specifies ssl.client.auth=none, or does not specify ssl.client.auth, then the client will + not be required to present a certificate. In this case, the SSL Context Service selected may specify only + a truststore containing the public key of the certificate authority used to sign the broker's key. +

+

+ If the broker specifies ssl.client.auth=required then the client will be required to present a certificate. + In this case, the SSL Context Service must also specify a keystore containing a client key, in addition to + a truststore as described above. +

+

SASL_PLAINTEXT

+

+ This option uses SASL with a PLAINTEXT transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_PLAINTEXT://host.name:port
+
+ In addition, the Kerberos Service Name must be specified in the processor. +

+

SASL_PLAINTEXT - GSSAPI

+

+ If the SASL mechanism is GSSAPI, then the client must provide a JAAS configuration to authenticate. The + JAAS configuration can be provided by specifying the java.security.auth.login.config system property in + NiFi's bootstrap.conf, such as: + 

+    java.arg.16=-Djava.security.auth.login.config=/path/to/kafka_client_jaas.conf
+
+

+

+ An example of the JAAS config file would be the following: + 

+    KafkaClient {
+        com.sun.security.auth.module.Krb5LoginModule required
+        useKeyTab=true
+        storeKey=true
+        keyTab="/path/to/nifi.keytab"
+        serviceName="kafka"
+        principal="nifi@YOURREALM.COM";
+    };
+
+ NOTE: The serviceName in the JAAS file must match the Kerberos Service Name in the processor. +

+

+ Alternatively, starting with Apache NiFi 1.2.0 which uses the Kafka 0.10.2 client, the JAAS + configuration when using GSSAPI can be provided by specifying the Kerberos Principal and Kerberos Keytab + directly in the processor properties. This will dynamically create a JAAS configuration like above, and + will take precedence over the java.security.auth.login.config system property. +

+

SASL_PLAINTEXT - PLAIN

+

+ If the SASL mechanism is PLAIN, then client must provide a JAAS configuration to authenticate, but + the JAAS configuration must use Kafka's PlainLoginModule. An example of the JAAS config file would + be the following: + 

+    KafkaClient {
+      org.apache.kafka.common.security.plain.PlainLoginModule required
+      username="nifi"
+      password="nifi-password";
+    };
+
+

+

+ NOTE: It is not recommended to use a SASL mechanism of PLAIN with SASL_PLAINTEXT, as it would transmit + the username and password unencrypted. +

+

+ NOTE: Using the PlainLoginModule will cause it be registered in the JVM's static list of Providers, making + it visible to components in other NARs that may access the providers. There is currently a known issue + where Kafka processors using the PlainLoginModule will cause HDFS processors with Keberos to no longer work. +

+

SASL_SSL

+

+ This option uses SASL with an SSL/TLS transport layer to authenticate to the broker. In order to use this + option the broker must be configured with a listener of the form: + 

+    SASL_SSL://host.name:port
+
+

+

+ See the SASL_PLAINTEXT section for a description of how to provide the proper JAAS configuration + depending on the SASL mechanism (GSSAPI or PLAIN). +

+

+ See the SSL section for a description of how to configure the SSL Context Service based on the + ssl.client.auth property. +