From b6941551aee143074049c8187c84b84aa1014a49 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 21 May 2019 11:30:14 -0700 Subject: [PATCH] Upgrade various build and doc links to https. (#7722) * Upgrade various build and doc links to https. Where it wasn't possible to upgrade build-time dependencies to https, I kept http in place but used hardcoded checksums or GPG keys to ensure that artifacts fetched over http are verified properly. * Switch to https://apache.org. --- distribution/docker/Dockerfile.mysql | 2 +- .../extensions-core/datasketches-extension.md | 2 +- .../extensions-core/datasketches-hll.md | 2 +- .../extensions-core/datasketches-quantiles.md | 2 +- .../extensions-core/datasketches-theta.md | 2 +- .../extensions-core/datasketches-tuple.md | 2 +- docs/content/development/extensions.md | 2 +- docs/content/querying/aggregations.md | 4 +- .../tutorial/hadoop/docker/Dockerfile | 5 +- .../tutorial/hadoop/docker/setup-zulu-repo.sh | 67 +++++++++++++++++++ .../ambari-metrics-emitter/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/README.md | 2 +- extensions-core/datasketches/pom.xml | 2 +- hll/pom.xml | 2 +- integration-tests/docker-base/setup.sh | 14 ++-- pom.xml | 6 +- 17 files changed, 95 insertions(+), 25 deletions(-) create mode 100755 examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh diff --git a/distribution/docker/Dockerfile.mysql b/distribution/docker/Dockerfile.mysql index d2d4288d744..5664dc8af97 100644 --- a/distribution/docker/Dockerfile.mysql +++ b/distribution/docker/Dockerfile.mysql @@ -21,7 +21,7 @@ ARG DRUID_RELEASE=druid/druid:0.14.0 FROM $DRUID_RELEASE COPY sha256sums.txt /tmp -RUN wget -O /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar \ +RUN wget -O /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar \ && sed -e '/^#/d' /tmp/sha256sums.txt > /tmp/sha256sums-stripped.txt \ && sha256sum -c /tmp/sha256sums-stripped.txt \ && rm -f /opt/druid/lib/mysql-connector-java-5.1.38.jar \ diff --git a/docs/content/development/extensions-core/datasketches-extension.md b/docs/content/development/extensions-core/datasketches-extension.md index 3a5b126c99f..49ac2257f02 100644 --- a/docs/content/development/extensions-core/datasketches-extension.md +++ b/docs/content/development/extensions-core/datasketches-extension.md @@ -24,7 +24,7 @@ title: "DataSketches extension" # DataSketches extension -Apache Druid (incubating) aggregators based on [datasketches](http://datasketches.github.io/) library. Sketches are data structures implementing approximate streaming mergeable algorithms. Sketches can be ingested from the outside of Druid or built from raw data at ingestion time. Sketches can be stored in Druid segments as additive metrics. +Apache Druid (incubating) aggregators based on [datasketches](https://datasketches.github.io/) library. Sketches are data structures implementing approximate streaming mergeable algorithms. Sketches can be ingested from the outside of Druid or built from raw data at ingestion time. Sketches can be stored in Druid segments as additive metrics. To use the datasketches aggregators, make sure you [include](../../operations/including-extensions.html) the extension in your config file: diff --git a/docs/content/development/extensions-core/datasketches-hll.md b/docs/content/development/extensions-core/datasketches-hll.md index 799cbc07b30..90e284f155b 100644 --- a/docs/content/development/extensions-core/datasketches-hll.md +++ b/docs/content/development/extensions-core/datasketches-hll.md @@ -24,7 +24,7 @@ title: "DataSketches HLL Sketch module" # DataSketches HLL Sketch module -This module provides Apache Druid (incubating) aggregators for distinct counting based on HLL sketch from [datasketches](http://datasketches.github.io/) library. At ingestion time, this aggregator creates the HLL sketch objects to be stored in Druid segments. At query time, sketches are read and merged together. In the end, by default, you receive the estimate of the number of distinct values presented to the sketch. Also, you can use post aggregator to produce a union of sketch columns in the same row. +This module provides Apache Druid (incubating) aggregators for distinct counting based on HLL sketch from [datasketches](https://datasketches.github.io/) library. At ingestion time, this aggregator creates the HLL sketch objects to be stored in Druid segments. At query time, sketches are read and merged together. In the end, by default, you receive the estimate of the number of distinct values presented to the sketch. Also, you can use post aggregator to produce a union of sketch columns in the same row. You can use the HLL sketch aggregator on columns of any identifiers. It will return estimated cardinality of the column. To use this aggregator, make sure you [include](../../operations/including-extensions.html) the extension in your config file: diff --git a/docs/content/development/extensions-core/datasketches-quantiles.md b/docs/content/development/extensions-core/datasketches-quantiles.md index 62b2e1fd926..39b7cb9f54c 100644 --- a/docs/content/development/extensions-core/datasketches-quantiles.md +++ b/docs/content/development/extensions-core/datasketches-quantiles.md @@ -24,7 +24,7 @@ title: "DataSketches Quantiles Sketch module" # DataSketches Quantiles Sketch module -This module provides Apache Druid (incubating) aggregators based on numeric quantiles DoublesSketch from [datasketches](http://datasketches.github.io/) library. Quantiles sketch is a mergeable streaming algorithm to estimate the distribution of values, and approximately answer queries about the rank of a value, probability mass function of the distribution (PMF) or histogram, cummulative distribution function (CDF), and quantiles (median, min, max, 95th percentile and such). See [Quantiles Sketch Overview](https://datasketches.github.io/docs/Quantiles/QuantilesOverview.html). +This module provides Apache Druid (incubating) aggregators based on numeric quantiles DoublesSketch from [datasketches](https://datasketches.github.io/) library. Quantiles sketch is a mergeable streaming algorithm to estimate the distribution of values, and approximately answer queries about the rank of a value, probability mass function of the distribution (PMF) or histogram, cummulative distribution function (CDF), and quantiles (median, min, max, 95th percentile and such). See [Quantiles Sketch Overview](https://datasketches.github.io/docs/Quantiles/QuantilesOverview.html). There are three major modes of operation: diff --git a/docs/content/development/extensions-core/datasketches-theta.md b/docs/content/development/extensions-core/datasketches-theta.md index e248da35314..5a2d1afdbde 100644 --- a/docs/content/development/extensions-core/datasketches-theta.md +++ b/docs/content/development/extensions-core/datasketches-theta.md @@ -24,7 +24,7 @@ title: "DataSketches Theta Sketch module" # DataSketches Theta Sketch module -This module provides Apache Druid (incubating) aggregators based on Theta sketch from [datasketches](http://datasketches.github.io/) library. Note that sketch algorithms are approximate; see details in the "Accuracy" section of the datasketches doc. +This module provides Apache Druid (incubating) aggregators based on Theta sketch from [datasketches](https://datasketches.github.io/) library. Note that sketch algorithms are approximate; see details in the "Accuracy" section of the datasketches doc. At ingestion time, this aggregator creates the Theta sketch objects which get stored in Druid segments. Logically speaking, a Theta sketch object can be thought of as a Set data structure. At query time, sketches are read and aggregated (set unioned) together. In the end, by default, you receive the estimate of the number of unique entries in the sketch object. Also, you can use post aggregators to do union, intersection or difference on sketch columns in the same row. Note that you can use `thetaSketch` aggregator on columns which were not ingested using the same. It will return estimated cardinality of the column. It is recommended to use it at ingestion time as well to make querying faster. diff --git a/docs/content/development/extensions-core/datasketches-tuple.md b/docs/content/development/extensions-core/datasketches-tuple.md index 69db25a1b7a..bd83c9f2643 100644 --- a/docs/content/development/extensions-core/datasketches-tuple.md +++ b/docs/content/development/extensions-core/datasketches-tuple.md @@ -24,7 +24,7 @@ title: "DataSketches Tuple Sketch module" # DataSketches Tuple Sketch module -This module provides Apache Druid (incubating) aggregators based on Tuple sketch from [datasketches](http://datasketches.github.io/) library. ArrayOfDoublesSketch sketches extend the functionality of the count-distinct Theta sketches by adding arrays of double values associated with unique keys. +This module provides Apache Druid (incubating) aggregators based on Tuple sketch from [datasketches](https://datasketches.github.io/) library. ArrayOfDoublesSketch sketches extend the functionality of the count-distinct Theta sketches by adding arrays of double values associated with unique keys. To use this aggregator, make sure you [include](../../operations/including-extensions.html) the extension in your config file: diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md index 2190793bbd4..5112ee9ac56 100644 --- a/docs/content/development/extensions.md +++ b/docs/content/development/extensions.md @@ -45,7 +45,7 @@ Core extensions are maintained by Druid committers. |druid-basic-security|Support for Basic HTTP authentication and role-based access control.|[link](../development/extensions-core/druid-basic-security.html)| |druid-bloom-filter|Support for providing Bloom filters in druid queries.|[link](../development/extensions-core/bloom-filter.html)| |druid-caffeine-cache|A local cache implementation backed by Caffeine.|[link](../configuration/index.html#cache-configuration)| -|druid-datasketches|Support for approximate counts and set operations with [DataSketches](http://datasketches.github.io/).|[link](../development/extensions-core/datasketches-extension.html)| +|druid-datasketches|Support for approximate counts and set operations with [DataSketches](https://datasketches.github.io/).|[link](../development/extensions-core/datasketches-extension.html)| |druid-hdfs-storage|HDFS deep storage.|[link](../development/extensions-core/hdfs.html)| |druid-histogram|Approximate histograms and quantiles aggregator. Deprecated, please use the [DataSketches quantiles aggregator](../development/extensions-core/datasketches-quantiles.html) from the `druid-datasketches` extension instead.|[link](../development/extensions-core/approximate-histograms.html)| |druid-kafka-eight|Kafka ingest firehose (high level consumer) for realtime nodes(deprecated).|[link](../development/extensions-core/kafka-eight-firehose.html)| diff --git a/docs/content/querying/aggregations.md b/docs/content/querying/aggregations.md index b6b3e039054..ba9b80ef4f9 100644 --- a/docs/content/querying/aggregations.md +++ b/docs/content/querying/aggregations.md @@ -271,7 +271,7 @@ JavaScript-based functionality is disabled by default. Please refer to the Druid #### DataSketches Theta Sketch -The [DataSketches Theta Sketch](../development/extensions-core/datasketches-theta.html) extension-provided aggregator gives distinct count estimates with support for set union, intersection, and difference post-aggregators, using Theta sketches from the [datasketches](http://datasketches.github.io/) library. +The [DataSketches Theta Sketch](../development/extensions-core/datasketches-theta.html) extension-provided aggregator gives distinct count estimates with support for set union, intersection, and difference post-aggregators, using Theta sketches from the [datasketches](https://datasketches.github.io/) library. #### DataSketches HLL Sketch @@ -304,7 +304,7 @@ Note the DataSketches Theta and HLL aggregators currently only support single-co #### DataSketches Quantiles Sketch -The [DataSketches Quantiles Sketch](../development/extensions-core/datasketches-quantiles.html) extension-provided aggregator provides quantile estimates and histogram approximations using the numeric quantiles DoublesSketch from the [datasketches](http://datasketches.github.io/) library. +The [DataSketches Quantiles Sketch](../development/extensions-core/datasketches-quantiles.html) extension-provided aggregator provides quantile estimates and histogram approximations using the numeric quantiles DoublesSketch from the [datasketches](https://datasketches.github.io/) library. We recommend this aggregator in general for quantiles/histogram use cases, as it provides formal error bounds and has distribution-independent accuracy. diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile index 1801da2bc11..b02496fbb99 100644 --- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile +++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile @@ -41,9 +41,8 @@ RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys # zulu java 8 -RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems -RUN rpm --rebuilddb -RUN sudo curl -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo +COPY setup-zulu-repo.sh /root/setup-zulu-repo.sh +RUN /root/setup-zulu-repo.sh RUN yum install -y zulu-8 ENV JAVA_HOME /usr/lib/jvm/zulu-8 diff --git a/examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh b/examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh new file mode 100755 index 00000000000..087240d2a05 --- /dev/null +++ b/examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Script to set up the Azul Zulu JDK yum repository. +# + +# Hardcode GPG key so we don't have to fetch it over http. +cat <<'EOT' > /root/RPM-GPG-KEY-azulsystems +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFNgFa8BEADTL/REB10M+TfiZOtFHqL5LHKkzTMn/O2r5iIqXGhi6iwZazFs +9S5g1eU7WMen5Xp9AREs+OvaHx91onPZ7ZiP7VpZ6ZdwWrnVk1Y/HfI59tWxmNYW +DmKYBGMj4EUpFPSE9EnFj7dm1WdlCvpognCwZQl9D3BseGqN7OLHfwqqmOlbYN9h +HYkT+CaqOoWDIGMB3UkBlMr0GuujEP8N1gxg7EOcSCsZH5aKtXubdUlVSphfAAwD +z4MviB39J22sPBnKmaOT3TUTO5vGeKtC9BAvtgA82jY2TtCEjetnfK/qtzj/6j2N +xVUbHQydwNQVRU92A7334YvCbn3xUUNI0WOscdmfpgCU0Z9Gb2IqDb9cMjgUi8F6 +MG/QY9/CZjX62XrHRPm3aXsCJOVh/PO1sl2A/rvv8AkpJKYyhm6T8OBFptCsA3V4 +Oic7ZyYhqV0u2r4NON+1MoUeuuoeY2tIrbRxe3ffVOxPzrESzSbc8LC2tYaP+wGd +W0f57/CoDkUzlvpReCUI1Bv5zP4/jhC63Rh6lffvSf2tQLwOsf5ivPhUtwUfOQjg +v9P8Wc8K7XZpSOMnDZuDe9wuvB/DiH/P5yiTs2RGsbDdRh5iPfwbtf2+IX6h2lNZ +XiDKt9Gc26uzeJRx/c7+sLunxq6DLIYvrsEipVI9frHIHV6fFTmqMJY6SwARAQAB +tEdBenVsIFN5c3RlbXMsIEluYy4gKFBhY2thZ2Ugc2lnbmluZyBrZXkuKSA8cGtp +LXNpZ25pbmdAYXp1bHN5c3RlbXMuY29tPokCOAQTAQIAIgUCU2AVrwIbAwYLCQgH +AwIGFQgCCQoLBBYCAwECHgECF4AACgkQsZmDYSGb2cnJ8xAAz1V1PJnfOyaRIP2N +Ho2uRwGdPsA4eFMXb4Z08eGjDMD3b9WW3D0XnCLbJpaZ6klz0W0s2tcYSneTBaSs +RAqxgJgBZ5ZMXtrrHld/5qFoBbStLZLefmcPhnfvamwHDCTLUex8NIAI1u3e9Rhb +5fbH+gpuYpwHX7hz0FOfpn1sxR03UyxU+ey4AdKe9LG3TJVnB0WcgxpobpbqweLH +yzcEQCNoFV3r1rlE13Y0aE31/9apoEwiYvqAzEmE38TukDLl/Qg8rkR1t0/lok2P +G6pWqdN7pmoUovBTvDi5YOthcjZcdOTXXn2Yw4RZVF9uhRsVfku1Eg25SnOje3uY +smtQLME4eESbePdjyV/okCIle66uHZse+7gNyNmWpf01hM+VmAySIAyKa0Ku8AXZ +MydEcJTebrNfW9uMLsBx3Ts7z/CBfRng6F8louJGlZtlSwddTkZVcb26T20xeo0a +ZvdFXM2djTi/a5nbBoZQL85AEeV7HaphFLdPrgmMtS8sSZUEVvdaxp7WJsVuF9cO +Nxsvx40OYTvfco0W41Lm8/sEuQ7YueEVpZxiv5kX56GTU9vXaOOi+8Z7Ee2w6Adz +4hrGZkzztggs4tM9geNYnd0XCdZ/ICAskKJABg7biDD1PhEBrqCIqSE3U497vibQ +Mpkkl/Zpp0BirhGWNyTg8K4JrsQ= +=d320 +-----END PGP PUBLIC KEY BLOCK----- +EOT + +rpm --import /root/RPM-GPG-KEY-azulsystems +rpm --rebuilddb + +# Do not include "gpgkey" in the repo definition -- we've already imported it, above. +cat <<'EOT' > /etc/yum.repos.d/zulu.repo +[zulu] +name=zulu-$releasever - Azul Systems Inc., Zulu packages for $basearch +baseurl=http://repos.azulsystems.com/rhel/$releasever/$basearch +enabled=1 +gpgcheck=1 +protect=1 +EOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index dc83ba5326b..761a19d9a7d 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -84,7 +84,7 @@ hortonworks hortonworks - http://repo.hortonworks.com/content/repositories/releases + https://repo.hortonworks.com/content/repositories/releases diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index f884b4abcbb..4f272f254df 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -44,7 +44,7 @@ confluent - http://packages.confluent.io/maven/ + https://packages.confluent.io/maven/ diff --git a/extensions-core/datasketches/README.md b/extensions-core/datasketches/README.md index 12f2a87f8fb..410c206fef9 100644 --- a/extensions-core/datasketches/README.md +++ b/extensions-core/datasketches/README.md @@ -17,7 +17,7 @@ ~ under the License. --> -This module provides druid aggregators based on http://datasketches.github.io/ . +This module provides Druid aggregators based on https://datasketches.github.io/. Credits: This module is a result of feedback and work done by following people. diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 11708a73bdc..83cf452cb34 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -25,7 +25,7 @@ org.apache.druid.extensions druid-datasketches druid-datasketches - Druid Aggregators based on datasketches lib http://datasketches.github.io/ + Druid Aggregators based on datasketches lib https://datasketches.github.io/ org.apache.druid diff --git a/hll/pom.xml b/hll/pom.xml index 84ae4021c70..3e0e25701f9 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -34,7 +34,7 @@ Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 diff --git a/integration-tests/docker-base/setup.sh b/integration-tests/docker-base/setup.sh index 13ad92fdc9b..f2be680903b 100644 --- a/integration-tests/docker-base/setup.sh +++ b/integration-tests/docker-base/setup.sh @@ -34,14 +34,18 @@ apt-get install -y mysql-server apt-get install -y supervisor # Zookeeper -wget -q -O - http://www.us.apache.org/dist/zookeeper/zookeeper-3.4.13/zookeeper-3.4.13.tar.gz | tar -xzf - -C /usr/local \ - && cp /usr/local/zookeeper-3.4.13/conf/zoo_sample.cfg /usr/local/zookeeper-3.4.13/conf/zoo.cfg \ - && ln -s /usr/local/zookeeper-3.4.13 /usr/local/zookeeper +wget -q -O /tmp/zookeeper-3.4.14.tar.gz "https://apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz" +tar -xzf /tmp/zookeeper-3.4.14.tar.gz -C /usr/local +cp /usr/local/zookeeper-3.4.14/conf/zoo_sample.cfg /usr/local/zookeeper-3.4.14/conf/zoo.cfg +ln -s /usr/local/zookeeper-3.4.14 /usr/local/zookeeper +rm /tmp/zookeeper-3.4.14.tar.gz # Kafka # Match the version to the Kafka client used by KafkaSupervisor -wget -q -O - http://www.us.apache.org/dist/kafka/2.1.0/kafka_2.12-2.1.0.tgz | tar -xzf - -C /usr/local \ - && ln -s /usr/local/kafka_2.12-2.1.0 /usr/local/kafka +wget -q -O /tmp/kafka_2.12-2.1.1.tgz "https://apache.org/dist/kafka/2.1.1/kafka_2.12-2.1.1.tgz" +tar -xzf /tmp/kafka_2.12-2.1.1.tgz -C /usr/local +ln -s /usr/local/kafka_2.12-2.1.1 /usr/local/kafka +rm /tmp/kafka_2.12-2.1.1.tgz # Druid system user adduser --system --group --no-create-home druid \ diff --git a/pom.xml b/pom.xml index eb295d02edb..8219edc0c20 100644 --- a/pom.xml +++ b/pom.xml @@ -40,14 +40,14 @@ Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 - Druid Committers - http://druid.io/community/index.html#committers + Apache Druid Committers + https://druid.apache.org/community/#committers