From 748695bf974cfdff30cd75118f7e8c7de698c2e2 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Thu, 20 Jun 2013 14:59:52 -0700 Subject: [PATCH 01/25] trying to recover my files on intellij --- .DS_Store | Bin 0 -> 15364 bytes .../com/metamx/druid/client/DruidServer.java | 1 + common/.DS_Store | Bin 0 -> 6148 bytes common/src/.DS_Store | Bin 0 -> 6148 bytes common/src/main/.DS_Store | Bin 0 -> 6148 bytes common/src/main/java/.DS_Store | Bin 0 -> 6148 bytes common/src/main/java/com/.DS_Store | Bin 0 -> 6148 bytes common/src/main/java/com/metamx/.DS_Store | Bin 0 -> 6148 bytes examples/.DS_Store | Bin 0 -> 6148 bytes examples/query.body | 12 + examples/search_query.body | 13 + .../main/java/druid/examples/HttpTest.java | 55 + .../druid/examples/twitter/Configuration.java | 59 + .../twitter/ConfigurationContext.java | 57 + .../twitter/ConfigurationFactory.java | 42 + .../twitter/HttpClientConfiguration.java | 46 + .../HttpClientWrapperConfiguration.java | 13 + .../druid/examples/twitter/HttpParameter.java | 288 ++++ .../druid/examples/twitter/HttpRequest.java | 120 ++ .../druid/examples/twitter/HttpResponse.java | 239 +++ .../examples/twitter/HttpResponseEvent.java | 92 ++ .../examples/twitter/HttpResponseImpl.java | 75 + .../twitter/HttpResponseListener.java | 10 + .../druid/examples/twitter/JSONArray.java | 726 ++++++++++ .../druid/examples/twitter/JSONObject.java | 1285 +++++++++++++++++ .../druid/examples/twitter/JSONTokener.java | 359 +++++ .../java/druid/examples/twitter/Logger.java | 180 +++ .../druid/examples/twitter/LoggerFactory.java | 34 + .../twitter/PropertyConfiguration.java | 325 +++++ .../twitter/PropertyConfigurationFactory.java | 61 + .../druid/examples/twitter/RequestMethod.java | 26 + .../twitter/StreamingGZipInputStream.java | 53 + .../TwitterSpritzerFirehoseFactory.java | 2 +- .../twitter/USGovFirehoseFactory.java | 173 +++ .../java/druid/examples/twitter/WebBase.java | 15 + .../druid/examples/twitter/WebBaseImpl.java | 170 +++ .../twitter/WebConfigurationBase.java | 494 +++++++ .../examples/twitter/WebStatusStreamBase.java | 176 +++ .../examples/twitter/WebStreamFactory.java | 111 ++ .../druid/examples/twitter/WebStreamImpl.java | 750 ++++++++++ .../twitter/z_InternalStringUtil.java | 111 ++ examples/twitter4j.properties | 5 + examples/twitter_realtime.spec | 44 + indexing-common/.DS_Store | Bin 0 -> 6148 bytes indexing-common/src/.DS_Store | Bin 0 -> 6148 bytes indexing-common/src/main/.DS_Store | Bin 0 -> 6148 bytes indexing-common/src/main/java/.DS_Store | Bin 0 -> 6148 bytes install/.DS_Store | Bin 0 -> 6148 bytes realtime/.DS_Store | Bin 0 -> 6148 bytes .../realtime/firehose/WebFirehoseFactory.java | 85 ++ server/.DS_Store | Bin 0 -> 6148 bytes services/.DS_Store | Bin 0 -> 6148 bytes 52 files changed, 6306 insertions(+), 1 deletion(-) create mode 100644 .DS_Store create mode 100644 common/.DS_Store create mode 100644 common/src/.DS_Store create mode 100644 common/src/main/.DS_Store create mode 100644 common/src/main/java/.DS_Store create mode 100644 common/src/main/java/com/.DS_Store create mode 100644 common/src/main/java/com/metamx/.DS_Store create mode 100644 examples/.DS_Store create mode 100644 examples/query.body create mode 100644 examples/search_query.body create mode 100644 examples/src/main/java/druid/examples/HttpTest.java create mode 100644 examples/src/main/java/druid/examples/twitter/Configuration.java create mode 100644 examples/src/main/java/druid/examples/twitter/ConfigurationContext.java create mode 100644 examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpParameter.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpRequest.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponse.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseListener.java create mode 100644 examples/src/main/java/druid/examples/twitter/JSONArray.java create mode 100644 examples/src/main/java/druid/examples/twitter/JSONObject.java create mode 100644 examples/src/main/java/druid/examples/twitter/JSONTokener.java create mode 100644 examples/src/main/java/druid/examples/twitter/Logger.java create mode 100644 examples/src/main/java/druid/examples/twitter/LoggerFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java create mode 100644 examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/RequestMethod.java create mode 100644 examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java create mode 100644 examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebBase.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebBaseImpl.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebStreamFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebStreamImpl.java create mode 100644 examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java create mode 100644 examples/twitter4j.properties create mode 100644 examples/twitter_realtime.spec create mode 100644 indexing-common/.DS_Store create mode 100644 indexing-common/src/.DS_Store create mode 100644 indexing-common/src/main/.DS_Store create mode 100644 indexing-common/src/main/java/.DS_Store create mode 100644 install/.DS_Store create mode 100644 realtime/.DS_Store create mode 100644 realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java create mode 100644 server/.DS_Store create mode 100644 services/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ee1df41e8fb05f1e8db6280875eaa40f45860a2c GIT binary patch literal 15364 zcmeI1&rcIU6vy9CXtxm~7mVIEo;VgCd?BXQo)|cp#|ztebe)+32k{G1cZPP5CTF#2wVpO_|E2H-3p{!g@6zc0z(44 zKG>)%i$)Ftsh191^cDbe5|4GkJ@x_G#x}BOfD(w zxFbqa6}m3fs7YPk>xXoIjL`L|fsaqksIJ<&Ot*5}X&JxYDlO0LbaT0%Y9ci`mCjh% ztTny6<2D`tc-22_I-S?0x_?}9om!>bexvWZ^%L$G(Qw#DrFPTvn+I%!ngV8@sxcV9@Fom73$3;e3m>Yu0O-XmfKQ!@{|MDVf655`WxhB8qP}V- zU0OIAs?i?p>s~eBsf&*b{|D$-Z}bQwzAV`HVAsGXJluDp<50{n+;&1glvLlZAgS*6 z`}w89GP0_0X;yJkeMH|;F3hUWc$pYaEVJ%tnvSLvaD5og!ze9|y0Su1|AV#Dsd^R6 z9Zumm;jEEo!eg(Ze0hX!{P5hm$crEszBIp8<7t z#y}~i9ZG>4xUR$o6N`Tab9@uCbAw(|i8i7bb-a6_cxUwb&=@`Uh=Uf^PSY0ru3%+& z5%;VE)C1}XgWN(WYdSbQzZ;2wa)?-e>LUHC4^lpl1^-c@FpkAPG5lDV(t07}Y)8(I z2+}FgOd2#1-K)nk-tWZoNseL1kXS<`E>^6>%C57Kv809>bMDLN-)uCA;m7b`S5Z<*_H>i+{(BtB{NJ=gB;30$x9)%Qou%d|(8wg2BQhTV;;4}ChzKzf0 z%M5lQ@mCrI#(zsJQZH+0u`r`!L9_ z6)%HwQFuYFJeNT^1CQe4EK7h-P4py=%Gr(GS*`kx)0Mr!pz6xia0vXQe_B8qkQ5*VNC8rS6i^D7148VP z9;5&%Kni?Q0QUz81<^HFYgAhYRCs+xe;W}EIKCwir9szVtr0w6T&Du+RBoOaT&F|7 zG;yxMTBA;9T+IyQn30tS5T5&#X{M>@)>{SrCExdv;EI12WwbU?ZY MC_$(q1%5$+PkJjxzW@LL literal 0 HcmV?d00001 diff --git a/common/src/.DS_Store b/common/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..73307bf9352f15b2d73524ac7d4be4c294bd50f0 GIT binary patch literal 6148 zcmeHK!AiqG5Z!H~Ca92$(BtB{NJ=gB;30$x9)%Qou%ZbO8;D71QhTV;;5Yam{*B+` z%0Qb%X6wBNT7SeMO6f^KhKF+cP_*6$v;;5Ki+nx2g?>Jr28w|>>SPzH5kNYQOX_=;F zznTY!N2llG;IVvqu9S`(WR|wCv4$5QpT~Ic7ip5EcQ9AS*KvfT04YEUkOHJYeL0xB zqy;HJ3XlTd6u{z9KtpsjRtjZZ2_2Tt=r<72fa6;NQD}5ERtmub##JhyO6B^A!BslU z3mxZbtQ4wr#^uhij^4R`yl}ZY%nKRLxT}zQQh*d_D$rI#8}I)!_{nU1kp*nQg0J5nPLnt<9Os=kn#RuVo@rU;!FUv>QMUB5c^c){{v=!aadaOB zvm42apqS@gFq59EpqPM1@?n-Gz{fgz6i3D6)^4p&)r&(b7IA7HMIuj2?w0aAbzAO%Q)`f@P$ zNefbd6d(n@DS*YJfQIO5tQ5++5;`oO(QhE40mrulqR{AStQ3L=jH^^YmCE%KgR69y z7dp<>SSeKLjLV&29ldk?c;Rw)m=`jfaaSSrqyQ7k-wQjBT|4A z_-6`mW8e=wIF!Cye;g0*S^@128XCq`NWg$^6ag^6|B;q-D!)JuajwQnA+Cb+G98c# N0VxP|q`)sI@By;bM|=PP literal 0 HcmV?d00001 diff --git a/common/src/main/java/.DS_Store b/common/src/main/java/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..3c38389f4b6ce453a087e350f406df53d2a5baad GIT binary patch literal 6148 zcmeHK%}T>S5Z-O8Ca92$(BtB{NJ=gB;30$x9)%Qou%d|(8;IG`r1nsw!DsM2d>fy~ zncWRo?O8--VE6lWXJ$A1LH36+#{EUyWz1oWSx^u;k`0F48(j?(jL30}d={lLg4TO9 zGrqqL`0Y)0%OrbUe^4C+SMGBAt z|4aei8U{lj7G=)XAIrnDRzSIdf`)Mw5-{LvmjGzsKGIfB?H9-)&NEnP#8I$cr32DM NKnX$>Dewyld;;U3My&t< literal 0 HcmV?d00001 diff --git a/common/src/main/java/com/.DS_Store b/common/src/main/java/com/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7274933556578f5532ba45ef704dae81dded9531 GIT binary patch literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~v94NJ=gB;30$x9)%Qou%ZbmHjpNzNv%+$!DsM2d>fy~ zncWQ-%vr?F!0b0WKfBovvOkP5?yQ3zV+LbP14ZO$)Cjt3Lk*RT$i)~7_T+EUr5`1J z%|L(Ah2P$03pQgJ^8mkp|A)Vf!Zfq2Pu^&@_S*ZJu4{+WNf-yo#!1$3klwiSWaEax z%=cEe!p^*GojTr1M6SJT4iRCGlOzIrs!0Vj zu4%gQt{tD8UR;g6=ltchko<9=TF8;Y65hb5EalN%#ZeMJfWNFMa|ww7Vt^PR26mVM zbtq^DJInzsofsen>KVZO!30IL6&4c3)&UJ(pE2G-L;)M$5{RZkTVWv)A|PCr0?JaZ zo)}z~gWptnw!%W9EN5KI4CAPo%f|~BvxDCh>5SVFsU-%8fhq${*){O|zl2|=_K{yL zp&l_n4E!?&cx&X299Wb(TYoGM&sr1o9ux)h3N%2#u3Q43gZoHF1ZCVr8}e+0g+v?$ S{h}O@E&_rOYKVbfVBiZyhe?V6 literal 0 HcmV?d00001 diff --git a/common/src/main/java/com/metamx/.DS_Store b/common/src/main/java/com/metamx/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..de97bb0c5e9924efedd9e0823865ec36191fac9d GIT binary patch literal 6148 zcmeHKOH0E*5Z>)W6I94W=rMRMl2S`OcnG0_McZy)Zn86s+KoL0_4TA2*P)j8vaxq5Wud<~dC4R#& zf76BE-emzxS;0cK`Tbx1G77WYvc7q%-RT}2YPzl+&u3vABr7K=;vlKUQI<0S5Z-NTO;I5ip~uB@krt(`;30$x9)%Qou%ZbmZ6G8iN$sIVgU{f5_%=R| zGrOBZFlP}v1GC@k{Oo2w$o??KxV`ZQjG2rv0~C>?Q6uQC4K;K!B9~(zyQ7DRr$wc}0Xr}xe(-a3K5 z@Z97<+L@beQrk^rodk%h$vv=TLMvNv@{k9Ap*ixDWEFl z`ia3+IrxRnvosb8RXO8wW*A4$Ts~g7oE`i^rZa9Sq?Q;U2C57+RoBGx{|0`U+DCr1 zgnGmPG4RhA;H`->v0+jAZ2hr3JZlB$6DSJi6=;BfUAY862ltV#463+58}clTg+d$! S{jwa8E&`GeYKVbfVBj0ZI7x*7 literal 0 HcmV?d00001 diff --git a/examples/query.body b/examples/query.body new file mode 100644 index 00000000000..e0607aa1554 --- /dev/null +++ b/examples/query.body @@ -0,0 +1,12 @@ +{ + "queryType": "groupBy", + "dataSource": "twitterstream", + "granularity": "all", + "dimensions": ["lang", "utc_offset"], + "aggregations":[ + { "type": "count", "name": "rows"}, + { "type": "doubleSum", "fieldName": "tweets", "name": "tweets"} + ], + "filter": { "type": "selector", "dimension": "lang", "value": "en" }, + "intervals":["2012-10-01T00:00/2020-01-01T00"] +} diff --git a/examples/search_query.body b/examples/search_query.body new file mode 100644 index 00000000000..f2feb27d4fe --- /dev/null +++ b/examples/search_query.body @@ -0,0 +1,13 @@ +{ + "queryType": "search", + "dataSource": "twitterstream", + "granularity": "all", + "searchDimensions": ["htags"], + "limit": 1, + "query": { + "type": "fragment", + "values": ["men"], + "sort": { "type": "strlen" } + }, + "intervals":["2012-10-01T00:00/2020-01-01T00"] +} diff --git a/examples/src/main/java/druid/examples/HttpTest.java b/examples/src/main/java/druid/examples/HttpTest.java new file mode 100644 index 00000000000..ca39df38bf2 --- /dev/null +++ b/examples/src/main/java/druid/examples/HttpTest.java @@ -0,0 +1,55 @@ +package druid.examples; + +import org.apache.http.*; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.HashMap; + +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/20/13 + * Time: 12:13 PM + * To change this template use File | Settings | File Templates. + */ +public class HttpTest +{ + public static void main(String[] args) throws Exception{ + URL url = new URL("http://developer.usa.gov/1usagov"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setDoInput(true); + connection.setDoOutput(true); + connection.setRequestMethod("GET"); + + BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); + String line; + ObjectMapper mapper = new ObjectMapper(); + TypeReference> typeRef = new TypeReference> () {}; + try{ + while ((line = reader.readLine())!= null){ + try{ + HashMap map=mapper.readValue(line, typeRef); + System.out.println(map); + } + catch (Exception e){ + System.out.println(e); + } + } + } + catch (IOException e){ + + } + } +} diff --git a/examples/src/main/java/druid/examples/twitter/Configuration.java b/examples/src/main/java/druid/examples/twitter/Configuration.java new file mode 100644 index 00000000000..4c6cc76f202 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/Configuration.java @@ -0,0 +1,59 @@ +package druid.examples.twitter; + +import druid.examples.twitter.HttpClientWrapperConfiguration.HttpClientWrapperConfiguration; + +import java.util.Map; + + +public interface Configuration extends HttpClientConfiguration + , HttpClientWrapperConfiguration + , java.io.Serializable { + + + boolean isDebugEnabled(); + + + Map getRequestHeaders(); + + // methods for HttpClientConfiguration + + String getHttpProxyHost(); + + String getHttpProxyUser(); + + String getHttpProxyPassword(); + + int getHttpProxyPort(); + + int getHttpConnectionTimeout(); + + int getHttpReadTimeout(); + + int getHttpStreamingReadTimeout(); + + int getHttpRetryCount(); + + int getHttpRetryIntervalSeconds(); + + int getHttpMaxTotalConnections(); + + int getHttpDefaultMaxPerRoute(); + + + String getStreamBaseURL(); + + + boolean isJSONStoreEnabled(); + + + boolean isStallWarningsEnabled(); + + int getAsyncNumThreads(); + + long getContributingTo(); + + String getDispatcherImpl(); + + String getLoggerFactory(); + +} diff --git a/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java b/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java new file mode 100644 index 00000000000..15ba09c7f6c --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java @@ -0,0 +1,57 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/** +* Static factory of Configuration. This class wraps ConfigurationFactory implementations.
+* By default, twitter4j.conf.PropertyConfigurationFactory will be used and can be changed with -Dtwitter4j.configurationFactory system property. +* +*/ +public final class ConfigurationContext { + public static final String DEFAULT_CONFIGURATION_FACTORY = "twitter4j.conf.PropertyConfigurationFactory"; + public static final String CONFIGURATION_IMPL = "twitter4j.configurationFactory"; + private static final ConfigurationFactory factory; + + static { + String CONFIG_IMPL; + try { + CONFIG_IMPL = System.getProperty(CONFIGURATION_IMPL, DEFAULT_CONFIGURATION_FACTORY); + } catch (SecurityException ignore) { + // Unsigned applets are not allowed to access System properties + CONFIG_IMPL = DEFAULT_CONFIGURATION_FACTORY; + } + + try { + factory = (ConfigurationFactory) Class.forName(CONFIG_IMPL).newInstance(); + } catch (ClassNotFoundException cnfe) { + throw new AssertionError(cnfe); + } catch (InstantiationException ie) { + throw new AssertionError(ie); + } catch (IllegalAccessException iae) { + throw new AssertionError(iae); + } + } + + + public static Configuration getInstance() { + return factory.getInstance(); + } + + public static Configuration getInstance(String configTreePath) { + return factory.getInstance(configTreePath); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java b/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java new file mode 100644 index 00000000000..d4c1acdf69e --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java @@ -0,0 +1,42 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public interface ConfigurationFactory { + /** + * returns the root configuration + * + * @return root configuration + */ + Configuration getInstance(); + + /** + * returns the configuration specified by the path + * + * @param configTreePath the path + * @return the configuratoin + */ + Configuration getInstance(String configTreePath); + + /** + * clean up resources acquired by this factory. + */ + void dispose(); +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java b/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java new file mode 100644 index 00000000000..882ca03091a --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java @@ -0,0 +1,46 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +public interface HttpClientConfiguration { + + String getHttpProxyHost(); + + int getHttpProxyPort(); + + String getHttpProxyUser(); + + String getHttpProxyPassword(); + + int getHttpConnectionTimeout(); + + int getHttpReadTimeout(); + + int getHttpRetryCount(); + + int getHttpRetryIntervalSeconds(); + + int getHttpMaxTotalConnections(); + + int getHttpDefaultMaxPerRoute(); + + boolean isPrettyDebugEnabled(); + + boolean isGZIPEnabled(); +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java b/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java new file mode 100644 index 00000000000..e419f6da6e0 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java @@ -0,0 +1,13 @@ +package druid.examples.twitter.HttpClientWrapperConfiguration; + +import druid.examples.twitter.HttpClientConfiguration; + +import java.util.Map; + +public interface HttpClientWrapperConfiguration extends HttpClientConfiguration +{ + /** + * @return request headers + */ + Map getRequestHeaders(); +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpParameter.java b/examples/src/main/java/druid/examples/twitter/HttpParameter.java new file mode 100644 index 00000000000..0ca5643df1b --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpParameter.java @@ -0,0 +1,288 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +import java.io.File; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +/** + * A data class representing HTTP Post parameter + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public final class HttpParameter implements Comparable, java.io.Serializable { + private String name = null; + private String value = null; + private File file = null; + private InputStream fileBody = null; + private static final long serialVersionUID = -8708108746980739212L; + + public HttpParameter(String name, String value) { + this.name = name; + this.value = value; + } + + public HttpParameter(String name, File file) { + this.name = name; + this.file = file; + } + + public HttpParameter(String name, String fileName, InputStream fileBody) { + this.name = name; + this.file = new File(fileName); + this.fileBody = fileBody; + } + + public HttpParameter(String name, int value) { + this.name = name; + this.value = String.valueOf(value); + } + + public HttpParameter(String name, long value) { + this.name = name; + this.value = String.valueOf(value); + } + + public HttpParameter(String name, double value) { + this.name = name; + this.value = String.valueOf(value); + } + + public HttpParameter(String name, boolean value) { + this.name = name; + this.value = String.valueOf(value); + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + public File getFile() { + return file; + } + + public InputStream getFileBody() { + return fileBody; + } + + public boolean isFile() { + return file != null; + } + + public boolean hasFileBody() { + return fileBody != null; + } + + private static final String JPEG = "image/jpeg"; + private static final String GIF = "image/gif"; + private static final String PNG = "image/png"; + private static final String OCTET = "application/octet-stream"; + + /** + * @return content-type + */ + public String getContentType() { + if (!isFile()) { + throw new IllegalStateException("not a file"); + } + String contentType; + String extensions = file.getName(); + int index = extensions.lastIndexOf("."); + if (-1 == index) { + // no extension + contentType = OCTET; + } else { + extensions = extensions.substring(extensions.lastIndexOf(".") + 1).toLowerCase(); + if (extensions.length() == 3) { + if ("gif".equals(extensions)) { + contentType = GIF; + } else if ("png".equals(extensions)) { + contentType = PNG; + } else if ("jpg".equals(extensions)) { + contentType = JPEG; + } else { + contentType = OCTET; + } + } else if (extensions.length() == 4) { + if ("jpeg".equals(extensions)) { + contentType = JPEG; + } else { + contentType = OCTET; + } + } else { + contentType = OCTET; + } + } + return contentType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof HttpParameter)) return false; + + HttpParameter that = (HttpParameter) o; + + if (file != null ? !file.equals(that.file) : that.file != null) + return false; + if (fileBody != null ? !fileBody.equals(that.fileBody) : that.fileBody != null) + return false; + if (!name.equals(that.name)) return false; + if (value != null ? !value.equals(that.value) : that.value != null) + return false; + + return true; + } + + public static boolean containsFile(HttpParameter[] params) { + boolean containsFile = false; + if (null == params) { + return false; + } + for (HttpParameter param : params) { + if (param.isFile()) { + containsFile = true; + break; + } + } + return containsFile; + } + + /*package*/ + static boolean containsFile(List params) { + boolean containsFile = false; + for (HttpParameter param : params) { + if (param.isFile()) { + containsFile = true; + break; + } + } + return containsFile; + } + + public static HttpParameter[] getParameterArray(String name, String value) { + return new HttpParameter[]{new HttpParameter(name, value)}; + } + + public static HttpParameter[] getParameterArray(String name, int value) { + return getParameterArray(name, String.valueOf(value)); + } + + public static HttpParameter[] getParameterArray(String name1, String value1 + , String name2, String value2) { + return new HttpParameter[]{new HttpParameter(name1, value1) + , new HttpParameter(name2, value2)}; + } + + public static HttpParameter[] getParameterArray(String name1, int value1 + , String name2, int value2) { + return getParameterArray(name1, String.valueOf(value1), name2, String.valueOf(value2)); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + (value != null ? value.hashCode() : 0); + result = 31 * result + (file != null ? file.hashCode() : 0); + result = 31 * result + (fileBody != null ? fileBody.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "PostParameter{" + + "name='" + name + '\'' + + ", value='" + value + '\'' + + ", file=" + file + + ", fileBody=" + fileBody + + '}'; + } + + @Override + public int compareTo(Object o) { + int compared; + HttpParameter that = (HttpParameter) o; + compared = name.compareTo(that.name); + if (0 == compared) { + compared = value.compareTo(that.value); + } + return compared; + } + + public static String encodeParameters(HttpParameter[] httpParams) { + if (null == httpParams) { + return ""; + } + StringBuilder buf = new StringBuilder(); + for (int j = 0; j < httpParams.length; j++) { + if (httpParams[j].isFile()) { + throw new IllegalArgumentException("parameter [" + httpParams[j].name + "]should be text"); + } + if (j != 0) { + buf.append("&"); + } + buf.append(encode(httpParams[j].name)) + .append("=").append(encode(httpParams[j].value)); + } + return buf.toString(); + } + + /** + * @param value string to be encoded + * @return encoded string + * @see OAuth / TestCases + * @see Space encoding - OAuth | Google Groups + * @see RFC 3986 - Uniform Resource Identifier (URI): Generic Syntax - 2.1. Percent-Encoding + */ + public static String encode(String value) { + String encoded = null; + try { + encoded = URLEncoder.encode(value, "UTF-8"); + } catch (UnsupportedEncodingException ignore) { + } + StringBuilder buf = new StringBuilder(encoded.length()); + char focus; + for (int i = 0; i < encoded.length(); i++) { + focus = encoded.charAt(i); + if (focus == '*') { + buf.append("%2A"); + } else if (focus == '+') { + buf.append("%20"); + } else if (focus == '%' && (i + 1) < encoded.length() + && encoded.charAt(i + 1) == '7' && encoded.charAt(i + 2) == 'E') { + buf.append('~'); + i += 2; + } else { + buf.append(focus); + } + } + return buf.toString(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpRequest.java b/examples/src/main/java/druid/examples/twitter/HttpRequest.java new file mode 100644 index 00000000000..9d233a423f8 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpRequest.java @@ -0,0 +1,120 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.util.Arrays; +import java.util.Map; + + +import java.util.Arrays; +import java.util.Map; + +/** + * HTTP Request parameter object + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public final class HttpRequest implements java.io.Serializable { + + private final RequestMethod method; + + private final String url; + + private final HttpParameter[] parameters; + + + private Map requestHeaders; + + private static final long serialVersionUID = -3463594029098858381L; + + + private static final HttpParameter[] NULL_PARAMETERS = new HttpParameter[0]; + + /** + * @param method Specifies the HTTP method + * @param url the request to request + * @param parameters parameters + * @param requestHeaders + */ + public HttpRequest(RequestMethod method, String url, HttpParameter[] parameters + , Map requestHeaders) { + this.method = method; + if (method != RequestMethod.POST && parameters != null && parameters.length != 0) { + this.url = url + "?" + HttpParameter.encodeParameters(parameters); + this.parameters = NULL_PARAMETERS; + } else { + this.url = url; + this.parameters = parameters; + } + this.requestHeaders = requestHeaders; + } + + public RequestMethod getMethod() { + return method; + } + + public HttpParameter[] getParameters() { + return parameters; + } + + public String getURL() { + return url; + } + + + public Map getRequestHeaders() { + return requestHeaders; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + HttpRequest that = (HttpRequest) o; + + if (!Arrays.equals(parameters, that.parameters)) return false; + if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) + return false; + if (method != null ? !method.equals(that.method) : that.method != null) + return false; + if (url != null ? !url.equals(that.url) : that.url != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = method != null ? method.hashCode() : 0; + result = 31 * result + (url != null ? url.hashCode() : 0); + result = 31 * result + (parameters != null ? Arrays.hashCode(parameters) : 0); + result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "HttpRequest{" + + "requestMethod=" + method + + ", url='" + url + '\'' + + ", postParams=" + (parameters == null ? null : Arrays.asList(parameters)) + + ", requestHeaders=" + requestHeaders + + '}'; + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponse.java b/examples/src/main/java/druid/examples/twitter/HttpResponse.java new file mode 100644 index 00000000000..ae2bc6f27d4 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponse.java @@ -0,0 +1,239 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import org.codehaus.jettison.json.JSONException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.List; +import java.util.Map; + +import java.io.*; +import java.util.List; +import java.util.Map; + +/** +* A data class representing HTTP Response +* +* @author Yusuke Yamamoto - yusuke at mac.com +*/ +public abstract class HttpResponse { + private static final Logger logger = Logger.getLogger(HttpResponseImpl.class); + protected final HttpClientConfiguration CONF; + + HttpResponse() { + this.CONF = ConfigurationContext.getInstance(); + } + + public HttpResponse(HttpClientConfiguration conf) { + this.CONF = conf; + } + + protected int statusCode; + protected String responseAsString = null; + protected InputStream is; + private boolean streamConsumed = false; + + public int getStatusCode() { + return statusCode; + } + + public abstract String getResponseHeader(String name); + + public abstract Map> getResponseHeaderFields(); + + /** + * Returns the response stream.
+ * This method cannot be called after calling asString() or asDcoument()
+ * It is suggested to call disconnect() after consuming the stream. + *

+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body stream + * @see #disconnect() + */ + public InputStream asStream() { + if (streamConsumed) { + throw new IllegalStateException("Stream has already been consumed."); + } + return is; + } + + /** + * Returns the response body as string.
+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body + */ + public String asString() throws RuntimeException { + if (null == responseAsString) { + BufferedReader br = null; + InputStream stream = null; + try { + stream = asStream(); + if (null == stream) { + return null; + } + br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); + StringBuilder buf = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + buf.append(line).append("\n"); + } + this.responseAsString = buf.toString(); + //logger.debug(responseAsString); + stream.close(); + streamConsumed = true; + } catch (IOException ioe) { + throw new RuntimeException(ioe.getMessage(), ioe); + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException ignore) { + } + } + if (br != null) { + try { + br.close(); + } catch (IOException ignore) { + } + } + disconnectForcibly(); + } + } + return responseAsString; + } + + private JSONObject json = null; + + /** + * Returns the response body as twitter4j.internal.org.json.JSONObject.
+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body as JSONObject + */ + public JSONObject asJSONObject() throws RuntimeException { + if (json == null) { + Reader reader = null; + try { + if (responseAsString == null) { + reader = asReader(); + json = new JSONObject(new JSONTokener(reader)); + } else { + json = new JSONObject(responseAsString); + } + if (CONF.isPrettyDebugEnabled()) { + //logger.debug(json.toString(1)); + } else { + //logger.debug(responseAsString != null ? responseAsString : +// json.toString()); + } + } catch (JSONException jsone) { + if (responseAsString == null) { + throw new RuntimeException(jsone.getMessage(), jsone); + } else { + throw new RuntimeException(jsone.getMessage() + ":" + this.responseAsString, jsone); + } + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException ignore) { + } + } + disconnectForcibly(); + } + } + return json; + } + + private JSONArray jsonArray = null; + + /** + * Returns the response body as twitter4j.internal.org.json.JSONArray.
+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body as twitter4j.internal.org.json.JSONArray + * @throws RuntimeException + */ + public JSONArray asJSONArray() throws RuntimeException { + if (jsonArray == null) { + Reader reader = null; + try { + if (responseAsString == null) { + reader = asReader(); + jsonArray = new JSONArray(new JSONTokener(reader)); + } else { + jsonArray = new JSONArray(responseAsString); + } + if (CONF.isPrettyDebugEnabled()) { + //logger.debug(jsonArray.toString(1)); + } else { + //logger.debug(responseAsString != null ? responseAsString : +// jsonArray.toString()); + } + } catch (JSONException jsone) { +// if (logger.isDebugEnabled()) { +// throw new RuntimeException(jsone.getMessage() + ":" + this.responseAsString, jsone); +// } else { +// throw new RuntimeException(jsone.getMessage(), jsone); +// } + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException ignore) { + } + } + disconnectForcibly(); + } + } + return jsonArray; + } + + public Reader asReader() { + try { + return new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException uee) { + return new InputStreamReader(is); + } + } + + private void disconnectForcibly() { + try { + disconnect(); + } catch (Exception ignore) { + } + } + + public abstract void disconnect() throws IOException; + + @Override + public String toString() { + return "HttpResponse{" + + "statusCode=" + statusCode + + ", responseAsString='" + responseAsString + '\'' + + ", is=" + is + + ", streamConsumed=" + streamConsumed + + '}'; + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java b/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java new file mode 100644 index 00000000000..2f92bae3d7d --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java @@ -0,0 +1,92 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/** +* @author Andrew Hedges - andrew.hedges at gmail.com +*/ +public final class HttpResponseEvent { + + private HttpRequest request; + + private HttpResponse response; + + private RuntimeException e; + + HttpResponseEvent(HttpRequest request, HttpResponse response, RuntimeException e) { + this.request = request; + this.response = response; + this.e = e; + } + + /** + * returns the request associated with the event + * + * @return the request associated with the event + */ + public HttpRequest getRequest() { + return request; + } + + /** + * returns the response associated with the event + * + * @return the response associated with the event + */ + public HttpResponse getResponse() { + return response; + } + + /** + * returns the TwitterException associated with the event + * + * @return the TwitterException associated with the event + */ + public RuntimeException getRuntimeException() { + return this.e; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + HttpResponseEvent that = (HttpResponseEvent) o; + + if (request != null ? !request.equals(that.request) : that.request != null) + return false; + if (response != null ? !response.equals(that.response) : that.response != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = request != null ? request.hashCode() : 0; + result = 31 * result + (response != null ? response.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "HttpResponseEvent{" + + "request=" + request + + ", response=" + response + + '}'; + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java b/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java new file mode 100644 index 00000000000..0faf59f50c9 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java @@ -0,0 +1,75 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; + + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; + +/** +* @author Yusuke Yamamoto - yusuke at mac.com +* @since Twitter4J 2.1.2 +*/ +public class HttpResponseImpl extends HttpResponse +{ + private HttpURLConnection con; + + HttpResponseImpl(HttpURLConnection con, HttpClientConfiguration conf) throws IOException + { + super(conf); + this.con = con; + this.statusCode = con.getResponseCode(); + if (null == (is = con.getErrorStream())) { + is = con.getInputStream(); + } + if (is != null && "gzip".equals(con.getContentEncoding())) { + // the response is gzipped + is = new StreamingGZIPInputStream(is); + } + } + + // for test purpose + /*package*/ HttpResponseImpl(String content) { + super(); + this.responseAsString = content; + } + + @Override + public String getResponseHeader(String name) { + return con.getHeaderField(name); + } + + @Override + public Map> getResponseHeaderFields() { + return con.getHeaderFields(); + } + + /** + * {@inheritDoc} + */ + @Override + public void disconnect() { + con.disconnect(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java b/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java new file mode 100644 index 00000000000..5d1d639bcae --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java @@ -0,0 +1,10 @@ +package druid.examples.twitter; + +/** + * @author Andrew Hedges - andrew.hedges at gmail.com + */ +public interface HttpResponseListener { + + public void httpResponseReceived(HttpResponseEvent event); + +} diff --git a/examples/src/main/java/druid/examples/twitter/JSONArray.java b/examples/src/main/java/druid/examples/twitter/JSONArray.java new file mode 100644 index 00000000000..897d9825826 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/JSONArray.java @@ -0,0 +1,726 @@ +package druid.examples.twitter; + +import org.codehaus.jettison.json.JSONException; + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +/** + * A JSONArray is an ordered sequence of values. Its external text form is a + * string wrapped in square brackets with commas separating the values. The + * internal form is an object having get and opt + * methods for accessing the values by index, and put methods for + * adding or replacing values. The values can be any of these types: + * Boolean, JSONArray, JSONObject, + * Number, String, or the + * JSONObject.NULL object. + *

+ * The constructor can convert a JSON text into a Java object. The + * toString method converts to JSON text. + *

+ * A get method returns a value if one can be found, and throws an + * exception if one cannot be found. An opt method returns a + * default value instead of throwing an exception, and so is useful for + * obtaining optional values. + *

+ * The generic get() and opt() methods return an + * object which you can cast or query for type. There are also typed + * get and opt methods that do type checking and type + * coercion for you. + *

+ * The texts produced by the toString methods strictly conform to + * JSON syntax rules. The constructors are more forgiving in the texts they will + * accept: + *

    + *
  • An extra , (comma) may appear just + * before the closing bracket.
    • + *
  • The null value will be inserted when there + * is , (comma) elision.
    • + *
  • Strings may be quoted with ' (single + * quote).
    • + *
  • Strings do not need to be quoted at all if they do not begin with a quote + * or single quote, and if they do not contain leading or trailing spaces, + * and if they do not contain any of these characters: + * { } [ ] / \ : , = ; # and if they do not look like numbers + * and if they are not the reserved words true, + * false, or null.
    • + *
  • Values can be separated by ; (semicolon) as + * well as by , (comma).
    • + *
  • Numbers may have the + * 0x- (hex) prefix.
    • + *
+ * + * @author JSON.org + * @version 2010-12-28 + */ +public class JSONArray { + + + /** + * The arrayList where the JSONArray's properties are kept. + */ + private ArrayList myArrayList; + + + /** + * Construct an empty JSONArray. + */ + public JSONArray() { + this.myArrayList = new ArrayList(); + } + + /** + * Construct a JSONArray from a JSONTokener. + * + * @param x A JSONTokener + * @throws JSONException If there is a syntax error. + */ + public JSONArray(JSONTokener x) throws JSONException + { + this(); + if (x.nextClean() != '[') { + throw x.syntaxError("A JSONArray text must start with '['"); + } + if (x.nextClean() != ']') { + x.back(); + for (; ; ) { + if (x.nextClean() == ',') { + x.back(); + this.myArrayList.add(twitter4j.internal.org.json.JSONObject.NULL); + } else { + x.back(); + this.myArrayList.add(x.nextValue()); + } + switch (x.nextClean()) { + case ';': + case ',': + if (x.nextClean() == ']') { + return; + } + x.back(); + break; + case ']': + return; + default: + throw x.syntaxError("Expected a ',' or ']'"); + } + } + } + } + + + /** + * Construct a JSONArray from a source JSON text. + * + * @param source A string that begins with + * [ (left bracket) + * and ends with ] (right bracket). + * @throws JSONException If there is a syntax error. + */ + public JSONArray(String source) throws JSONException { + this(new twitter4j.internal.org.json.JSONTokener(source)); + } + + + /** + * Construct a JSONArray from a Collection. + * + * @param collection A Collection. + */ + public JSONArray(Collection collection) { + this.myArrayList = new ArrayList(); + if (collection != null) { + for (Object aCollection : collection) { + this.myArrayList.add(twitter4j.internal.org.json.JSONObject.wrap(aCollection)); + } + } + } + + + /** + * Construct a JSONArray from an array + * + * @throws JSONException If not an array. + */ + public JSONArray(Object array) throws JSONException { + this(); + if (array.getClass().isArray()) { + int length = Array.getLength(array); + for (int i = 0; i < length; i += 1) { + this.put(twitter4j.internal.org.json.JSONObject.wrap(Array.get(array, i))); + } + } else { + throw new JSONException( + "JSONArray initial value should be a string or collection or array."); + } + } + + + /** + * Get the object value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return An object value. + * @throws JSONException If there is no value for the index. + */ + public Object get(int index) throws JSONException { + Object object = opt(index); + if (object == null) { + throw new JSONException("JSONArray[" + index + "] not found."); + } + return object; + } + + + /** + * Get the boolean value associated with an index. + * The string values "true" and "false" are converted to boolean. + * + * @param index The index must be between 0 and length() - 1. + * @return The truth. + * @throws JSONException If there is no value for the index or if the + * value is not convertible to boolean. + */ + public boolean getBoolean(int index) throws JSONException { + Object object = get(index); + if (object.equals(Boolean.FALSE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("false"))) { + return false; + } else if (object.equals(Boolean.TRUE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("true"))) { + return true; + } + throw new JSONException("JSONArray[" + index + "] is not a boolean."); + } + + + /** + * Get the double value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + */ + public double getDouble(int index) throws JSONException { + Object object = get(index); + try { + return object instanceof Number ? + ((Number) object).doubleValue() : + Double.parseDouble((String) object); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the int value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value is not a number. + */ + public int getInt(int index) throws JSONException { + Object object = get(index); + try { + return object instanceof Number ? + ((Number) object).intValue() : + Integer.parseInt((String) object); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the JSONArray associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return A JSONArray value. + * @throws JSONException If there is no value for the index. or if the + * value is not a JSONArray + */ + public JSONArray getJSONArray(int index) throws JSONException { + Object object = get(index); + if (object instanceof JSONArray) { + return (JSONArray) object; + } + throw new JSONException("JSONArray[" + index + + "] is not a JSONArray."); + } + + + /** + * Get the JSONObject associated with an index. + * + * @param index subscript + * @return A JSONObject value. + * @throws JSONException If there is no value for the index or if the + * value is not a JSONObject + */ + public twitter4j.internal.org.json.JSONObject getJSONObject(int index) throws JSONException { + Object object = get(index); + if (object instanceof twitter4j.internal.org.json.JSONObject) { + return (twitter4j.internal.org.json.JSONObject) object; + } + throw new JSONException("JSONArray[" + index + + "] is not a JSONObject."); + } + + + /** + * Get the long value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + */ + public long getLong(int index) throws JSONException { + Object object = get(index); + try { + return object instanceof Number ? + ((Number) object).longValue() : + Long.parseLong((String) object); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the string associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return A string value. + * @throws JSONException If there is no value for the index. + */ + public String getString(int index) throws JSONException { + Object object = get(index); + return object == twitter4j.internal.org.json.JSONObject.NULL ? null : object.toString(); + } + + + /** + * Determine if the value is null. + * + * @param index The index must be between 0 and length() - 1. + * @return true if the value at the index is null, or if there is no value. + */ + public boolean isNull(int index) { + return twitter4j.internal.org.json.JSONObject.NULL.equals(opt(index)); + } + + + /** + * Make a string from the contents of this JSONArray. The + * separator string is inserted between each element. + * Warning: This method assumes that the data structure is acyclical. + * + * @param separator A string that will be inserted between the elements. + * @return a string. + * @throws JSONException If the array contains an invalid number. + */ + public String join(String separator) throws JSONException { + int len = length(); + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < len; i += 1) { + if (i > 0) { + sb.append(separator); + } + sb.append(JSONObject.valueToString(this.myArrayList.get(i))); + } + return sb.toString(); + } + + + /** + * Get the number of elements in the JSONArray, included nulls. + * + * @return The length (or size). + */ + public int length() { + return this.myArrayList.size(); + } + + + /** + * Get the optional object value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return An object value, or null if there is no + * object at that index. + */ + public Object opt(int index) { + return (index < 0 || index >= length()) ? + null : this.myArrayList.get(index); + } + + /** + * Append a boolean value. This increases the array's length by one. + * + * @param value A boolean value. + * @return this. + */ + public JSONArray put(boolean value) { + put(value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONArray which is produced from a Collection. + * + * @param value A Collection value. + * @return this. + */ + public JSONArray put(Collection value) { + put(new JSONArray(value)); + return this; + } + + /** + * Append an int value. This increases the array's length by one. + * + * @param value An int value. + * @return this. + */ + public JSONArray put(int value) { + put(new Integer(value)); + return this; + } + + + /** + * Append an long value. This increases the array's length by one. + * + * @param value A long value. + * @return this. + */ + public JSONArray put(long value) { + put(new Long(value)); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONObject which is produced from a Map. + * + * @param value A Map value. + * @return this. + */ + public JSONArray put(Map value) { + put(new twitter4j.internal.org.json.JSONObject(value)); + return this; + } + + + /** + * Append an object value. This increases the array's length by one. + * + * @param value An object value. The value should be a + * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the + * JSONObject.NULL object. + * @return this. + */ + public JSONArray put(Object value) { + this.myArrayList.add(value); + return this; + } + + + /** + * Put or replace a boolean value in the JSONArray. If the index is greater + * than the length of the JSONArray, then null elements will be added as + * necessary to pad it out. + * + * @param index The subscript. + * @param value A boolean value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, boolean value) throws JSONException { + put(index, value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONArray which is produced from a Collection. + * + * @param index The subscript. + * @param value A Collection value. + * @return this. + * @throws JSONException If the index is negative or if the value is + * not finite. + */ + public JSONArray put(int index, Collection value) throws JSONException { + put(index, new JSONArray(value)); + return this; + } + + + /** + * Put or replace a double value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * + * @param index The subscript. + * @param value A double value. + * @return this. + * @throws JSONException If the index is negative or if the value is + * not finite. + */ + public JSONArray put(int index, double value) throws JSONException { + put(index, new Double(value)); + return this; + } + + + /** + * Put or replace an int value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * + * @param index The subscript. + * @param value An int value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, int value) throws JSONException { + put(index, new Integer(value)); + return this; + } + + + /** + * Put or replace a long value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * + * @param index The subscript. + * @param value A long value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, long value) throws JSONException { + put(index, new Long(value)); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONObject which is produced from a Map. + * + * @param index The subscript. + * @param value The Map value. + * @return this. + * @throws JSONException If the index is negative or if the the value is + * an invalid number. + */ + public JSONArray put(int index, Map value) throws JSONException { + put(index, new twitter4j.internal.org.json.JSONObject(value)); + return this; + } + + + /** + * Put or replace an object value in the JSONArray. If the index is greater + * than the length of the JSONArray, then null elements will be added as + * necessary to pad it out. + * + * @param index The subscript. + * @param value The value to put into the array. The value should be a + * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the + * JSONObject.NULL object. + * @return this. + * @throws JSONException If the index is negative or if the the value is + * an invalid number. + */ + public JSONArray put(int index, Object value) throws JSONException { + JSONObject.testValidity(value); + if (index < 0) { + throw new JSONException("JSONArray[" + index + "] not found."); + } + if (index < length()) { + this.myArrayList.set(index, value); + } else { + while (index != length()) { + put(twitter4j.internal.org.json.JSONObject.NULL); + } + put(value); + } + return this; + } + + /** + * Make a JSON text of this JSONArray. For compactness, no + * unnecessary whitespace is added. If it is not possible to produce a + * syntactically correct JSON text then null will be returned instead. This + * could occur if the array contains an invalid number. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return a printable, displayable, transmittable + * representation of the array. + */ + public String toString() { + try { + return '[' + join(",") + ']'; + } catch (Exception e) { + return null; + } + } + + + /** + * Make a prettyprinted JSON text of this JSONArray. + * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with [ (left bracket) and ending + * with ] (right bracket). + * @throws JSONException + */ + public String toString(int indentFactor) throws JSONException { + return toString(indentFactor, 0); + } + + + /** + * Make a prettyprinted JSON text of this JSONArray. + * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indention of the top level. + * @return a printable, displayable, transmittable + * representation of the array. + * @throws JSONException + */ + String toString(int indentFactor, int indent) throws JSONException { + int len = length(); + if (len == 0) { + return "[]"; + } + int i; + StringBuilder sb = new StringBuilder("["); + if (len == 1) { + sb.append( + JSONObject.valueToString( + this.myArrayList.get(0), + indentFactor, indent + )); + } else { + int newindent = indent + indentFactor; + sb.append('\n'); + for (i = 0; i < len; i += 1) { + if (i > 0) { + sb.append(",\n"); + } + for (int j = 0; j < newindent; j += 1) { + sb.append(' '); + } + sb.append( + JSONObject.valueToString( + this.myArrayList.get(i), + indentFactor, newindent + )); + } + sb.append('\n'); + for (i = 0; i < indent; i += 1) { + sb.append(' '); + } + } + sb.append(']'); + return sb.toString(); + } + + + /** + * Write the contents of the JSONArray as JSON text to a writer. + * For compactness, no whitespace is added. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return The writer. + * @throws JSONException + */ + public Writer write(Writer writer) throws JSONException { + try { + boolean b = false; + int len = length(); + + writer.write('['); + + for (int i = 0; i < len; i += 1) { + if (b) { + writer.write(','); + } + Object v = this.myArrayList.get(i); + if (v instanceof twitter4j.internal.org.json.JSONObject) { + ((JSONObject) v).write(writer); + } else if (v instanceof JSONArray) { + ((JSONArray) v).write(writer); + } else { + writer.write(JSONObject.valueToString(v)); + } + b = true; + } + writer.write(']'); + return writer; + } catch (IOException e) { + throw new JSONException(e); + } + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/JSONObject.java b/examples/src/main/java/druid/examples/twitter/JSONObject.java new file mode 100644 index 00000000000..373dcd10726 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/JSONObject.java @@ -0,0 +1,1285 @@ +package druid.examples.twitter; + + +import org.codehaus.jettison.json.JSONException; + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.ResourceBundle; +import java.util.TreeSet; + + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.*; + +/** + * A JSONObject is an unordered collection of name/value pairs. Its + * external form is a string wrapped in curly braces with colons between the + * names and values, and commas between the values and names. The internal form + * is an object having get and opt methods for + * accessing the values by name, and put methods for adding or + * replacing values by name. The values can be any of these types: + * Boolean, JSONArray, JSONObject, + * Number, String, or the JSONObject.NULL + * object. A JSONObject constructor can be used to convert an external form + * JSON text into an internal form whose values can be retrieved with the + * get and opt methods, or to convert values into a + * JSON text using the put and toString methods. + * A get method returns a value if one can be found, and throws an + * exception if one cannot be found. An opt method returns a + * default value instead of throwing an exception, and so is useful for + * obtaining optional values. + *

+ * The generic get() and opt() methods return an + * object, which you can cast or query for type. There are also typed + * get and opt methods that do type checking and type + * coercion for you. The opt methods differ from the get methods in that they + * do not throw. Instead, they return a specified value, such as null. + *

+ * The put methods add or replace values in an object. For example, + * 

myString = new JSONObject().put("JSON", "Hello, World!").toString();
+ * produces the string {"JSON": "Hello, World"}. + *

+ * The texts produced by the toString methods strictly conform to + * the JSON syntax rules. + * The constructors are more forgiving in the texts they will accept: + *

    + *
  • An extra , (comma) may appear just + * before the closing brace.
    • + *
  • Strings may be quoted with ' (single + * quote).
    • + *
  • Strings do not need to be quoted at all if they do not begin with a quote + * or single quote, and if they do not contain leading or trailing spaces, + * and if they do not contain any of these characters: + * { } [ ] / \ : , = ; # and if they do not look like numbers + * and if they are not the reserved words true, + * false, or null.
    • + *
  • Keys can be followed by = or => as well as + * by :.
    • + *
  • Values can be followed by ; (semicolon) as + * well as by , (comma).
    • + *
  • Numbers may have the 0x- (hex) prefix.
    • + *
+ * + * @author JSON.org + * @version 2010-12-28 + */ +public class JSONObject { + + /** + * JSONObject.NULL is equivalent to the value that JavaScript calls null, + * whilst Java's null is equivalent to the value that JavaScript calls + * undefined. + */ + private static final class Null { + + /** + * There is only intended to be a single instance of the NULL object, + * so the clone method returns itself. + * + * @return NULL. + */ + protected final Object clone() { + return this; + } + + /** + * A Null object is equal to the null value and to itself. + * + * @param object An object to test for nullness. + * @return true if the object parameter is the JSONObject.NULL object + * or null. + */ + public boolean equals(Object object) { + return object == null || object == this; + } + + /** + * Get the "null" string value. + * + * @return The string "null". + */ + public String toString() { + return "null"; + } + } + + + /** + * The map where the JSONObject's properties are kept. + */ + private Map map; + + + /** + * It is sometimes more convenient and less ambiguous to have a + * NULL object than to use Java's null value. + * JSONObject.NULL.equals(null) returns true. + * JSONObject.NULL.toString() returns "null". + */ + public static final Object NULL = new Null(); + + + /** + * Construct an empty JSONObject. + */ + public JSONObject() { + this.map = new HashMap(); + } + + + /** + * Construct a JSONObject from a subset of another JSONObject. + * An array of strings is used to identify the keys that should be copied. + * Missing keys are ignored. + * + * @param jo A JSONObject. + * @param names An array of strings. + */ + public JSONObject(JSONObject jo, String[] names) { + this(); + for (int i = 0; i < names.length; i += 1) { + try { + putOnce(names[i], jo.opt(names[i])); + } catch (Exception ignore) { + } + } + } + + + /** + * Construct a JSONObject from a JSONTokener. + * + * @param x A JSONTokener object containing the source string. + * @throws twitter4j.internal.org.json.JSONException If there is a syntax error in the source string + * or a duplicated key. + */ + public JSONObject(JSONTokener x) throws JSONException + { + this(); + char c; + String key; + + if (x.nextClean() != '{') { + throw x.syntaxError("A JSONObject text must begin with '{' found:" + x.nextClean()); + } + for (; ; ) { + c = x.nextClean(); + switch (c) { + case 0: + throw x.syntaxError("A JSONObject text must end with '}'"); + case '}': + return; + default: + x.back(); + key = x.nextValue().toString(); + } + +// The key is followed by ':'. We will also tolerate '=' or '=>'. + + c = x.nextClean(); + if (c == '=') { + if (x.next() != '>') { + x.back(); + } + } else if (c != ':') { + throw x.syntaxError("Expected a ':' after a key"); + } + putOnce(key, x.nextValue()); + +// Pairs are separated by ','. We will also tolerate ';'. + + switch (x.nextClean()) { + case ';': + case ',': + if (x.nextClean() == '}') { + return; + } + x.back(); + break; + case '}': + return; + default: + throw x.syntaxError("Expected a ',' or '}'"); + } + } + } + + + /** + * Construct a JSONObject from a Map. + * + * @param map A map object that can be used to initialize the contents of + * the JSONObject. + * @throws JSONException + */ + public JSONObject(Map map) { + this.map = new HashMap(); + if (map != null) { + Iterator i = map.entrySet().iterator(); + while (i.hasNext()) { + Map.Entry e = (Map.Entry) i.next(); + Object value = e.getValue(); + if (value != null) { + this.map.put(e.getKey(), wrap(value)); + } + } + } + } + + + /** + * Construct a JSONObject from an Object using bean getters. + * It reflects on all of the public methods of the object. + * For each of the methods with no parameters and a name starting + * with "get" or "is" followed by an uppercase letter, + * the method is invoked, and a key and the value returned from the getter method + * are put into the new JSONObject. + *

+ * The key is formed by removing the "get" or "is" prefix. + * If the second remaining character is not upper case, then the first + * character is converted to lower case. + *

+ * For example, if an object has a method named "getName", and + * if the result of calling object.getName() is "Larry Fine", + * then the JSONObject will contain "name": "Larry Fine". + * + * @param bean An object that has getter methods that should be used + * to make a JSONObject. + */ + public JSONObject(Object bean) { + this(); + populateMap(bean); + } + + + /** + * Construct a JSONObject from an Object, using reflection to find the + * public members. The resulting JSONObject's keys will be the strings + * from the names array, and the values will be the field values associated + * with those keys in the object. If a key is not found or not visible, + * then it will not be copied into the new JSONObject. + * + * @param object An object that has fields that should be used to make a + * JSONObject. + * @param names An array of strings, the names of the fields to be obtained + * from the object. + */ + public JSONObject(Object object, String names[]) { + this(); + Class c = object.getClass(); + for (int i = 0; i < names.length; i += 1) { + String name = names[i]; + try { + putOpt(name, c.getField(name).get(object)); + } catch (Exception ignore) { + } + } + } + + + /** + * Construct a JSONObject from a source JSON text string. + * This is the most commonly used JSONObject constructor. + * + * @param source A string beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If there is a syntax error in the source + * string or a duplicated key. + */ + public JSONObject(String source) throws JSONException { + this(new twitter4j.internal.org.json.JSONTokener(source)); + } + + + /** + * Construct a JSONObject from a ResourceBundle. + * + * @param baseName The ResourceBundle base name. + * @param locale The Locale to load the ResourceBundle for. + * @throws JSONException If any JSONExceptions are detected. + */ + public JSONObject(String baseName, Locale locale) throws JSONException { + this(); + ResourceBundle r = ResourceBundle.getBundle(baseName, locale, + Thread.currentThread().getContextClassLoader()); + +// Iterate through the keys in the bundle. + + Enumeration keys = r.getKeys(); + while (keys.hasMoreElements()) { + Object key = keys.nextElement(); + if (key instanceof String) { + +// Go through the path, ensuring that there is a nested JSONObject for each +// segment except the last. Add the value using the last segment's name into +// the deepest nested JSONObject. + + String[] path = ((String) key).split("\\."); + int last = path.length - 1; + JSONObject target = this; + for (int i = 0; i < last; i += 1) { + String segment = path[i]; + Object object = target.opt(segment); + JSONObject nextTarget = object instanceof JSONObject ? (JSONObject) object : null; + if (nextTarget == null) { + nextTarget = new JSONObject(); + target.put(segment, nextTarget); + } + target = nextTarget; + } + target.put(path[last], r.getString((String) key)); + } + } + } + + /** + * Append values to the array under a key. If the key does not exist in the + * JSONObject, then the key is put in the JSONObject with its value being a + * JSONArray containing the value parameter. If the key was already + * associated with a JSONArray, then the value parameter is appended to it. + * + * @param key A key string. + * @param value An object to be accumulated under the key. + * @return this. + * @throws JSONException If the key is null or if the current value + * associated with the key is not a JSONArray. + */ + public JSONObject append(String key, Object value) throws JSONException { + testValidity(value); + Object object = opt(key); + if (object == null) { + put(key, new JSONArray().put(value)); + } else if (object instanceof JSONArray) { + put(key, ((JSONArray) object).put(value)); + } else { + throw new JSONException("JSONObject[" + key + + "] is not a JSONArray."); + } + return this; + } + + /** + * Get the value object associated with a key. + * + * @param key A key string. + * @return The object associated with the key. + * @throws JSONException if the key is not found. + */ + public Object get(String key) throws JSONException { + if (key == null) { + throw new JSONException("Null key."); + } + Object object = opt(key); + if (object == null) { + throw new JSONException("JSONObject[" + quote(key) + + "] not found."); + } + return object; + } + + + /** + * Get the boolean value associated with a key. + * + * @param key A key string. + * @return The truth. + * @throws JSONException if the value is not a Boolean or the String "true" or "false". + */ + public boolean getBoolean(String key) throws JSONException { + Object object = get(key); + if (object.equals(Boolean.FALSE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("false"))) { + return false; + } else if (object.equals(Boolean.TRUE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("true"))) { + return true; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a Boolean."); + } + + /** + * Get the int value associated with a key. + * + * @param key A key string. + * @return The integer value. + * @throws JSONException if the key is not found or if the value cannot + * be converted to an integer. + */ + public int getInt(String key) throws JSONException { + Object object = get(key); + try { + return object instanceof Number ? + ((Number) object).intValue() : + Integer.parseInt((String) object); + } catch (Exception e) { + throw new JSONException("JSONObject[" + quote(key) + + "] is not an int."); + } + } + + + /** + * Get the JSONArray value associated with a key. + * + * @param key A key string. + * @return A JSONArray which is the value. + * @throws JSONException if the key is not found or + * if the value is not a JSONArray. + */ + public JSONArray getJSONArray(String key) throws JSONException { + Object object = get(key); + if (object instanceof JSONArray) { + return (JSONArray) object; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a JSONArray."); + } + + + /** + * Get the JSONObject value associated with a key. + * + * @param key A key string. + * @return A JSONObject which is the value. + * @throws JSONException if the key is not found or + * if the value is not a JSONObject. + */ + public JSONObject getJSONObject(String key) throws JSONException { + Object object = get(key); + if (object instanceof JSONObject) { + return (JSONObject) object; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a JSONObject."); + } + + + /** + * Get the long value associated with a key. + * + * @param key A key string. + * @return The long value. + * @throws JSONException if the key is not found or if the value cannot + * be converted to a long. + */ + public long getLong(String key) throws JSONException { + Object object = get(key); + try { + return object instanceof Number ? + ((Number) object).longValue() : + Long.parseLong((String) object); + } catch (Exception e) { + throw new JSONException("JSONObject[" + quote(key) + + "] is not a long."); + } + } + + /** + * Get the string associated with a key. + * + * @param key A key string. + * @return A string which is the value. + * @throws JSONException if the key is not found. + */ + public String getString(String key) throws JSONException { + Object object = get(key); + return object == NULL ? null : object.toString(); + } + + + /** + * Determine if the JSONObject contains a specific key. + * + * @param key A key string. + * @return true if the key exists in the JSONObject. + */ + public boolean has(String key) { + return this.map.containsKey(key); + } + + /** + * Determine if the value associated with the key is null or if there is + * no value. + * + * @param key A key string. + * @return true if there is no value associated with the key or if + * the value is the JSONObject.NULL object. + */ + public boolean isNull(String key) { + return JSONObject.NULL.equals(opt(key)); + } + + + /** + * Get an enumeration of the keys of the JSONObject. + * + * @return An iterator of the keys. + */ + public Iterator keys() { + return this.map.keySet().iterator(); + } + + + /** + * Get the number of keys stored in the JSONObject. + * + * @return The number of keys in the JSONObject. + */ + public int length() { + return this.map.size(); + } + + + /** + * Produce a JSONArray containing the names of the elements of this + * JSONObject. + * + * @return A JSONArray containing the key strings, or null if the JSONObject + * is empty. + */ + public JSONArray names() { + JSONArray ja = new JSONArray(); + Iterator keys = keys(); + while (keys.hasNext()) { + ja.put(keys.next()); + } + return ja.length() == 0 ? null : ja; + } + + /** + * Produce a string from a Number. + * + * @param number A Number + * @return A String. + * @throws JSONException If n is a non-finite number. + */ + public static String numberToString(Number number) + throws JSONException { + if (number == null) { + throw new JSONException("Null pointer"); + } + testValidity(number); + +// Shave off trailing zeros and decimal point, if possible. + + String string = number.toString(); + if (string.indexOf('.') > 0 && string.indexOf('e') < 0 && + string.indexOf('E') < 0) { + while (string.endsWith("0")) { + string = string.substring(0, string.length() - 1); + } + if (string.endsWith(".")) { + string = string.substring(0, string.length() - 1); + } + } + return string; + } + + + /** + * Get an optional value associated with a key. + * + * @param key A key string. + * @return An object which is the value, or null if there is no value. + */ + public Object opt(String key) { + return key == null ? null : this.map.get(key); + } + + + private void populateMap(Object bean) { + Class klass = bean.getClass(); + +// If klass is a System class then set includeSuperClass to false. + + boolean includeSuperClass = klass.getClassLoader() != null; + + Method[] methods = (includeSuperClass) ? + klass.getMethods() : klass.getDeclaredMethods(); + for (int i = 0; i < methods.length; i += 1) { + try { + Method method = methods[i]; + if (Modifier.isPublic(method.getModifiers())) { + String name = method.getName(); + String key = ""; + if (name.startsWith("get")) { + if (name.equals("getClass") || + name.equals("getDeclaringClass")) { + key = ""; + } else { + key = name.substring(3); + } + } else if (name.startsWith("is")) { + key = name.substring(2); + } + if (key.length() > 0 && + Character.isUpperCase(key.charAt(0)) && + method.getParameterTypes().length == 0) { + if (key.length() == 1) { + key = key.toLowerCase(); + } else if (!Character.isUpperCase(key.charAt(1))) { + key = key.substring(0, 1).toLowerCase() + + key.substring(1); + } + + Object result = method.invoke(bean, (Object[]) null); + if (result != null) { + map.put(key, wrap(result)); + } + } + } + } catch (Exception ignore) { + } + } + } + + + /** + * Put a key/boolean pair in the JSONObject. + * + * @param key A key string. + * @param value A boolean which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, boolean value) throws JSONException { + put(key, value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a key/value pair in the JSONObject, where the value will be a + * JSONArray which is produced from a Collection. + * + * @param key A key string. + * @param value A Collection value. + * @return this. + * @throws JSONException + */ + public JSONObject put(String key, Collection value) throws JSONException { + put(key, new JSONArray(value)); + return this; + } + + + /** + * Put a key/double pair in the JSONObject. + * + * @param key A key string. + * @param value A double which is the value. + * @return this. + * @throws JSONException If the key is null or if the number is invalid. + */ + public JSONObject put(String key, double value) throws JSONException { + put(key, new Double(value)); + return this; + } + + + /** + * Put a key/int pair in the JSONObject. + * + * @param key A key string. + * @param value An int which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, int value) throws JSONException { + put(key, new Integer(value)); + return this; + } + + + /** + * Put a key/long pair in the JSONObject. + * + * @param key A key string. + * @param value A long which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, long value) throws JSONException { + put(key, new Long(value)); + return this; + } + + + /** + * Put a key/value pair in the JSONObject, where the value will be a + * JSONObject which is produced from a Map. + * + * @param key A key string. + * @param value A Map value. + * @return this. + * @throws JSONException + */ + public JSONObject put(String key, Map value) throws JSONException { + put(key, new JSONObject(value)); + return this; + } + + + /** + * Put a key/value pair in the JSONObject. If the value is null, + * then the key will be removed from the JSONObject if it is present. + * + * @param key A key string. + * @param value An object which is the value. It should be of one of these + * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, + * or the JSONObject.NULL object. + * @return this. + * @throws JSONException If the value is non-finite number + * or if the key is null. + */ + public JSONObject put(String key, Object value) throws JSONException { + if (key == null) { + throw new JSONException("Null key."); + } + if (value != null) { + testValidity(value); + this.map.put(key, value); + } else { + remove(key); + } + return this; + } + + + /** + * Put a key/value pair in the JSONObject, but only if the key and the + * value are both non-null, and only if there is not already a member + * with that name. + * + * @param key + * @param value + * @return his. + * @throws JSONException if the key is a duplicate + */ + public JSONObject putOnce(String key, Object value) throws JSONException { + if (key != null && value != null) { + if (opt(key) != null) { + throw new JSONException("Duplicate key \"" + key + "\""); + } + put(key, value); + } + return this; + } + + + /** + * Put a key/value pair in the JSONObject, but only if the + * key and the value are both non-null. + * + * @param key A key string. + * @param value An object which is the value. It should be of one of these + * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, + * or the JSONObject.NULL object. + * @return this. + * @throws JSONException If the value is a non-finite number. + */ + public JSONObject putOpt(String key, Object value) throws JSONException { + if (key != null && value != null) { + put(key, value); + } + return this; + } + + + /** + * Produce a string in double quotes with backslash sequences in all the + * right places. A backslash will be inserted within = '\u0080' && c < '\u00a0') || + (c >= '\u2000' && c < '\u2100')) { + hhhh = "000" + Integer.toHexString(c); + sb.append("\\u").append(hhhh.substring(hhhh.length() - 4)); + } else { + sb.append(c); + } + } + } + sb.append('"'); + return sb.toString(); + } + + /** + * Remove a name and its value, if present. + * + * @param key The name to be removed. + * @return The value that was associated with the name, + * or null if there was no value. + */ + public Object remove(String key) { + return this.map.remove(key); + } + + /** + * Get an enumeration of the keys of the JSONObject. + * The keys will be sorted alphabetically. + * + * @return An iterator of the keys. + */ + public Iterator sortedKeys() { + return new TreeSet(this.map.keySet()).iterator(); + } + + /** + * Try to convert a string into a number, boolean, or null. If the string + * can't be converted, return the string. + * + * @param string A String. + * @return A simple JSON value. + */ + public static Object stringToValue(String string) { + if (string.equals("")) { + return string; + } + if (string.equalsIgnoreCase("true")) { + return Boolean.TRUE; + } + if (string.equalsIgnoreCase("false")) { + return Boolean.FALSE; + } + if (string.equalsIgnoreCase("null")) { + return JSONObject.NULL; + } + + /* + * If it might be a number, try converting it. + * We support the non-standard 0x- convention. + * If a number cannot be produced, then the value will just + * be a string. Note that the 0x-, plus, and implied string + * conventions are non-standard. A JSON parser may accept + * non-JSON forms as long as it accepts all correct JSON forms. + */ + + char b = string.charAt(0); + if ((b >= '0' && b <= '9') || b == '.' || b == '-' || b == '+') { + if (b == '0' && string.length() > 2 && + (string.charAt(1) == 'x' || string.charAt(1) == 'X')) { + try { + return Integer.parseInt(string.substring(2), 16); + } catch (Exception ignore) { + } + } + try { + if (string.indexOf('.') > -1 || + string.indexOf('e') > -1 || string.indexOf('E') > -1) { + return Double.valueOf(string); + } else { + Long myLong = new Long(string); + if (myLong == myLong.intValue()) { + return myLong.intValue(); + } else { + return myLong; + } + } + } catch (Exception ignore) { + } + } + return string; + } + + + /** + * Throw an exception if the object is a NaN or infinite number. + * + * @param o The object to test. + * @throws JSONException If o is a non-finite number. + */ + public static void testValidity(Object o) throws JSONException { + if (o != null) { + if (o instanceof Double) { + if (((Double) o).isInfinite() || ((Double) o).isNaN()) { + throw new JSONException( + "JSON does not allow non-finite numbers."); + } + } else if (o instanceof Float) { + if (((Float) o).isInfinite() || ((Float) o).isNaN()) { + throw new JSONException( + "JSON does not allow non-finite numbers."); + } + } + } + } + + /** + * Make a JSON text of this JSONObject. For compactness, no whitespace + * is added. If this would not result in a syntactically correct JSON text, + * then null will be returned instead. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return a printable, displayable, portable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + */ + public String toString() { + try { + Iterator keys = keys(); + StringBuilder sb = new StringBuilder("{"); + + while (keys.hasNext()) { + if (sb.length() > 1) { + sb.append(','); + } + Object o = keys.next(); + sb.append(quote(o.toString())); + sb.append(':'); + sb.append(valueToString(this.map.get(o))); + } + sb.append('}'); + return sb.toString(); + } catch (Exception e) { + return null; + } + } + + + /** + * Make a prettyprinted JSON text of this JSONObject. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @return a printable, displayable, portable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + public String toString(int indentFactor) throws JSONException { + return toString(indentFactor, 0); + } + + + /** + * Make a prettyprinted JSON text of this JSONObject. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indentation of the top level. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + String toString(int indentFactor, int indent) throws JSONException { + int i; + int length = this.length(); + if (length == 0) { + return "{}"; + } + Iterator keys = sortedKeys(); + int newindent = indent + indentFactor; + Object object; + StringBuilder sb = new StringBuilder("{"); + if (length == 1) { + object = keys.next(); + sb.append(quote(object.toString())); + sb.append(": "); + sb.append(valueToString(this.map.get(object), indentFactor, + indent)); + } else { + while (keys.hasNext()) { + object = keys.next(); + if (sb.length() > 1) { + sb.append(",\n"); + } else { + sb.append('\n'); + } + for (i = 0; i < newindent; i += 1) { + sb.append(' '); + } + sb.append(quote(object.toString())); + sb.append(": "); + sb.append(valueToString(this.map.get(object), indentFactor, + newindent)); + } + if (sb.length() > 1) { + sb.append('\n'); + for (i = 0; i < indent; i += 1) { + sb.append(' '); + } + } + } + sb.append('}'); + return sb.toString(); + } + + + /** + * Make a JSON text of an Object value. If the object has an + * value.toJSONString() method, then that method will be used to produce + * the JSON text. The method is required to produce a strictly + * conforming text. If the object does not contain a toJSONString + * method (which is the most common case), then a text will be + * produced by other means. If the value is an array or Collection, + * then a JSONArray will be made from it and its toJSONString method + * will be called. If the value is a MAP, then a JSONObject will be made + * from it and its toJSONString method will be called. Otherwise, the + * value's toString method will be called, and the result will be quoted. + *

+ *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param value The value to be serialized. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the value is or contains an invalid number. + */ + public static String valueToString(Object value) throws JSONException { + if (value == null || value.equals(null)) { + return "null"; + } + if (value instanceof Number) { + return numberToString((Number) value); + } + if (value instanceof Boolean || value instanceof JSONObject || + value instanceof JSONArray) { + return value.toString(); + } + if (value instanceof Map) { + return new JSONObject((Map) value).toString(); + } + if (value instanceof Collection) { + return new JSONArray((Collection) value).toString(); + } + if (value.getClass().isArray()) { + return new JSONArray(value).toString(); + } + return quote(value.toString()); + } + + + /** + * Make a prettyprinted JSON text of an object value. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param value The value to be serialized. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indentation of the top level. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + static String valueToString(Object value, int indentFactor, int indent) + throws JSONException { + if (value == null || value.equals(null)) { + return "null"; + } + if (value instanceof Number) { + return numberToString((Number) value); + } + if (value instanceof Boolean) { + return value.toString(); + } + if (value instanceof JSONObject) { + return ((JSONObject) value).toString(indentFactor, indent); + } + if (value instanceof JSONArray) { + return ((JSONArray) value).toString(indentFactor, indent); + } + if (value instanceof Map) { + return new JSONObject((Map) value).toString(indentFactor, indent); + } + if (value instanceof Collection) { + return new JSONArray((Collection) value).toString(indentFactor, indent); + } + if (value.getClass().isArray()) { + return new JSONArray(value).toString(indentFactor, indent); + } + return quote(value.toString()); + } + + + /** + * Wrap an object, if necessary. If the object is null, return the NULL + * object. If it is an array or collection, wrap it in a JSONArray. If + * it is a map, wrap it in a JSONObject. If it is a standard property + * (Double, String, et al) then it is already wrapped. Otherwise, if it + * comes from one of the java packages, turn it into a string. And if + * it doesn't, try to wrap it in a JSONObject. If the wrapping fails, + * then null is returned. + * + * @param object The object to wrap + * @return The wrapped value + */ + public static Object wrap(Object object) { + try { + if (object == null) { + return NULL; + } + if (object instanceof JSONObject || object instanceof JSONArray || + NULL.equals(object) || + object instanceof Byte || object instanceof Character || + object instanceof Short || object instanceof Integer || + object instanceof Long || object instanceof Boolean || + object instanceof Float || object instanceof Double || + object instanceof String) { + return object; + } + + if (object instanceof Collection) { + return new JSONArray((Collection) object); + } + if (object.getClass().isArray()) { + return new JSONArray(object); + } + if (object instanceof Map) { + return new JSONObject((Map) object); + } + Package objectPackage = object.getClass().getPackage(); + String objectPackageName = (objectPackage != null ? objectPackage.getName() : ""); + if (objectPackageName.startsWith("java.") || + objectPackageName.startsWith("javax.") || + object.getClass().getClassLoader() == null) { + return object.toString(); + } + return new JSONObject(object); + } catch (Exception exception) { + return null; + } + } + + + /** + * Write the contents of the JSONObject as JSON text to a writer. + * For compactness, no whitespace is added. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return The writer. + * @throws JSONException + */ + public Writer write(Writer writer) throws JSONException { + try { + boolean commanate = false; + Iterator keys = keys(); + writer.write('{'); + + while (keys.hasNext()) { + if (commanate) { + writer.write(','); + } + Object key = keys.next(); + writer.write(quote(key.toString())); + writer.write(':'); + Object value = this.map.get(key); + if (value instanceof JSONObject) { + ((JSONObject) value).write(writer); + } else if (value instanceof JSONArray) { + ((JSONArray) value).write(writer); + } else { + writer.write(valueToString(value)); + } + commanate = true; + } + writer.write('}'); + return writer; + } catch (IOException exception) { + throw new JSONException(exception); + } + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/JSONTokener.java b/examples/src/main/java/druid/examples/twitter/JSONTokener.java new file mode 100644 index 00000000000..db694a335a1 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/JSONTokener.java @@ -0,0 +1,359 @@ +package druid.examples.twitter; + +import org.codehaus.jettison.json.JSONException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.StringReader; + + +import java.io.*; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/** + * A JSONTokener takes a source string and extracts characters and tokens from + * it. It is used by the JSONObject and JSONArray constructors to parse + * JSON source strings. + * + * @author JSON.org + * @version 2010-12-24 + */ +public class JSONTokener { + + private int character; + private boolean eof; + private int index; + private int line; + private char previous; + private Reader reader; + private boolean usePrevious; + + + /** + * Construct a JSONTokener from a Reader. + * + * @param reader A reader. + */ + public JSONTokener(Reader reader) { + this.reader = reader.markSupported() ? + reader : new BufferedReader(reader); + this.eof = false; + this.usePrevious = false; + this.previous = 0; + this.index = 0; + this.character = 1; + this.line = 1; + } + + + /** + * Construct a JSONTokener from an InputStream. + */ + public JSONTokener(InputStream inputStream) throws JSONException + { + this(new InputStreamReader(inputStream)); + } + + + /** + * Construct a JSONTokener from a string. + * + * @param s A source string. + */ + public JSONTokener(String s) { + this(new StringReader(s)); + } + + + /** + * Back up one character. This provides a sort of lookahead capability, + * so that you can test for a digit or letter before attempting to parse + * the next number or identifier. + */ + public void back() throws JSONException { + if (usePrevious || index <= 0) { + throw new JSONException("Stepping back two steps is not supported"); + } + this.index -= 1; + this.character -= 1; + this.usePrevious = true; + this.eof = false; + } + + + public boolean end() { + return eof && !usePrevious; + } + + + /** + * Determine if the source string still contains characters that next() + * can consume. + * + * @return true if not yet at the end of the source. + */ + public boolean more() throws JSONException { + next(); + if (end()) { + return false; + } + back(); + return true; + } + + + /** + * Get the next character in the source string. + * + * @return The next character, or 0 if past the end of the source string. + */ + public char next() throws JSONException { + int c; + if (this.usePrevious) { + this.usePrevious = false; + c = this.previous; + } else { + try { + c = this.reader.read(); + } catch (IOException exception) { + throw new JSONException(exception); + } + + if (c <= 0) { // End of stream + this.eof = true; + c = 0; + } + } + this.index += 1; + if (this.previous == '\r') { + this.line += 1; + this.character = c == '\n' ? 0 : 1; + } else if (c == '\n') { + this.line += 1; + this.character = 0; + } else { + this.character += 1; + } + this.previous = (char) c; + return this.previous; + } + + + /** + * Consume the next character, and check that it matches a specified + * character. + * + * @param c The character to match. + * @return The character. + * @throws JSONException if the character does not match. + */ + public char next(char c) throws JSONException { + char n = next(); + if (n != c) { + throw syntaxError("Expected '" + c + "' and instead saw '" + + n + "'"); + } + return n; + } + + + /** + * Get the next n characters. + * + * @param n The number of characters to take. + * @return A string of n characters. + * @throws JSONException Substring bounds error if there are not + * n characters remaining in the source string. + */ + public String next(int n) throws JSONException { + if (n == 0) { + return ""; + } + + char[] chars = new char[n]; + int pos = 0; + + while (pos < n) { + chars[pos] = next(); + if (end()) { + throw syntaxError("Substring bounds error"); + } + pos += 1; + } + return new String(chars); + } + + + /** + * Get the next char in the string, skipping whitespace. + * + * @return A character, or 0 if there are no more characters. + * @throws JSONException + */ + public char nextClean() throws JSONException { + for (; ; ) { + char c = next(); + if (c == 0 || c > ' ') { + return c; + } + } + } + + + /** + * Return the characters up to the next close quote character. + * Backslash processing is done. The formal JSON format does not + * allow strings in single quotes, but an implementation is allowed to + * accept them. + * + * @param quote The quoting character, either + * " (double quote) or + * ' (single quote). + * @return A String. + * @throws JSONException Unterminated string. + */ + public String nextString(char quote) throws JSONException { + char c; + StringBuilder sb = new StringBuilder(); + for (; ; ) { + c = next(); + switch (c) { + case 0: + case '\n': + case '\r': + throw syntaxError("Unterminated string"); + case '\\': + c = next(); + switch (c) { + case 'b': + sb.append('\b'); + break; + case 't': + sb.append('\t'); + break; + case 'n': + sb.append('\n'); + break; + case 'f': + sb.append('\f'); + break; + case 'r': + sb.append('\r'); + break; + case 'u': + sb.append((char) Integer.parseInt(next(4), 16)); + break; + case '"': + case '\'': + case '\\': + case '/': + sb.append(c); + break; + default: + throw syntaxError("Illegal escape."); + } + break; + default: + if (c == quote) { + return sb.toString(); + } + sb.append(c); + } + } + } + + /** + * Get the next value. The value can be a Boolean, Double, Integer, + * JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object. + * + * @return An object. + * @throws JSONException If syntax error. + */ + public Object nextValue() throws JSONException { + char c = nextClean(); + String string; + + switch (c) { + case '"': + case '\'': + return nextString(c); + case '{': + back(); + return new JSONObject(this); + case '[': + back(); + return new JSONArray(this); + } + + /* + * Handle unquoted text. This could be the values true, false, or + * null, or it can be a number. An implementation (such as this one) + * is allowed to also accept non-standard forms. + * + * Accumulate characters until we reach the end of the text or a + * formatting character. + */ + + StringBuilder sb = new StringBuilder(); + while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) { + sb.append(c); + c = next(); + } + back(); + + string = sb.toString().trim(); + if (string.equals("")) { + throw syntaxError("Missing value"); + } + return JSONObject.stringToValue(string); + } + + + + /** + * Make a JSONException to signal a syntax error. + * + * @param message The error message. + * @return A JSONException object, suitable for throwing + */ + public JSONException syntaxError(String message) { + return new JSONException(message + toString()); + } + + + /** + * Make a printable string of this JSONTokener. + * + * @return " at {index} [character {character} line {line}]" + */ + public String toString() { + return " at " + index + " [character " + this.character + " line " + + this.line + "]"; + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/Logger.java b/examples/src/main/java/druid/examples/twitter/Logger.java new file mode 100644 index 00000000000..0f9a4eb3769 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/Logger.java @@ -0,0 +1,180 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + + + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + * @since Twitter4J 2.1.0 + */ +public abstract class Logger { + private static final LoggerFactory LOGGER_FACTORY; + private static final String LOGGER_FACTORY_IMPLEMENTATION = "twitter4j.loggerFactory"; + + static { + LoggerFactory loggerFactory = null; + // -Dtwitter4j.debug=true -Dtwitter4j.loggerFactory=twitter4j.internal.logging.StdOutLoggerFactory + String loggerFactoryImpl = System.getProperty(LOGGER_FACTORY_IMPLEMENTATION); + if (loggerFactoryImpl != null) { + loggerFactory = getLoggerFactoryIfAvailable(loggerFactoryImpl, loggerFactoryImpl); + } + + Configuration conf = ConfigurationContext.getInstance(); + // configuration in twitter4j.properties + // loggerFactory=twitter4j.internal.logging.StdOutLoggerFactory + loggerFactoryImpl = conf.getLoggerFactory(); + if (loggerFactoryImpl != null) { + loggerFactory = getLoggerFactoryIfAvailable(loggerFactoryImpl, loggerFactoryImpl); + } + // use SLF4J if it's found in the classpath + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("org.slf4j.impl.StaticLoggerBinder", "twitter4j.internal.logging.SLF4JLoggerFactory"); + } + // otherwise, use commons-logging if it's found in the classpath + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("org.apache.commons.logging.Log", "twitter4j.internal.logging.CommonsLoggingLoggerFactory"); + } + // otherwise, use log4j if it's found in the classpath + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("org.apache.log4j.Logger", "twitter4j.internal.logging.Log4JLoggerFactory"); + } + // on Google App Engine, use java.util.logging + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("com.google.appengine.api.urlfetch.URLFetchService", "twitter4j.internal.logging.JULLoggerFactory"); + } + // otherwise, use the default logger +// if (null == loggerFactory) { +// loggerFactory = new StdOutLoggerFactory(); +// } + LOGGER_FACTORY = loggerFactory; + + try { + Method method = conf.getClass().getMethod("dumpConfiguration", new Class[]{}); + method.setAccessible(true); + method.invoke(conf); + } catch (IllegalAccessException ignore) { + } catch (InvocationTargetException ignore) { + } catch (NoSuchMethodException ignore) { + } + } + + private static LoggerFactory getLoggerFactoryIfAvailable(String checkClassName, String implementationClass) { + try { + Class.forName(checkClassName); + return (LoggerFactory) Class.forName(implementationClass).newInstance(); + } catch (ClassNotFoundException ignore) { + } catch (InstantiationException e) { + throw new AssertionError(e); + } catch (SecurityException ignore) { + // Unsigned applets are not allowed to access System properties + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } + return null; + } + + /** + * Returns a Logger instance associated with the specified class. + * + * @param clazz class + * @return logger instance + */ + public static Logger getLogger(Class clazz) { + return LOGGER_FACTORY.getLogger(clazz); + } + + /** + * tests if debug level logging is enabled + * + * @return if debug level logging is enabled + */ + public abstract boolean isDebugEnabled(); + + /** + * tests if info level logging is enabled + * + * @return if info level logging is enabled + */ + public abstract boolean isInfoEnabled(); + + /** + * tests if warn level logging is enabled + * + * @return if warn level logging is enabled + */ + public abstract boolean isWarnEnabled(); + + /** + * tests if error level logging is enabled + * + * @return if error level logging is enabled + */ + public abstract boolean isErrorEnabled(); + + /** + * @param message message + */ + public abstract void debug(String message); + + /** + * @param message message + * @param message2 message2 + */ + public abstract void debug(String message, String message2); + + /** + * @param message message + */ + public abstract void info(String message); + + /** + * @param message message + * @param message2 message2 + */ + public abstract void info(String message, String message2); + + /** + * @param message message + */ + public abstract void warn(String message); + + /** + * @param message message + * @param message2 message2 + */ + public abstract void warn(String message, String message2); + + /** + * @param message message + */ + public abstract void error(String message); + + /** + * @param message message + * @param th throwable + */ + public abstract void error(String message, Throwable th); + +} diff --git a/examples/src/main/java/druid/examples/twitter/LoggerFactory.java b/examples/src/main/java/druid/examples/twitter/LoggerFactory.java new file mode 100644 index 00000000000..5dece70aedc --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/LoggerFactory.java @@ -0,0 +1,34 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import twitter4j.internal.logging.*; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + * @since Twitter4J 2.1.1 + */ +public abstract class LoggerFactory { + + /** + * Returns a logger associated with the specified class. + * + * @param clazz class + * @return a logger instance + */ + public abstract Logger getLogger(Class clazz); +} diff --git a/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java b/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java new file mode 100644 index 00000000000..a69da55a7e1 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java @@ -0,0 +1,325 @@ +package druid.examples.twitter; +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectStreamException; +import java.util.ArrayList; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import java.io.*; +import java.util.ArrayList; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public final class PropertyConfiguration extends WebConfigurationBase implements java.io.Serializable { + + public static final String DEBUG = "debug"; + public static final String HTTP_USER_AGENT = "http.userAgent"; + + public static final String HTTP_USE_SSL = "http.useSSL"; + public static final String HTTP_PRETTY_DEBUG = "http.prettyDebug"; + public static final String HTTP_GZIP = "http.gzip"; + public static final String HTTP_PROXY_HOST = "http.proxyHost"; + public static final String HTTP_PROXY_HOST_FALLBACK = "http.proxyHost"; + public static final String HTTP_PROXY_USER = "http.proxyUser"; + public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword"; + public static final String HTTP_PROXY_PORT = "http.proxyPort"; + public static final String HTTP_PROXY_PORT_FALLBACK = "http.proxyPort"; + public static final String HTTP_CONNECTION_TIMEOUT = "http.connectionTimeout"; + public static final String HTTP_READ_TIMEOUT = "http.readTimeout"; + + public static final String HTTP_STREAMING_READ_TIMEOUT = "http.streamingReadTimeout"; + + public static final String HTTP_RETRY_COUNT = "http.retryCount"; + public static final String HTTP_RETRY_INTERVAL_SECS = "http.retryIntervalSecs"; + + public static final String HTTP_MAX_TOTAL_CONNECTIONS = "http.maxTotalConnections"; + public static final String HTTP_DEFAULT_MAX_PER_ROUTE = "http.defaultMaxPerRoute"; + + + public static final String STREAM_BASE_URL = "streamBaseURL"; + + public static final String ASYNC_NUM_THREADS = "async.numThreads"; + public static final String CONTRIBUTING_TO = "contributingTo"; + public static final String ASYNC_DISPATCHER_IMPL = "async.dispatcherImpl"; + public static final String LOGGER_FACTORY = "loggerFactory"; + public static final String JSON_STORE_ENABLED = "jsonStoreEnabled"; + public static final String STREAM_STALL_WARNINGS_ENABLED = "stream.enableStallWarnings"; + + // hidden portion + private static final long serialVersionUID = 6458764415636588373L; + + public PropertyConfiguration(InputStream is) { + super(); + Properties props = new Properties(); + loadProperties(props, is); + setFieldsWithTreePath(props, "/"); + } + + public PropertyConfiguration(Properties props) { + this(props, "/"); + } + + public PropertyConfiguration(Properties props, String treePath) { + super(); + setFieldsWithTreePath(props, treePath); + } + + PropertyConfiguration(String treePath) { + super(); + Properties props; + // load from system properties + try { + props = (Properties) System.getProperties().clone(); + try { + Map envMap = System.getenv(); + for(String key :envMap.keySet()){ + props.setProperty(key, envMap.get(key)); + } + }catch(SecurityException ignore){} + normalize(props); + } catch (SecurityException ignore) { + // Unsigned applets are not allowed to access System properties + props = new Properties(); + } +// final String TWITTER4J_PROPERTIES = "twitter4j.properties"; +// // override System properties with ./twitter4j.properties in the classpath +// loadProperties(props, "." + File.separatorChar + TWITTER4J_PROPERTIES); +// // then, override with /twitter4j.properties in the classpath +// loadProperties(props, twitter4j.conf.Configuration.class.getResourceAsStream("/" + TWITTER4J_PROPERTIES)); +// // then, override with /WEB/INF/twitter4j.properties in the classpath +// loadProperties(props, twitter4j.conf.Configuration.class.getResourceAsStream("/WEB-INF/" + TWITTER4J_PROPERTIES)); +// // for Google App Engine +// try { +// loadProperties(props, new FileInputStream("WEB-INF/" + TWITTER4J_PROPERTIES)); +// } catch (SecurityException ignore) { +// } catch (FileNotFoundException ignore) { +// } + + setFieldsWithTreePath(props, treePath); + } + + /** + * Creates a root PropertyConfiguration. This constructor is equivalent to new PropertyConfiguration("/"). + */ + PropertyConfiguration() { + this("/"); + } + + private boolean notNull(Properties props, String prefix, String name) { + return props.getProperty(prefix + name) != null; + } + + private boolean loadProperties(Properties props, String path) { + FileInputStream fis = null; + try { + File file = new File(path); + if (file.exists() && file.isFile()) { + fis = new FileInputStream(file); + props.load(fis); + normalize(props); + return true; + } + } catch (Exception ignore) { + } finally { + try { + if (fis != null) { + fis.close(); + } + } catch (IOException ignore) { + + } + } + return false; + } + + private boolean loadProperties(Properties props, InputStream is) { + try { + props.load(is); + normalize(props); + return true; + } catch (Exception ignore) { + } + return false; + } + + private void normalize(Properties props) { + Set keys = props.keySet(); + ArrayList toBeNormalized = new ArrayList(10); + for (Object key : keys) { + String keyStr = (String) key; + if (-1 != (keyStr.indexOf("twitter4j."))) { + toBeNormalized.add(keyStr); + } + } + for (String keyStr : toBeNormalized) { + String property = props.getProperty(keyStr); + int index = keyStr.indexOf("twitter4j."); + String newKey = keyStr.substring(0, index) + keyStr.substring(index + 10); + props.setProperty(newKey, property); + } + } + + /** + * passing "/foo/bar" as treePath will result:
+ * 1. load [twitter4j.]restBaseURL
+ * 2. override the value with foo.[twitter4j.]restBaseURL
+ * 3. override the value with foo.bar.[twitter4j.]restBaseURL
+ * + * @param props properties to be loaded + * @param treePath the path + */ + private void setFieldsWithTreePath(Properties props, String treePath) { + setFieldsWithPrefix(props, ""); + String[] splitArray = z_InternalStringUtil.split(treePath, "/"); + String prefix = null; + for (String split : splitArray) { + if (!"".equals(split)) { + if (null == prefix) { + prefix = split + "."; + } else { + prefix += split + "."; + } + setFieldsWithPrefix(props, prefix); + } + } + } + + private void setFieldsWithPrefix(Properties props, String prefix) { + if (notNull(props, prefix, DEBUG)) { + setDebug(getBoolean(props, prefix, DEBUG)); + } + + if (notNull(props, prefix, HTTP_USE_SSL)) { + setUseSSL(getBoolean(props, prefix, HTTP_USE_SSL)); + } + if (notNull(props, prefix, HTTP_PRETTY_DEBUG)) { + setPrettyDebugEnabled(getBoolean(props, prefix, HTTP_PRETTY_DEBUG)); + } + if (notNull(props, prefix, HTTP_GZIP)) { + setGZIPEnabled(getBoolean(props, prefix, HTTP_GZIP)); + } + if (notNull(props, prefix, HTTP_PROXY_HOST)) { + setHttpProxyHost(getString(props, prefix, HTTP_PROXY_HOST)); + } else if (notNull(props, prefix, HTTP_PROXY_HOST_FALLBACK)) { + setHttpProxyHost(getString(props, prefix, HTTP_PROXY_HOST_FALLBACK)); + } + if (notNull(props, prefix, HTTP_PROXY_USER)) { + setHttpProxyUser(getString(props, prefix, HTTP_PROXY_USER)); + } + if (notNull(props, prefix, HTTP_PROXY_PASSWORD)) { + setHttpProxyPassword(getString(props, prefix, HTTP_PROXY_PASSWORD)); + } + if (notNull(props, prefix, HTTP_PROXY_PORT)) { + setHttpProxyPort(getIntProperty(props, prefix, HTTP_PROXY_PORT)); + } else if (notNull(props, prefix, HTTP_PROXY_PORT_FALLBACK)) { + setHttpProxyPort(getIntProperty(props, prefix, HTTP_PROXY_PORT_FALLBACK)); + } + if (notNull(props, prefix, HTTP_CONNECTION_TIMEOUT)) { + setHttpConnectionTimeout(getIntProperty(props, prefix, HTTP_CONNECTION_TIMEOUT)); + } + if (notNull(props, prefix, HTTP_READ_TIMEOUT)) { + setHttpReadTimeout(getIntProperty(props, prefix, HTTP_READ_TIMEOUT)); + } + if (notNull(props, prefix, HTTP_STREAMING_READ_TIMEOUT)) { + setHttpStreamingReadTimeout(getIntProperty(props, prefix, HTTP_STREAMING_READ_TIMEOUT)); + } + if (notNull(props, prefix, HTTP_RETRY_COUNT)) { + setHttpRetryCount(getIntProperty(props, prefix, HTTP_RETRY_COUNT)); + } + if (notNull(props, prefix, HTTP_RETRY_INTERVAL_SECS)) { + setHttpRetryIntervalSeconds(getIntProperty(props, prefix, HTTP_RETRY_INTERVAL_SECS)); + } + if (notNull(props, prefix, HTTP_MAX_TOTAL_CONNECTIONS)) { + setHttpMaxTotalConnections(getIntProperty(props, prefix, HTTP_MAX_TOTAL_CONNECTIONS)); + } + if (notNull(props, prefix, HTTP_DEFAULT_MAX_PER_ROUTE)) { + setHttpDefaultMaxPerRoute(getIntProperty(props, prefix, HTTP_DEFAULT_MAX_PER_ROUTE)); + } + if (notNull(props, prefix, ASYNC_NUM_THREADS)) { + setAsyncNumThreads(getIntProperty(props, prefix, ASYNC_NUM_THREADS)); + } + if (notNull(props, prefix, CONTRIBUTING_TO)) { + setContributingTo(getLongProperty(props, prefix, CONTRIBUTING_TO)); + } + if (notNull(props, prefix, ASYNC_DISPATCHER_IMPL)) { + setDispatcherImpl(getString(props, prefix, ASYNC_DISPATCHER_IMPL)); + } + if (notNull(props, prefix, HTTP_USER_AGENT)) { + setUserAgent(getString(props, prefix, HTTP_USER_AGENT)); + } + + + if (notNull(props, prefix, STREAM_BASE_URL)) { + setStreamBaseURL(getString(props, prefix, STREAM_BASE_URL)); + } + if (notNull(props, prefix, LOGGER_FACTORY)) { + setLoggerFactory(getString(props, prefix, LOGGER_FACTORY)); + } + if (notNull(props, prefix, JSON_STORE_ENABLED)) { + setJSONStoreEnabled(getBoolean(props, prefix, JSON_STORE_ENABLED)); + } + if (notNull(props, prefix, STREAM_STALL_WARNINGS_ENABLED)) { + setStallWarningsEnabled(getBoolean(props, prefix, STREAM_STALL_WARNINGS_ENABLED)); + } + + cacheInstance(); + } + + protected boolean getBoolean(Properties props, String prefix, String name) { + String value = props.getProperty(prefix + name); + return Boolean.valueOf(value); + } + + protected int getIntProperty(Properties props, String prefix, String name) { + String value = props.getProperty(prefix + name); + try { + return Integer.parseInt(value); + } catch (NumberFormatException nfe) { + return -1; + } + } + + protected long getLongProperty(Properties props, String prefix, String name) { + String value = props.getProperty(prefix + name); + try { + return Long.parseLong(value); + } catch (NumberFormatException nfe) { + return -1L; + } + } + + protected String getString(Properties props, String prefix, String name) { + return props.getProperty(prefix + name); + } + + // assures equality after deserialization + protected Object readResolve() throws ObjectStreamException + { + return super.readResolve(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java b/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java new file mode 100644 index 00000000000..7926874be2a --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java @@ -0,0 +1,61 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * ConfigurationFactory implementation for PropertyConfiguration. + * Currently getInstance calls concrete constructor each time. No caching at all. + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +class PropertyConfigurationFactory implements ConfigurationFactory { + private static final PropertyConfiguration ROOT_CONFIGURATION; + + static { + ROOT_CONFIGURATION = new PropertyConfiguration(); + // calling ROOT_CONFIGURATION.dumpConfiguration() will cause ExceptionInInitializerError as Logger has not been initialized. + // as a quick and dirty solution, static initializer of twitter4j.internal.logging.Logger will call dumpConfiguration() on behalf. + } + + /** + * {@inheritDoc} + */ + @Override + public Configuration getInstance() { + return ROOT_CONFIGURATION; + } + + // It may be preferable to cache the config instance + + /** + * {@inheritDoc} + */ + @Override + public Configuration getInstance(String configTreePath) { + PropertyConfiguration conf = new PropertyConfiguration(configTreePath); + conf.dumpConfiguration(); + return conf; + } + + /** + * {@inheritDoc} + */ + @Override + public void dispose() { + // nothing to do for property based configuration + } +} diff --git a/examples/src/main/java/druid/examples/twitter/RequestMethod.java b/examples/src/main/java/druid/examples/twitter/RequestMethod.java new file mode 100644 index 00000000000..16fea4c21b0 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/RequestMethod.java @@ -0,0 +1,26 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @author Dan Checkoway - dcheckoway at gmail.com + */ +public enum RequestMethod { + GET, POST, DELETE, HEAD, PUT +} + diff --git a/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java b/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java new file mode 100644 index 00000000000..ecbd9e9b1f0 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java @@ -0,0 +1,53 @@ +package druid.examples.twitter; +/* + * Copyright 2007 Yusuke Yamamoto + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +final class StreamingGZIPInputStream extends GZIPInputStream +{ + + private final InputStream wrapped; + + public StreamingGZIPInputStream(InputStream is) throws IOException + { + super(is); + wrapped = is; + } + + /** + * Overrides behavior of GZIPInputStream which assumes we have all the data available + * which is not true for streaming. We instead rely on the underlying stream to tell us + * how much data is available. + *

+ * Programs should not count on this method to return the actual number + * of bytes that could be read without blocking. + * + * @return - whatever the wrapped InputStream returns + * @throws IOException if an I/O error occurs. + */ + @Override + public int available() throws IOException { + return wrapped.available(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java index e87b5e44de2..ea1d8331924 100644 --- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java @@ -139,7 +139,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { public void onStatus(Status status) { // time to stop? - if (Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { diff --git a/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java new file mode 100644 index 00000000000..7f880d23959 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java @@ -0,0 +1,173 @@ +//package druid.examples.twitter; +// +//import com.fasterxml.jackson.annotation.JsonCreator; +//import com.google.common.collect.Maps; +//import com.metamx.common.logger.Logger; +//import com.metamx.druid.input.InputRow; +//import com.metamx.druid.input.MapBasedInputRow; +//import com.metamx.druid.realtime.firehose.Firehose; +//import com.metamx.druid.realtime.firehose.FirehoseFactory; +//import org.codehaus.jackson.annotate.JsonProperty; +//import org.codehaus.jackson.annotate.JsonTypeName; +//import twitter4j.ConnectionLifeCycleListener; +//import twitter4j.Status; +// +//import java.io.IOException; +//import java.util.LinkedList; +//import java.util.Map; +//import java.util.concurrent.ArrayBlockingQueue; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.TimeUnit; +// +//import static java.lang.Thread.sleep; +// +///** +//* Created with IntelliJ IDEA. +//* User: dhruvparthasarathy +//* Date: 6/18/13 +//* Time: 4:52 PM +//* To change this template use File | Settings | File Templates. +//*/ +//@JsonTypeName("Gov") +// +//public class USGovFirehoseFactory implements FirehoseFactory +//{ +// private static final Logger log = new Logger(TwitterSpritzerFirehoseFactory.class); +// +// private final int maxEventCount; +// +// private final int rowCount; +// +// @JsonCreator +// public USGovFirehoseFactory( +// @JsonProperty("maxEventCount") Integer maxEventCount, +// @JsonProperty("rowCount") Integer rowCount +// ) +// { +// this.maxEventCount=maxEventCount; +// this.rowCount=rowCount; +// log.info("maxEventCount=" + ((maxEventCount <= 0) ? "no limit" : maxEventCount)); +// log.info("rowCount=" + ((rowCount <= 0) ? "no limit" : rowCount)); +// } +// +// @Override +// public Firehose connect() throws IOException +// { +// final LinkedList dimensions = new LinkedList(); +// final int QUEUE_SIZE = 2000; +// final BlockingQueue queue = new ArrayBlockingQueue(QUEUE_SIZE); +// dimensions.add("device"); +// dimensions.add("country_code"); +// dimensions.add("known_user"); +// dimensions.add("base_url"); +// dimensions.add("referring_url"); +// dimensions.add("full_url"); +// dimensions.add("timestamp"); +// dimensions.add("city"); +// dimensions.add("tz"); +// +// WebListener listener = new WebListener() { +// @Override +// public void onUpdate(Update update) +// { +// if (Thread.currentThread().isInterrupted()) { +// throw new RuntimeException("Interrupted, time to stop"); +// } +// try { +// boolean success = queue.offer(update, 15L, TimeUnit.SECONDS); +// if (!success){ +// log.warn("queue too slow!"); +// } +// } +// catch (InterruptedException e){ +// throw new RuntimeException("InterruptedException", e); +// } +// } +// }; +// return new Firehose() +// { +// final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { +// @Override +// public void onConnect() +// { +// log.info("Connected_to_stream"); +// } +// +// @Override +// public void onDisconnect() +// { +// log.info("Disconnected_from_stream"); +// } +// +// @Override +// public void onCleanUp() +// { +// log.info("Cleanup_stream"); +// } +// }; +// private final Runnable doNothingRunnable = new Runnable() { +// public void run() +// { +// } +// }; +// private boolean waitIfmax = true; +// @Override +// public boolean hasMore() +// { +// if (maxEventCount >=0 && rowCount >= maxEventCount){ +// return waitIfmax; +// } +// else +// { +// return true; +// } +// } +// +// @Override +// public InputRow nextRow() +// { +// if (maxEventCount >=0 && rowCount >=maxEventCount && waitIfmax){ +// try { +// sleep(2000000000L); +// } +// catch (InterruptedException e) { +// throw new RuntimeException("InterruptedException"); +// } +// } +// Update udpate; +// try{ +// update=queue.take(); +// } +// catch (InterruptedException e) { +// throw new RuntimeException("InterruptedException", e); +// } +// final Map theMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); +// theMap.put("device", update.device()); +// theMap.put("country_code", update.country()); +// theMap.put("known_user", update.knownUser()); +// theMap.put("base_url", update.baseUrl()); +// theMap.put("referring_url", update.referringUrl()); +// theMap.put("full_url", update.fullUrl()); +// theMap.put("timestamp", update.timestamp()); +// theMap.put("city", update.city()); +// theMap.put("tz", update.tz()); +// +// return new MapBasedInputRow(update.timestamp,dimensions,theMap); //To change body of implemented methods use File | Settings | File Templates. +// } +// +// @Override +// public Runnable commit() +// { +// return doNothingRunnable; //To change body of implemented methods use File | Settings | File Templates. +// } +// +// @Override +// public void close() throws IOException +// { +// //To change body of implemented methods use File | Settings | File Templates. +// } +// +// }; +// +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebBase.java b/examples/src/main/java/druid/examples/twitter/WebBase.java new file mode 100644 index 00000000000..4e223b8ebd9 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebBase.java @@ -0,0 +1,15 @@ +package druid.examples.twitter; + +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/19/13 + * Time: 4:15 PM + * To change this template use File | Settings | File Templates. + */ +public interface WebBase +{ + Configuration getConfiguration(); + + void shutdown(); +} diff --git a/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java b/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java new file mode 100644 index 00000000000..30310e89739 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java @@ -0,0 +1,170 @@ +//package druid.examples.twitter; +// +///* +// * Copyright 2007 Yusuke Yamamoto +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +// +//import java.io.IOException; +//import java.io.ObjectInputStream; +//import java.util.ArrayList; +//import java.util.List; +// +// +//import java.io.IOException; +//import java.io.ObjectInputStream; +//import java.util.ArrayList; +//import java.util.List; +// +///** +// * Base class of Twitter / AsyncTwitter / TwitterStream supports OAuth. +// * +// * @author Yusuke Yamamoto - yusuke at mac.com +// */ +//abstract class WebBaseImpl implements WebBase, java.io.Serializable, HttpResponseListener +//{ +// protected Configuration conf; +// protected transient String screenName = null; +// protected transient long id = 0; +// +// protected transient HttpClientWrapper http; +// +// protected z_T4JInternalFactory factory; +// +// private static final long serialVersionUID = -3812176145960812140L; +// +// /*package*/ WebBaseImpl(Configuration conf) { +// this.conf = conf; +// init(); +// } +// +// private void init() { +// +// http = new HttpClientWrapper(conf); +// http.setHttpResponseListener(this); +// setFactory(); +// } +// +// protected void setFactory() { +// factory = new z_T4JInternalJSONImplFactory(conf); +// } +// +// /** +// * {@inheritDoc} +// */ +// +// @Override +// public void httpResponseReceived(HttpResponseEvent event) { +// if (rateLimitStatusListeners.size() != 0) { +// HttpResponse res = event.getResponse(); +// TwitterException te = event.getTwitterException(); +// RateLimitStatus rateLimitStatus; +// int statusCode; +// if (te != null) { +// rateLimitStatus = te.getRateLimitStatus(); +// statusCode = te.getStatusCode(); +// } else { +// rateLimitStatus = z_T4JInternalJSONImplFactory.createRateLimitStatusFromResponseHeader(res); +// statusCode = res.getStatusCode(); +// } +// if (rateLimitStatus != null) { +// RateLimitStatusEvent statusEvent +// = new RateLimitStatusEvent(this, rateLimitStatus, event.isAuthenticated()); +// if (statusCode == ENHANCE_YOUR_CLAIM +// || statusCode == SERVICE_UNAVAILABLE) { +// // EXCEEDED_RATE_LIMIT_QUOTA is returned by Rest API +// // SERVICE_UNAVAILABLE is returned by Search API +// for (RateLimitStatusListener listener : rateLimitStatusListeners) { +// listener.onRateLimitStatus(statusEvent); +// listener.onRateLimitReached(statusEvent); +// } +// } else { +// for (RateLimitStatusListener listener : rateLimitStatusListeners) { +// listener.onRateLimitStatus(statusEvent); +// } +// } +// } +// } +// } +// +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public Configuration getConfiguration() { +// return this.conf; +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void shutdown() { +// if (http != null) http.shutdown(); +// } +// +// +// private void writeObject(java.io.ObjectOutputStream out) throws IOException +// { +// // http://docs.oracle.com/javase/6/docs/platform/serialization/spec/output.html#861 +// out.putFields(); +// out.writeFields(); +// +// out.writeObject(conf); +// } +// +// private void readObject(ObjectInputStream stream) +// throws IOException, ClassNotFoundException { +// // http://docs.oracle.com/javase/6/docs/platform/serialization/spec/input.html#2971 +// stream.readFields(); +// +// conf = (Configuration) stream.readObject(); +// http = new HttpClientWrapper(conf); +// http.setHttpResponseListener(this); +// setFactory(); +// } +// +// +// +// @Override +// public boolean equals(Object o) { +// if (this == o) return true; +// if (!(o instanceof WebBaseImpl)) return false; +// +// WebBaseImpl that = (WebBaseImpl) o; +// +// if (!conf.equals(that.conf)) return false; +// if (http != null ? !http.equals(that.http) : that.http != null) +// return false; +// +// return true; +// } +// +// @Override +// public int hashCode() { +// int result = conf.hashCode(); +// result = 31 * result + (http != null ? http.hashCode() : 0); +// return result; +// } +// +// @Override +// public String toString() { +// return "TwitterBase{" + +// "conf=" + conf + +// ", http=" + http + +// '}'; +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java b/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java new file mode 100644 index 00000000000..64854151f31 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java @@ -0,0 +1,494 @@ +package druid.examples.twitter; +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/19/13 + * Time: 2:02 PM + * To change this template use File | Settings | File Templates. + */ + +import druid.examples.twitter.Configuration; +import twitter4j.internal.util.z_T4JInternalStringUtil; + +import java.io.ObjectStreamException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + + +import java.io.ObjectStreamException; +import java.lang.reflect.Field; +import java.util.*; +import java.util.logging.Logger; + +/** + * Configuration base class with default settings. + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +class WebConfigurationBase implements Configuration, java.io.Serializable { + private boolean debug; + private boolean useSSL; + private boolean prettyDebug; + private boolean gzipEnabled; + private String httpProxyHost; + private String httpProxyUser; + private String httpProxyPassword; + private String userAgent; + private int httpProxyPort; + private int httpConnectionTimeout; + private int httpReadTimeout; + + private int httpStreamingReadTimeout; + private int httpRetryCount; + private int httpRetryIntervalSeconds; + private int maxTotalConnections; + private int defaultMaxPerRoute; + + private String streamBaseURL; + + private String dispatcherImpl; + private String loggerFactory; + + private int asyncNumThreads; + + private long contributingTo; + private boolean includeEntitiesEnabled = true; + + private boolean jsonStoreEnabled; + + + private boolean stallWarningsEnabled; + + + + private static final String DEFAULT_STREAM_BASE_URL = "http://developer.usa.gov/1usagov/"; + + private static final long serialVersionUID = -6610497517837844232L; + + + protected WebConfigurationBase() { + setDebug(false); + setUseSSL(false); + setPrettyDebugEnabled(false); + setGZIPEnabled(true); + setHttpProxyHost(null); + setHttpProxyUser(null); + setHttpProxyPassword(null); + setUserAgent(null); + setHttpProxyPort(-1); + setHttpConnectionTimeout(20000); + setHttpReadTimeout(120000); + setHttpStreamingReadTimeout(40 * 1000); + setHttpRetryCount(0); + setHttpRetryIntervalSeconds(5); + setHttpMaxTotalConnections(20); + setHttpDefaultMaxPerRoute(2); + setAsyncNumThreads(1); + setContributingTo(-1L); + + setJSONStoreEnabled(false); + + + setStreamBaseURL(DEFAULT_STREAM_BASE_URL); + + setDispatcherImpl("twitter4j.internal.async.DispatcherImpl"); + setLoggerFactory(null); + + setStallWarningsEnabled(true); + + } + + public void dumpConfiguration() { + if (debug) { +// Field[] fields = WebConfigurationBase.class.getDeclaredFields(); +// for (Field field : fields) { +// try { +// Object value = field.get(this); +// String strValue = String.valueOf(value); +// if (value != null && field.getName().matches("oAuthConsumerSecret|oAuthAccessTokenSecret|password")) { +// strValue = z_T4JInternalStringUtil.maskString(String.valueOf(value)); +// } +// log.debug(field.getName() + ": " + strValue); +// } catch (IllegalAccessException ignore) { +// } +// } + } + } + + @Override + public final boolean isDebugEnabled() { + return debug; + } + + protected final void setDebug(boolean debug) { + this.debug = debug; + } + + + @Override + public boolean isPrettyDebugEnabled() { + return prettyDebug; + } + + protected final void setUseSSL(boolean useSSL) { + this.useSSL = useSSL; + } + + protected final void setPrettyDebugEnabled(boolean prettyDebug) { + this.prettyDebug = prettyDebug; + } + + protected final void setGZIPEnabled(boolean gzipEnabled) { + this.gzipEnabled = gzipEnabled; + initRequestHeaders(); + } + + @Override + public boolean isGZIPEnabled() { + return gzipEnabled; + } + + // method for HttpRequestFactoryConfiguration + Map requestHeaders; + + private void initRequestHeaders() { +// requestHeaders = new HashMap(); +// requestHeaders.put("X-Twitter-Client-Version", getClientVersion()); +// requestHeaders.put("X-Twitter-Client-URL", getClientURL()); +// requestHeaders.put("X-Twitter-Client", "Twitter4J"); +// +// requestHeaders.put("User-Agent", getUserAgent()); +// if (gzipEnabled) { +// requestHeaders.put("Accept-Encoding", "gzip"); +// } +// if (IS_DALVIK) { +// requestHeaders.put("Connection", "close"); +// } + } + + @Override + public Map getRequestHeaders() { + return requestHeaders; + } + + // methods for HttpClientConfiguration + + @Override + public final String getHttpProxyHost() { + return httpProxyHost; + } + + protected final void setHttpProxyHost(String proxyHost) { + this.httpProxyHost = proxyHost; + } + + @Override + public final String getHttpProxyUser() { + return httpProxyUser; + } + + protected final void setHttpProxyUser(String proxyUser) { + this.httpProxyUser = proxyUser; + } + + @Override + public final String getHttpProxyPassword() { + return httpProxyPassword; + } + + protected final void setHttpProxyPassword(String proxyPassword) { + this.httpProxyPassword = proxyPassword; + } + + @Override + public final int getHttpProxyPort() { + return httpProxyPort; + } + + protected final void setHttpProxyPort(int proxyPort) { + this.httpProxyPort = proxyPort; + } + + @Override + public final int getHttpConnectionTimeout() { + return httpConnectionTimeout; + } + + protected final void setHttpConnectionTimeout(int connectionTimeout) { + this.httpConnectionTimeout = connectionTimeout; + } + + @Override + public final int getHttpReadTimeout() { + return httpReadTimeout; + } + + protected final void setHttpReadTimeout(int readTimeout) { + this.httpReadTimeout = readTimeout; + } + + @Override + public int getHttpStreamingReadTimeout() { + return httpStreamingReadTimeout; + } + + protected final void setHttpStreamingReadTimeout(int httpStreamingReadTimeout) { + this.httpStreamingReadTimeout = httpStreamingReadTimeout; + } + + + @Override + public final int getHttpRetryCount() { + return httpRetryCount; + } + + protected final void setHttpRetryCount(int retryCount) { + this.httpRetryCount = retryCount; + } + + @Override + public final int getHttpRetryIntervalSeconds() { + return httpRetryIntervalSeconds; + } + + protected final void setHttpRetryIntervalSeconds(int retryIntervalSeconds) { + this.httpRetryIntervalSeconds = retryIntervalSeconds; + } + + @Override + public final int getHttpMaxTotalConnections() { + return maxTotalConnections; + } + + protected final void setHttpMaxTotalConnections(int maxTotalConnections) { + this.maxTotalConnections = maxTotalConnections; + } + + @Override + public final int getHttpDefaultMaxPerRoute() { + return defaultMaxPerRoute; + } + + protected final void setHttpDefaultMaxPerRoute(int defaultMaxPerRoute) { + this.defaultMaxPerRoute = defaultMaxPerRoute; + } + + + @Override + public final int getAsyncNumThreads() { + return asyncNumThreads; + } + + protected final void setAsyncNumThreads(int asyncNumThreads) { + this.asyncNumThreads = asyncNumThreads; + } + + @Override + public final long getContributingTo() { + return contributingTo; + } + + protected final void setContributingTo(long contributingTo) { + this.contributingTo = contributingTo; + } + + + @Override + public String getStreamBaseURL() { + return streamBaseURL; + } + + protected final void setStreamBaseURL(String streamBaseURL) { + this.streamBaseURL = streamBaseURL; + } + + + @Override + public String getDispatcherImpl() { + return dispatcherImpl; + } + + protected final void setDispatcherImpl(String dispatcherImpl) { + this.dispatcherImpl = dispatcherImpl; + } + + protected final void setUserAgent(String userAgent) { + this.userAgent= userAgent; + } + + @Override + public String getLoggerFactory() { + return loggerFactory; + } + + + protected final void setLoggerFactory(String loggerImpl) { + this.loggerFactory = loggerImpl; + } + + + public boolean isJSONStoreEnabled() { + return this.jsonStoreEnabled; + } + + protected final void setJSONStoreEnabled(boolean enabled) { + this.jsonStoreEnabled = enabled; + } + + @Override + public boolean isStallWarningsEnabled() { + return stallWarningsEnabled; + } + + protected final void setStallWarningsEnabled(boolean stallWarningsEnabled) { + this.stallWarningsEnabled = stallWarningsEnabled; + } + + static String fixURL(boolean useSSL, String url) { + if (null == url) { + return null; + } + int index = url.indexOf("://"); + if (-1 == index) { + throw new IllegalArgumentException("url should contain '://'"); + } + String hostAndLater = url.substring(index + 3); + if (useSSL) { + return "https://" + hostAndLater; + } else { + return "http://" + hostAndLater; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + WebConfigurationBase that = (WebConfigurationBase) o; + + if (asyncNumThreads != that.asyncNumThreads) return false; + if (contributingTo != that.contributingTo) return false; + if (debug != that.debug) return false; + if (defaultMaxPerRoute != that.defaultMaxPerRoute) return false; + if (gzipEnabled != that.gzipEnabled) return false; + if (httpConnectionTimeout != that.httpConnectionTimeout) return false; + if (httpProxyPort != that.httpProxyPort) return false; + if (httpReadTimeout != that.httpReadTimeout) return false; + if (httpRetryCount != that.httpRetryCount) return false; + if (httpRetryIntervalSeconds != that.httpRetryIntervalSeconds) return false; + if (httpStreamingReadTimeout != that.httpStreamingReadTimeout) return false; + if (includeEntitiesEnabled != that.includeEntitiesEnabled) return false; + if (jsonStoreEnabled != that.jsonStoreEnabled) return false; + if (maxTotalConnections != that.maxTotalConnections) return false; + if (prettyDebug != that.prettyDebug) return false; + if (stallWarningsEnabled != that.stallWarningsEnabled) return false; + if (useSSL != that.useSSL) return false; + if (dispatcherImpl != null ? !dispatcherImpl.equals(that.dispatcherImpl) : that.dispatcherImpl != null) + return false; + if (httpProxyHost != null ? !httpProxyHost.equals(that.httpProxyHost) : that.httpProxyHost != null) + return false; + if (httpProxyPassword != null ? !httpProxyPassword.equals(that.httpProxyPassword) : that.httpProxyPassword != null) + return false; + if (httpProxyUser != null ? !httpProxyUser.equals(that.httpProxyUser) : that.httpProxyUser != null) + return false; + if (loggerFactory != null ? !loggerFactory.equals(that.loggerFactory) : that.loggerFactory != null) + return false; + if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) + return false; + if (streamBaseURL != null ? !streamBaseURL.equals(that.streamBaseURL) : that.streamBaseURL != null) + return false; + return true; + } + + @Override + public int hashCode() { + int result = (debug ? 1 : 0); + result = 31 * result + (useSSL ? 1 : 0); + result = 31 * result + (prettyDebug ? 1 : 0); + result = 31 * result + (gzipEnabled ? 1 : 0); + result = 31 * result + (httpProxyHost != null ? httpProxyHost.hashCode() : 0); + result = 31 * result + (httpProxyUser != null ? httpProxyUser.hashCode() : 0); + result = 31 * result + (httpProxyPassword != null ? httpProxyPassword.hashCode() : 0); + result = 31 * result + httpProxyPort; + result = 31 * result + httpConnectionTimeout; + result = 31 * result + httpReadTimeout; + result = 31 * result + httpStreamingReadTimeout; + result = 31 * result + httpRetryCount; + result = 31 * result + httpRetryIntervalSeconds; + result = 31 * result + maxTotalConnections; + result = 31 * result + defaultMaxPerRoute; + result = 31 * result + (streamBaseURL != null ? streamBaseURL.hashCode() : 0); + result = 31 * result + (dispatcherImpl != null ? dispatcherImpl.hashCode() : 0); + result = 31 * result + (loggerFactory != null ? loggerFactory.hashCode() : 0); + result = 31 * result + asyncNumThreads; + result = 31 * result + (int) (contributingTo ^ (contributingTo >>> 32)); + result = 31 * result + (includeEntitiesEnabled ? 1 : 0); + result = 31 * result + (jsonStoreEnabled ? 1 : 0); + result = 31 * result + (stallWarningsEnabled ? 1 : 0); + result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "ConfigurationBase{" + + "debug=" + debug + + ", useSSL=" + useSSL + + ", prettyDebug=" + prettyDebug + + ", gzipEnabled=" + gzipEnabled + + ", httpProxyHost='" + httpProxyHost + '\'' + + ", httpProxyUser='" + httpProxyUser + '\'' + + ", httpProxyPassword='" + httpProxyPassword + '\'' + + ", httpProxyPort=" + httpProxyPort + + ", httpConnectionTimeout=" + httpConnectionTimeout + + ", httpReadTimeout=" + httpReadTimeout + + ", httpStreamingReadTimeout=" + httpStreamingReadTimeout + + ", httpRetryCount=" + httpRetryCount + + ", httpRetryIntervalSeconds=" + httpRetryIntervalSeconds + + ", maxTotalConnections=" + maxTotalConnections + + ", defaultMaxPerRoute=" + defaultMaxPerRoute + + ", streamBaseURL='" + streamBaseURL + '\'' + + ", dispatcherImpl='" + dispatcherImpl + '\'' + + ", loggerFactory='" + loggerFactory + '\'' + + ", asyncNumThreads=" + asyncNumThreads + + ", contributingTo=" + contributingTo + + ", includeEntitiesEnabled=" + includeEntitiesEnabled + + ", jsonStoreEnabled=" + jsonStoreEnabled + + ", stallWarningsEnabled=" + stallWarningsEnabled + + ", requestHeaders=" + requestHeaders + + '}'; + } + + private static final List instances = new ArrayList(); + + private static void cacheInstance(WebConfigurationBase conf) { + if (!instances.contains(conf)) { + instances.add(conf); + } + } + + protected void cacheInstance() { + cacheInstance(this); + } + + private static WebConfigurationBase getInstance(WebConfigurationBase configurationBase) { + int index; + if ((index = instances.indexOf(configurationBase)) == -1) { + instances.add(configurationBase); + return configurationBase; + } else { + return instances.get(index); + } + } + + // assures equality after deserializedation + protected Object readResolve() throws ObjectStreamException + { + return getInstance(this); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java b/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java new file mode 100644 index 00000000000..747fd9bb7b7 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java @@ -0,0 +1,176 @@ +//package druid.examples.twitter; +// +///* +// * Copyright 2007 Yusuke Yamamoto +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +// +// +//import org.mortbay.jetty.servlet.Dispatcher; +// +//import java.io.BufferedReader; +//import java.io.IOException; +//import java.io.InputStream; +//import java.io.InputStreamReader; +// +///** +// * @author Yusuke Yamamoto - yusuke at mac.com +// * @since Twitter4J 2.1.8 +// */ +//abstract class WebStatusStreamBase implements WebStatusStream +//{ +// protected static final twitter4j.internal.logging.Logger logger = twitter4j.internal +// .logging +// .Logger +// .getLogger(WebStatusStreamImpl.class); +// +// private boolean streamAlive = true; +// private BufferedReader br; +// private InputStream is; +// private HttpResponse response; +// protected final Dispatcher dispatcher; +// protected final Configuration CONF; +// protected z_InternalFactory factory; +// +// /*package*/ +// +// StatusStreamBase(Dispatcher dispatcher, InputStream stream, twitter4j.conf.Configuration conf) throws IOException +// { +// this.is = stream; +// this.br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); +// this.dispatcher = dispatcher; +// this.CONF = conf; +// this.factory = new z_T4JInternalJSONImplFactory(conf); +// } +// /*package*/ +// +// StatusStreamBase(Dispatcher dispatcher, HttpResponse response, twitter4j.conf.Configuration conf) throws IOException { +// this(dispatcher, response.asStream(), conf); +// this.response = response; +// } +// +// protected String parseLine(String line) { +// return line; +// } +// +// abstract class StreamEvent implements Runnable { +// String line; +// +// StreamEvent(String line) { +// this.line = line; +// } +// } +// +// protected void handleNextElement(final StreamListener[] listeners, +// final RawStreamListener[] rawStreamListeners) throws TwitterException +// { +// if (!streamAlive) { +// throw new IllegalStateException("Stream already closed."); +// } +// try { +// String line = br.readLine(); +// if (null == line) { +// //invalidate this status stream +// throw new IOException("the end of the stream has been reached"); +// } +// dispatcher.invokeLater(new StreamEvent(line) { +// public void run() { +// try { +// if (rawStreamListeners.length > 0) { +// onMessage(line, rawStreamListeners); +// } +// // SiteStreamsImpl will parse "forUser" attribute +// line = parseLine(line); +// if (line != null && line.length() > 0) { +// // parsing JSON is an expensive process and can be avoided when all listeners are instanceof RawStreamListener +// if (listeners.length > 0) { +// if (CONF.isJSONStoreEnabled()) { +// DataObjectFactoryUtil.clearThreadLocalMap(); +// } +// twitter4j.internal.org.json.JSONObject json = new twitter4j.internal.org.json.JSONObject(line); +// JSONObjectType.Type event = JSONObjectType.determine(json); +// if (logger.isDebugEnabled()) { +// logger.debug("Received:", CONF.isPrettyDebugEnabled() ? json.toString(1) : json.toString()); +// } +// switch (event) { +// case STATUS: +// onStatus(json, listeners); +// break; +// default: +// logger.warn("Received unknown event:", CONF.isPrettyDebugEnabled() ? json.toString(1) : json.toString()); +// } +// } +// } +// } catch (Exception ex) { +// onException(ex, listeners); +// } +// } +// }); +// +// } catch (IOException ioe) { +// try { +// is.close(); +// } catch (IOException ignore) { +// } +// boolean isUnexpectedException = streamAlive; +// streamAlive = false; +// if (isUnexpectedException) { +// throw new RuntimeException("Stream closed.", ioe); +// } +// } +// } +// +// +// +// protected void onStatus(twitter4j.internal.org.json.JSONObject json, StreamListener[] listeners) throws TwitterException { +// logger.warn("Unhandled event: onStatus"); +// } +// +// +// protected void onException(Exception e, StreamListener[] listeners) { +// logger.warn("Unhandled event: ", e.getMessage()); +// } +// +// public void close() throws IOException { +// streamAlive = false; +// is.close(); +// br.close(); +// if (response != null) { +// response.disconnect(); +// } +// } +// +// protected Status asStatus(twitter4j.internal.org.json.JSONObject json) throws TwitterException { +// Status status = factory.createStatus(json); +// if (CONF.isJSONStoreEnabled()) { +// DataObjectFactoryUtil.registerJSONObject(status, json); +// } +// return status; +// } +// +// +// public abstract void next(StatusListener listener) throws TwitterException; +// +// public abstract void next(StreamListener[] listeners, RawStreamListener[] rawStreamListeners) throws TwitterException; +// +// public void onException(Exception e, StreamListener[] listeners, RawStreamListener[] rawStreamListeners) { +// for (StreamListener listener : listeners) { +// listener.onException(e); +// } +// for (RawStreamListener listener : rawStreamListeners) { +// listener.onException(e); +// } +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java b/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java new file mode 100644 index 00000000000..53ef525f33b --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java @@ -0,0 +1,111 @@ +//package druid.examples.twitter; +// +///* +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +// +///** +// * An instance of this class is completely thread safe and can be re-used and used concurrently.
+// * Note that TwitterStream is NOT compatible with Google App Engine as GAE is not capable of handling requests longer than 30 seconds. +// * +// */ +//public final class WebStreamFactory implements java.io.Serializable { +// private static final long serialVersionUID = 8146074704915782233L; +// private final Configuration conf; +// private static final WebStream SINGLETON; +// +// static { +// SINGLETON = new TwitterStreamImpl(ConfigurationContext.getInstance(), TwitterFactory.DEFAULT_AUTHORIZATION); +// } +// +// /** +// * Creates a TwitterStreamFactory with the root configuration. +// */ +// public TwitterStreamFactory() { +// this(ConfigurationContext.getInstance()); +// } +// +// /** +// * Creates a TwitterStreamFactory with the given configuration. +// * +// * @param conf the configuration to use +// * @since Twitter4J 2.1.1 +// */ +// public WebStreamFactory(Configuration conf) { +// this.conf = conf; +// } +// +// /** +// * Creates a TwitterStreamFactory with a specified config tree. +// * +// * @param configTreePath the path +// */ +// public WebStreamFactory(String configTreePath) { +// this(ConfigurationContext.getInstance(configTreePath)); +// } +// +// // implementations for BasicSupportFactory +// +// /** +// * Returns a instance associated with the configuration bound to this factory. +// * +// * @return default instance +// */ +// public WebStream getInstance() { +// return getInstance(AuthorizationFactory.getInstance(conf)); +// } +// +// /** +// * Returns a OAuth Authenticated instance.
+// * consumer key and consumer Secret must be provided by twitter4j.properties, or system properties. +// * Unlike {@link TwitterStream#setOAuthAccessToken(twitter4j.auth.AccessToken)}, this factory method potentially returns a cached instance. +// * +// * @param accessToken access token +// * @return an instance +// */ +// public TwitterStream getInstance(AccessToken accessToken) { +// String consumerKey = conf.getOAuthConsumerKey(); +// String consumerSecret = conf.getOAuthConsumerSecret(); +// if (null == consumerKey && null == consumerSecret) { +// throw new IllegalStateException("Consumer key and Consumer secret not supplied."); +// } +// OAuthAuthorization oauth = new OAuthAuthorization(conf); +// oauth.setOAuthAccessToken(accessToken); +// return getInstance(conf, oauth); +// } +// +// /** +// * Returns a instance. +// * +// * @return an instance +// */ +// public TwitterStream getInstance(Authorization auth) { +// return getInstance(conf, auth); +// } +// +// private TwitterStream getInstance(twitter4j.conf.Configuration conf, Authorization auth) { +// return new TwitterStreamImpl(conf, auth); +// } +// +// /** +// * Returns default singleton TwitterStream instance. +// * +// * @return default singleton TwitterStream instance +// * @since Twitter4J 2.2.4 +// */ +// public static TwitterStream getSingleton() { +// return SINGLETON; +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java b/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java new file mode 100644 index 00000000000..330ff6b5f95 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java @@ -0,0 +1,750 @@ +//package druid.examples.twitter; +// +///* +// * Copyright 2007 Yusuke Yamamoto +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//import java.io.IOException; +//import java.io.InputStream; +//import java.util.ArrayList; +//import java.util.HashMap; +//import java.util.List; +//import java.util.Map; +// +// +// +//import java.io.IOException; +//import java.io.InputStream; +//import java.util.ArrayList; +//import java.util.HashMap; +//import java.util.List; +//import java.util.Map; +// +///** +// * A java representation of the Streaming API: Methods
+// * Note that this class is NOT compatible with Google App Engine as GAE is not capable of handling requests longer than 30 seconds. +// * +// * @author Yusuke Yamamoto - yusuke at mac.com +// * @since Twitter4J 2.0.4 +// */ +//class WebStreamImpl extends TwitterBaseImpl implements WebStream +//{ +// private static final long serialVersionUID = 5529611191443189901L; +// private final HttpClientWrapper http; +// private static final Logger logger = Logger.getLogger(TwitterStreamImpl.class); +// +// private List lifeCycleListeners = new ArrayList(0); +// private TwitterStreamConsumer handler = null; +// +// private String stallWarningsGetParam; +// private HttpParameter stallWarningsParam; +// +// /*package*/ +// TwitterStreamImpl(twitter4j.conf.Configuration conf, Authorization auth) { +// super(conf, auth); +// http = new HttpClientWrapper(new StreamingReadTimeoutConfiguration(conf)); +// stallWarningsGetParam = "stall_warnings=" + (conf.isStallWarningsEnabled() ? "true" : "false"); +// stallWarningsParam = new HttpParameter("stall_warnings", conf.isStallWarningsEnabled()); +// } +// +// /* Streaming API */ +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void firehose(final int count) { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException +// { +// return getFirehoseStream(count); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getFirehoseStream(int count) throws TwitterException { +// ensureAuthorizationEnabled(); +// return getCountStream("statuses/firehose.json", count); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void links(final int count) { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getLinksStream(count); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getLinksStream(int count) throws TwitterException { +// ensureAuthorizationEnabled(); +// return getCountStream("statuses/links.json", count); +// } +// +// private StatusStream getCountStream(String relativeUrl, int count) throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() + relativeUrl +// , new HttpParameter[]{new HttpParameter("count", String.valueOf(count)) +// , stallWarningsParam}, auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void retweet() { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getRetweetStream(); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getRetweetStream() throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() + "statuses/retweet.json" +// , new HttpParameter[]{stallWarningsParam}, auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void sample() { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getSampleStream(); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getSampleStream() throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.get(conf.getStreamBaseURL() + "statuses/sample.json?" +// + stallWarningsGetParam, auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// public void user() { +// user(null); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void user(final String[] track) { +// ensureAuthorizationEnabled(); +// ensureUserStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getUserStream(track); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public UserStream getUserStream() throws TwitterException { +// return getUserStream(null); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public UserStream getUserStream(String[] track) throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// List params = new ArrayList(); +// params.add(stallWarningsParam); +// if (conf.isUserStreamRepliesAllEnabled()) { +// params.add(new HttpParameter("replies", "all")); +// } +// if (track != null) { +// params.add(new HttpParameter("track", z_T4JInternalStringUtil.join(track))); +// } +// return new UserStreamImpl(getDispatcher(), http.post(conf.getUserStreamBaseURL() + "user.json" +// , params.toArray(new HttpParameter[params.size()]) +// , auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StreamController site(final boolean withFollowings, final long[] follow) { +// ensureOAuthEnabled(); +// ensureSiteStreamsListenerIsSet(); +// final StreamController cs = new StreamController(http, auth); +// startHandler(new TwitterStreamConsumer(siteStreamsListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// try { +// return new SiteStreamsImpl(getDispatcher(), getSiteStream(withFollowings, follow), conf, cs); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// }); +// return cs; +// } +// +// private Dispatcher getDispatcher() { +// if (null == TwitterStreamImpl.dispatcher) { +// synchronized (TwitterStreamImpl.class) { +// if (null == TwitterStreamImpl.dispatcher) { +// // dispatcher is held statically, but it'll be instantiated with +// // the configuration instance associated with this TwitterStream +// // instance which invokes getDispatcher() on the first time. +// TwitterStreamImpl.dispatcher = new DispatcherFactory(conf).getInstance(); +// } +// } +// } +// return TwitterStreamImpl.dispatcher; +// } +// +// private static transient Dispatcher dispatcher; +// +// InputStream getSiteStream(boolean withFollowings, long[] follow) throws TwitterException { +// ensureOAuthEnabled(); +// return http.post(conf.getSiteStreamBaseURL() + "site.json", +// new HttpParameter[]{ +// new HttpParameter("with", withFollowings ? "followings" : "user") +// , new HttpParameter("follow", z_T4JInternalStringUtil.join(follow)) +// , stallWarningsParam}, auth).asStream(); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void filter(final FilterQuery query) { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getFilterStream(query); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getFilterStream(FilterQuery query) throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() +// + "statuses/filter.json" +// , query.asHttpParameterArray(stallWarningsParam), auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// +// /** +// * check if any listener is set. Throws IllegalStateException if no listener is set. +// * +// * @throws IllegalStateException when no listener is set. +// */ +// +// private void ensureStatusStreamListenerIsSet() { +// if (statusListeners.size() == 0 && rawStreamListeners.size() == 0) { +// throw new IllegalStateException("StatusListener is not set."); +// } +// } +// +// private void ensureUserStreamListenerIsSet() { +// if (userStreamListeners.size() == 0 && rawStreamListeners.size() == 0) { +// throw new IllegalStateException("UserStreamListener is not set."); +// } +// } +// +// private void ensureSiteStreamsListenerIsSet() { +// if (siteStreamsListeners.size() == 0 && rawStreamListeners.size() == 0) { +// throw new IllegalStateException("SiteStreamsListener is not set."); +// } +// } +// +// private static int numberOfHandlers = 0; +// +// private synchronized void startHandler(TwitterStreamConsumer handler) { +// cleanUp(); +// this.handler = handler; +// this.handler.start(); +// numberOfHandlers++; +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public synchronized void cleanUp() { +// if (handler != null) { +// handler.close(); +// numberOfHandlers--; +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public synchronized void shutdown() { +// super.shutdown(); +// cleanUp(); +// synchronized (TwitterStreamImpl.class) { +// if (0 == numberOfHandlers) { +// if (dispatcher != null) { +// dispatcher.shutdown(); +// dispatcher = null; +// } +// } +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addConnectionLifeCycleListener(ConnectionLifeCycleListener listener) { +// this.lifeCycleListeners.add(listener); +// } +// +// private List userStreamListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addListener(UserStreamListener listener) { +// statusListeners.add(listener); +// userStreamListeners.add(listener); +// } +// +// private List statusListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addListener(StatusListener listener) { +// statusListeners.add(listener); +// } +// +// private List siteStreamsListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addListener(SiteStreamsListener listener) { +// siteStreamsListeners.add(listener); +// } +// +// private List rawStreamListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// public void addListener(RawStreamListener listener) { +// rawStreamListeners.add(listener); +// } +// +// /* +// https://dev.twitter.com/docs/streaming-api/concepts#connecting +// When a network error (TCP/IP level) is encountered, back off linearly. Perhaps start at 250 milliseconds, double, and cap at 16 seconds +// When a HTTP error (> 200) is returned, back off exponentially. +// Perhaps start with a 10 second wait, double on each subsequent failure, and finally cap the wait at 240 seconds. Consider sending an alert to a human operator after multiple HTTP errors, as there is probably a client configuration issue that is unlikely to be resolved without human intervention. There's not much point in polling any faster in the face of HTTP error codes and your client is may run afoul of a rate limit. +// */ +// private static final int TCP_ERROR_INITIAL_WAIT = 250; +// private static final int TCP_ERROR_WAIT_CAP = 16 * 1000; +// +// private static final int HTTP_ERROR_INITIAL_WAIT = 10 * 1000; +// private static final int HTTP_ERROR_WAIT_CAP = 240 * 1000; +// +// private static final int NO_WAIT = 0; +// +// static int count = 0; +// +// abstract class TwitterStreamConsumer extends Thread { +// private StatusStreamBase stream = null; +// private final String NAME = "Twitter Stream consumer-" + (++count); +// private volatile boolean closed = false; +// private final StreamListener[] streamListeners; +// private final RawStreamListener[] rawStreamListeners; +// +// TwitterStreamConsumer(List streamListeners, List rawStreamListeners) { +// super(); +// setName(NAME + "[initializing]"); +// this.streamListeners = streamListeners.toArray(new StreamListener[streamListeners.size()]); +// this.rawStreamListeners = rawStreamListeners.toArray(new RawStreamListener[rawStreamListeners.size()]); +// } +// +// @Override +// public void run() { +// int timeToSleep = NO_WAIT; +// boolean connected = false; +// while (!closed) { +// try { +// if (!closed && null == stream) { +// // try establishing connection +// logger.info("Establishing connection."); +// setStatus("[Establishing connection]"); +// stream = (StatusStreamBase) getStream(); +// connected = true; +// logger.info("Connection established."); +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onConnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// // connection established successfully +// timeToSleep = NO_WAIT; +// logger.info("Receiving status stream."); +// setStatus("[Receiving stream]"); +// while (!closed) { +// try { +// stream.next(this.streamListeners, this.rawStreamListeners); +// } catch (IllegalStateException ise) { +// logger.warn(ise.getMessage()); +// break; +// } catch (TwitterException e) { +// logger.info(e.getMessage()); +// stream.onException(e, this.streamListeners, this.rawStreamListeners); +// throw e; +// } catch (Exception e) { +// logger.info(e.getMessage()); +// stream.onException(e, this.streamListeners, this.rawStreamListeners); +// closed = true; +// break; +// } +// } +// } +// } catch (TwitterException te) { +// logger.info(te.getMessage()); +// if (!closed) { +// if (NO_WAIT == timeToSleep) { +// if (te.getStatusCode() == FORBIDDEN) { +// logger.warn("This account is not in required role. ", te.getMessage()); +// closed = true; +// for (StreamListener statusListener : streamListeners) { +// statusListener.onException(te); +// } +// break; +// } +// if (te.getStatusCode() == NOT_ACCEPTABLE) { +// logger.warn("Parameter not accepted with the role. ", te.getMessage()); +// closed = true; +// for (StreamListener statusListener : streamListeners) { +// statusListener.onException(te); +// } +// break; +// } +// connected = false; +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onDisconnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// if (te.getStatusCode() > 200) { +// timeToSleep = HTTP_ERROR_INITIAL_WAIT; +// } else if (0 == timeToSleep) { +// timeToSleep = TCP_ERROR_INITIAL_WAIT; +// } +// } +// if (te.getStatusCode() > 200 && timeToSleep < HTTP_ERROR_INITIAL_WAIT) { +// timeToSleep = HTTP_ERROR_INITIAL_WAIT; +// } +// if (connected) { +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onDisconnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// } +// for (StreamListener statusListener : streamListeners) { +// statusListener.onException(te); +// } +// // there was a problem establishing the connection, or the connection closed by peer +// if (!closed) { +// // wait for a moment not to overload Twitter API +// logger.info("Waiting for " + (timeToSleep) + " milliseconds"); +// setStatus("[Waiting for " + (timeToSleep) + " milliseconds]"); +// try { +// Thread.sleep(timeToSleep); +// } catch (InterruptedException ignore) { +// } +// timeToSleep = Math.min(timeToSleep * 2, (te.getStatusCode() > 200) ? HTTP_ERROR_WAIT_CAP : TCP_ERROR_WAIT_CAP); +// } +// stream = null; +// logger.debug(te.getMessage()); +// connected = false; +// } +// } +// } +// if (this.stream != null && connected) { +// try { +// this.stream.close(); +// } catch (IOException ignore) { +// } catch (Exception e) { +// e.printStackTrace(); +// logger.warn(e.getMessage()); +// } finally { +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onDisconnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// } +// } +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onCleanUp(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// } +// +// public synchronized void close() { +// setStatus("[Disposing thread]"); +// try { +// if (stream != null) { +// try { +// stream.close(); +// } catch (IOException ignore) { +// } catch (Exception e) { +// e.printStackTrace(); +// logger.warn(e.getMessage()); +// } +// } +// } finally { +// closed = true; +// } +// } +// +// private void setStatus(String message) { +// String actualMessage = NAME + message; +// setName(actualMessage); +// logger.debug(actualMessage); +// } +// +// abstract StatusStream getStream() throws TwitterException; +// +// } +// +// @Override +// public boolean equals(Object o) { +// if (this == o) return true; +// if (o == null || getClass() != o.getClass()) return false; +// if (!super.equals(o)) return false; +// +// TwitterStreamImpl that = (TwitterStreamImpl) o; +// +// if (handler != null ? !handler.equals(that.handler) : that.handler != null) return false; +// if (http != null ? !http.equals(that.http) : that.http != null) return false; +// if (lifeCycleListeners != null ? !lifeCycleListeners.equals(that.lifeCycleListeners) : that.lifeCycleListeners != null) +// return false; +// if (rawStreamListeners != null ? !rawStreamListeners.equals(that.rawStreamListeners) : that.rawStreamListeners != null) +// return false; +// if (siteStreamsListeners != null ? !siteStreamsListeners.equals(that.siteStreamsListeners) : that.siteStreamsListeners != null) +// return false; +// if (stallWarningsGetParam != null ? !stallWarningsGetParam.equals(that.stallWarningsGetParam) : that.stallWarningsGetParam != null) +// return false; +// if (stallWarningsParam != null ? !stallWarningsParam.equals(that.stallWarningsParam) : that.stallWarningsParam != null) +// return false; +// if (statusListeners != null ? !statusListeners.equals(that.statusListeners) : that.statusListeners != null) +// return false; +// if (userStreamListeners != null ? !userStreamListeners.equals(that.userStreamListeners) : that.userStreamListeners != null) +// return false; +// +// return true; +// } +// +// @Override +// public int hashCode() { +// int result = super.hashCode(); +// result = 31 * result + (http != null ? http.hashCode() : 0); +// result = 31 * result + (lifeCycleListeners != null ? lifeCycleListeners.hashCode() : 0); +// result = 31 * result + (handler != null ? handler.hashCode() : 0); +// result = 31 * result + (stallWarningsGetParam != null ? stallWarningsGetParam.hashCode() : 0); +// result = 31 * result + (stallWarningsParam != null ? stallWarningsParam.hashCode() : 0); +// result = 31 * result + (userStreamListeners != null ? userStreamListeners.hashCode() : 0); +// result = 31 * result + (statusListeners != null ? statusListeners.hashCode() : 0); +// result = 31 * result + (siteStreamsListeners != null ? siteStreamsListeners.hashCode() : 0); +// result = 31 * result + (rawStreamListeners != null ? rawStreamListeners.hashCode() : 0); +// return result; +// } +// +// @Override +// public String toString() { +// return "TwitterStreamImpl{" + +// "http=" + http + +// ", lifeCycleListeners=" + lifeCycleListeners + +// ", handler=" + handler + +// ", stallWarningsGetParam='" + stallWarningsGetParam + '\'' + +// ", stallWarningsParam=" + stallWarningsParam + +// ", userStreamListeners=" + userStreamListeners + +// ", statusListeners=" + statusListeners + +// ", siteStreamsListeners=" + siteStreamsListeners + +// ", rawStreamListeners=" + rawStreamListeners + +// '}'; +// } +//} +// +//class StreamingReadTimeoutConfiguration implements HttpClientWrapperConfiguration +//{ +// twitter4j.conf.Configuration nestedConf; +// +// StreamingReadTimeoutConfiguration(twitter4j.conf.Configuration httpConf) { +// this.nestedConf = httpConf; +// } +// +// @Override +// public String getHttpProxyHost() { +// return nestedConf.getHttpProxyHost(); +// } +// +// @Override +// public int getHttpProxyPort() { +// return nestedConf.getHttpProxyPort(); +// } +// +// @Override +// public String getHttpProxyUser() { +// return nestedConf.getHttpProxyUser(); +// } +// +// @Override +// public String getHttpProxyPassword() { +// return nestedConf.getHttpProxyPassword(); +// } +// +// @Override +// public int getHttpConnectionTimeout() { +// return nestedConf.getHttpConnectionTimeout(); +// } +// +// @Override +// public int getHttpReadTimeout() { +// // this is the trick that overrides connection timeout +// return nestedConf.getHttpStreamingReadTimeout(); +// } +// +// @Override +// public int getHttpRetryCount() { +// return nestedConf.getHttpRetryCount(); +// } +// +// @Override +// public int getHttpRetryIntervalSeconds() { +// return nestedConf.getHttpRetryIntervalSeconds(); +// } +// +// @Override +// public int getHttpMaxTotalConnections() { +// return nestedConf.getHttpMaxTotalConnections(); +// } +// +// @Override +// public int getHttpDefaultMaxPerRoute() { +// return nestedConf.getHttpDefaultMaxPerRoute(); +// } +// +// @Override +// public Map getRequestHeaders() { +// // turning off keepalive connection explicitly because Streaming API doesn't need keepalive connection. +// // and this will reduce the shutdown latency of streaming api connection +// // see also - http://jira.twitter4j.org/browse/TFJ-556 +// Map headers = new HashMap(nestedConf.getRequestHeaders()); +// headers.put("Connection", "close"); +// return headers; +// } +// +// @Override +// public boolean isPrettyDebugEnabled() { +// return nestedConf.isPrettyDebugEnabled(); +// } +// +// @Override +// public boolean isGZIPEnabled() { +// return nestedConf.isGZIPEnabled(); +// } +//} +// diff --git a/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java b/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java new file mode 100644 index 00000000000..d214d755260 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java @@ -0,0 +1,111 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.List; + + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + * @since Twitter4J 2.1.4 + */ +public class z_InternalStringUtil { + private z_InternalStringUtil() { + throw new AssertionError(); + } + + public static String maskString(String str) { + StringBuilder buf = new StringBuilder(str.length()); + for (int i = 0; i < str.length(); i++) { + buf.append("*"); + } + return buf.toString(); + } + + // for JDK1.4 compatibility + + public static String[] split(String str, String separator) { + String[] returnValue; + int index = str.indexOf(separator); + if (index == -1) { + returnValue = new String[]{str}; + } else { + List strList = new ArrayList(); + int oldIndex = 0; + while (index != -1) { + String subStr = str.substring(oldIndex, index); + strList.add(subStr); + oldIndex = index + separator.length(); + index = str.indexOf(separator, oldIndex); + } + if (oldIndex != str.length()) { + strList.add(str.substring(oldIndex)); + } + returnValue = strList.toArray(new String[strList.size()]); + } + + return returnValue; + } + + public static String join(int[] follows) { + StringBuilder buf = new StringBuilder(11 * follows.length); + for (int follow : follows) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(follow); + } + return buf.toString(); + } + + public static String join(long[] follows) { + StringBuilder buf = new StringBuilder(11 * follows.length); + for (long follow : follows) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(follow); + } + return buf.toString(); + } + + public static String join(String[] track) { + StringBuilder buf = new StringBuilder(11 * track.length); + for (String str : track) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(str); + } + return buf.toString(); + } + + public static String join(List strs) { + StringBuilder buf = new StringBuilder(11 * strs.size()); + for (String str : strs) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(str); + } + return buf.toString(); + } +} diff --git a/examples/twitter4j.properties b/examples/twitter4j.properties new file mode 100644 index 00000000000..002d832143b --- /dev/null +++ b/examples/twitter4j.properties @@ -0,0 +1,5 @@ +debug=true +oauth.consumerKey=HOQS4c7WveN7NCxzl2aNgg +oauth.consumerSecret=47mKCGRfzml268RWDmRYGkFF6pusVrzoYrhRZPfELY +oauth.accessToken=19520065-aBZl60q698VYksk78oayY1eRFucofWo3FLI1tvekA +oauth.accessTokenSecret=Ul0bSghDzlYGKzN6lH7W6qZxW4bFCANfVXMs3KPfM diff --git a/examples/twitter_realtime.spec b/examples/twitter_realtime.spec new file mode 100644 index 00000000000..f17aab07f17 --- /dev/null +++ b/examples/twitter_realtime.spec @@ -0,0 +1,44 @@ +[{ + "schema": { + "dataSource": "twitterstream", + "aggregators": [ + {"type": "count", "name": "tweets"}, + {"type": "doubleSum", "fieldName": "follower_count", "name": "total_follower_count"}, + {"type": "doubleSum", "fieldName": "retweet_count", "name": "total_retweet_count" }, + {"type": "doubleSum", "fieldName": "friends_count", "name": "total_friends_count" }, + {"type": "doubleSum", "fieldName": "statuses_count", "name": "total_statuses_count"}, + + {"type": "min", "fieldName": "follower_count", "name": "min_follower_count"}, + {"type": "max", "fieldName": "follower_count", "name": "max_follower_count"}, + + {"type": "min", "fieldName": "friends_count", "name": "min_friends_count"}, + {"type": "max", "fieldName": "friends_count", "name": "max_friends_count"}, + + {"type": "min", "fieldName": "statuses_count", "name": "min_statuses_count"}, + {"type": "max", "fieldName": "statuses_count", "name": "max_statuses_count"}, + + {"type": "min", "fieldName": "retweet_count", "name": "min_retweet_count"}, + {"type": "max", "fieldName": "retweet_count", "name": "max_retweet_count"} + ], + "indexGranularity": "minute", + "shardSpec": {"type": "none"} + }, + + "config": { + "maxRowsInMemory": 50000, + "intermediatePersistPeriod": "PT2m" + }, + + "firehose": { + "type": "twitzer", + "maxEventCount": 500000, + "maxRunMinutes": 120 + }, + + "plumber": { + "type": "realtime", + "windowPeriod": "PT3m", + "segmentGranularity": "hour", + "basePersistDirectory": "/tmp/example/twitter_realtime/basePersist" + } +}] diff --git a/indexing-common/.DS_Store b/indexing-common/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..feca8613504bcd7430524202412f605a970dd548 GIT binary patch literal 6148 zcmeH~JqiLr422VS3&Cbf%V|7-HyA`u-~~i21wpZ&qx41vI)z6#m+=T zbn`f`MFtU>!%bynVPcAXCwIBY>3lz3j@RpDteEA>YT$!Ro{xoGkN^pg011!)3H%HJ zJGWu;MJOW)kN^pc1nmD%;HEXTh5D}p!AAgSkF*=sK1)E0C7?C6g(3sfXoW_r`WRw) zZ-`o{NKSp&Hs}YZb^Uy{)~Y3 z+x>QfkIJ+4?ei>u$gHg!9O~r=FP8vp>?mHs-SE8F0$NjBC^9f!1RMhc34E2n6Mwc5 Aa{vGU literal 0 HcmV?d00001 diff --git a/indexing-common/src/.DS_Store b/indexing-common/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..9d6a56620545250bb9a34aa5ef87a586fdec28fa GIT binary patch literal 6148 zcmeH~J#ND=428+mgMlm_Giu2JdIP~oPS6W9ZI%FTJOtf!^gjNeK$Xr#5j+CqCsHB| zzriv^MCZ@*v&bYOBitx^3!N$QK|bUp7rCBq@5|e|E_r5?v@`HYy8YZHr~nn90#tws zP=SA>fZf}$aT{c$0#twsd@EqzhXOa&WDEMI1HnfCa6s7&Yo8^+Vg;}!TM!kPMk^St z>SKu2y&Wuht|nVB+C_8t(0sDm6a&*}7cEF&S{)2jfC_X9JVid){eOi2Hve}mOsN1B zcrXQYIJ_KYyj0$;cduvl*R0yQ!9hQc@Z%={iCx7zxEuD1Ex?*=K~!M;5pWq8sKB2p F@C6gX5~u(G literal 0 HcmV?d00001 diff --git a/indexing-common/src/main/.DS_Store b/indexing-common/src/main/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d38176301af6a9db8e9ffa5dc7916732489a254b GIT binary patch literal 6148 zcmeH~JqiLr422WjLa^D=avBfd4F=H@cme+^5iCUOIl3=D2(H#5@&d^>$xK-G6+0Ud z(dF&95a~r^0yoOs!oU=HCnwp-U3SOo;k>_G&Ut2(v@`Ht2K%{9Pys4H1*iZOpaMUn zfZf}$$u!7F1*iZOcq(AuhXOa&WDEMI1HnfCutV7mYo8^+Vg;}!TM!kPMk^St>SKu2 zy&Wuht|nVB+C_8t(0sDm6a&*}7cEF&S{)2jfC>y07)RdQ{lA8Pn*RqaOsN1B_%j8x zTWwZLyj0$;Z?9+dZB}jF;GiEzc=-uHVps7D?uPwh3$P|z5EU4I1Y8CND)3bW9?Vn{ Ar2qf` literal 0 HcmV?d00001 diff --git a/indexing-common/src/main/java/.DS_Store b/indexing-common/src/main/java/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2b615a767f6da08a02d4f0c4241f455170fff868 GIT binary patch literal 6148 zcmeH~JqiLr422W55Nx)zoW=uqgF*BJUO;q9Wec&Nqx41vI)z6#m+=T zboc1DBAtjV;ij^&Ffqk`E*BZ(*55DJ^Dtb;idn9#2Hwf!`B=yW36KB@kN^pgz|Rn{ za~rl?g)))=36Q`@!2S;fZdy}YsQ)?;d<1|lkaok`X9;Mr1hl5MP-I{lt-v z{c*p;N9Ec2_IZ{+W!BaW4)t<`mrDRPb`-DSZg^g70j;Sm6d4#V0*--!1inh(38@+p AY5)KL literal 0 HcmV?d00001 diff --git a/install/.DS_Store b/install/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..33eea665350de4626480e3f764234d3db87f9fef GIT binary patch literal 6148 zcmeHKyGjH>5UkbK950p*sT?z=0;xb>fo<u)Yv~NVO8`4siub?d6>T%G3hW$>&T*p?^CFJ5&P${lz9-`!Pzc zdXER<*ZLdQANehF!|JN4*Q=^tk*{K1(of&a7y(CkLQVc0&n)}wmmYhB9cNs3$CT?H z>kti<++xF3G(GNXkIQOUC=d#S0--=C@FN9yW~)u6j!}jJp+G3`setYeiLO{=Y#nXu zU|~-HqF$rT80+;|b5h14W9!Hsns_SFQzbt!#M9ZHGOx(kI(j-JKRzVS{PB1Zo1Nn) zqeBYEC_{lzpsT>ht1~_SFZjz0KJvRMQ40k^fq$lejF&gd1wSgEt=~RR&)P)4rK>5g p)quvha|vKY_mLCp^!}tVc}2$7QAg2sODD!fzyygR6!--N-T@|8LGJ(n literal 0 HcmV?d00001 diff --git a/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java b/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java new file mode 100644 index 00000000000..f57f5604f6b --- /dev/null +++ b/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java @@ -0,0 +1,85 @@ +package com.metamx.druid.realtime.firehose; + +import com.metamx.druid.input.InputRow; +import com.metamx.druid.input.MapBasedInputRow; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +/** +* Created with IntelliJ IDEA. +* User: dhruvparthasarathy +* Date: 6/20/13 +* Time: 2:03 PM +* To change this template use File | Settings | File Templates. +*/ +public class WebFirehoseFactory implements FirehoseFactory +{ + + @Override + public Firehose connect() throws IOException + { + final UpdateListener updateListener; + final int QUEUE_SIZE=2000; + final BlockingQueue> queue= new ArrayBlockingQueue>(QUEUE_SIZE); + final LinkedList dimensions = new LinkedList(); + + dimensions.add("BITLY_HASH"); + dimensions.add("LATUTUDE-LONGITUDE"); + dimensions.add("COUNTRTY_CODE"); + dimensions.add("USER_AGENT"); + dimensions.add("CITY"); + dimensions.add("ENCODING_USER_LOGIN"); + dimensions.add("SHORT_URL_CNAME"); + dimensions.add("TIMESTAMP OF TIME HASH WAS CREATED"); + dimensions.add("ENCODING_USER_BITLY_HASH"); + dimensions.add("LONG_URL"); + dimensions.add("TIMEZONE"); + dimensions.add("TIMESTAMP OF TIME HASH WAS CREATED"); + dimensions.add("REFERRING_URL"); + dimensions.add("GEO_REGION"); + dimensions.add("KNOWN_USER"); + + UpdateStream updateStream = new UpdateStream(queue); + updateStream.start(); + + return new Firehose() { + private final Map theMap = new HashMap(); + + private final Runnable doNothingRunnable = new Runnable() { + public void run(){ + + } + }; + + @Override + public boolean hasMore(){ + if (queue.size()>0){ + return true; + } + return false; + } + + @Override + public InputRow nextRow() + { + if (Thread.currentThread().isInterrupted()) { + throw new RuntimeException("Interrupted, time to stop"); + } + Map update; + try{ + update=queue.take(); + } + catch (InterruptedException e) { + throw new RuntimeException("InterrutpedException", e); + } + + return new MapBasedInputRow(((Long) update.get("TIMESTAMP")).intValue(),dimensions,update); + } + }; + } +} diff --git a/server/.DS_Store b/server/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..8c136e4f0413485ac8547313f579d9bbd85218f3 GIT binary patch literal 6148 zcmeHK%Sr=55Ue&q1G((c;~c#h@DG*{1+V%8nyg4*WhExT<9?tY;NNI<_o(c;2Tvl> z4Kp>5>fV`xotXw8^IdfY6aXeHilU7H)8WCXgG?S1#bQ)A#S`wZ!G0jnUmVi47pS=6 z9oEFJ_3JKX9D6{0SvRX?-7Jj@To6y*%xp_Xctk_q49^4XmmYgWg*I4qW6D*Jb%-r) z$-AeT+aC9|$F)U76$*p`p+G1Q3j9z3-q~uCsbkckKqwFjd@7*lLt;@ZGq#R)bg;4% zfT+)CGqyFCP@j~s%-A||h9>SxbXUnKhPXS&Q|6T!TSs??=D$M8m25C${{!mFx+#}+vm{@|x?nkZ%SI14z#|&+W_TW8zx3E6=3S%g=9H@* zYad(Ol6TK}Y7d{yOug zOotSPQHKJdKv#j0HK%(1pYxX)eB^gSq7@2+0{=__885CEb1ury)^E$xvo^8Zu&61o pRfk5ua|vKW_mSf~XyZwJ@(Rb+P_t-%4kyM%APW*zDDVpkyaPdALL2}9 literal 0 HcmV?d00001 From 8750ee88f20054d1cbd6933ca9aa0d9b7f57c42b Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 25 Jun 2013 14:37:58 -0700 Subject: [PATCH 02/25] Webstream example with tests --- .DS_Store | Bin 15364 -> 15364 bytes .../main/java/druid/examples/HttpTest.java | 55 - .../examples/RealtimeStandaloneMain.java | 5 +- .../druid/examples/twitter/Configuration.java | 59 - .../twitter/ConfigurationContext.java | 57 - .../twitter/ConfigurationFactory.java | 42 - .../twitter/HttpClientConfiguration.java | 46 - .../HttpClientWrapperConfiguration.java | 13 - .../druid/examples/twitter/HttpParameter.java | 288 ---- .../druid/examples/twitter/HttpRequest.java | 120 -- .../druid/examples/twitter/HttpResponse.java | 239 --- .../examples/twitter/HttpResponseEvent.java | 92 -- .../examples/twitter/HttpResponseImpl.java | 75 - .../twitter/HttpResponseListener.java | 10 - .../druid/examples/twitter/JSONArray.java | 726 ---------- .../druid/examples/twitter/JSONObject.java | 1285 ----------------- .../druid/examples/twitter/JSONTokener.java | 359 ----- .../java/druid/examples/twitter/Logger.java | 180 --- .../druid/examples/twitter/LoggerFactory.java | 34 - .../twitter/PropertyConfiguration.java | 325 ----- .../twitter/PropertyConfigurationFactory.java | 61 - .../druid/examples/twitter/RequestMethod.java | 26 - .../twitter/StreamingGZipInputStream.java | 53 - .../twitter/USGovFirehoseFactory.java | 173 --- .../java/druid/examples/twitter/WebBase.java | 15 - .../druid/examples/twitter/WebBaseImpl.java | 170 --- .../twitter/WebConfigurationBase.java | 494 ------- .../examples/twitter/WebStatusStreamBase.java | 176 --- .../examples/twitter/WebStreamFactory.java | 111 -- .../druid/examples/twitter/WebStreamImpl.java | 750 ---------- .../twitter/z_InternalStringUtil.java | 111 -- .../examples/webStream/TestCaseSupplier.java | 27 + .../examples/webStream/UpdateStream.java | 67 + .../webStream/WebFirehoseFactory.java | 139 ++ .../webStream/WebFirehoseFactoryTest.java | 172 +++ .../examples/webStream/WebJsonSupplier.java | 37 + examples/twitter4j.properties | 5 - .../realtime/firehose/WebFirehoseFactory.java | 85 -- 38 files changed, 446 insertions(+), 6236 deletions(-) delete mode 100644 examples/src/main/java/druid/examples/HttpTest.java delete mode 100644 examples/src/main/java/druid/examples/twitter/Configuration.java delete mode 100644 examples/src/main/java/druid/examples/twitter/ConfigurationContext.java delete mode 100644 examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpParameter.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpRequest.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponse.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java delete mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseListener.java delete mode 100644 examples/src/main/java/druid/examples/twitter/JSONArray.java delete mode 100644 examples/src/main/java/druid/examples/twitter/JSONObject.java delete mode 100644 examples/src/main/java/druid/examples/twitter/JSONTokener.java delete mode 100644 examples/src/main/java/druid/examples/twitter/Logger.java delete mode 100644 examples/src/main/java/druid/examples/twitter/LoggerFactory.java delete mode 100644 examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java delete mode 100644 examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java delete mode 100644 examples/src/main/java/druid/examples/twitter/RequestMethod.java delete mode 100644 examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java delete mode 100644 examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java delete mode 100644 examples/src/main/java/druid/examples/twitter/WebBase.java delete mode 100644 examples/src/main/java/druid/examples/twitter/WebBaseImpl.java delete mode 100644 examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java delete mode 100644 examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java delete mode 100644 examples/src/main/java/druid/examples/twitter/WebStreamFactory.java delete mode 100644 examples/src/main/java/druid/examples/twitter/WebStreamImpl.java delete mode 100644 examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java create mode 100644 examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java create mode 100644 examples/src/main/java/druid/examples/webStream/UpdateStream.java create mode 100644 examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java create mode 100644 examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java create mode 100644 examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java delete mode 100644 examples/twitter4j.properties delete mode 100644 realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java diff --git a/.DS_Store b/.DS_Store index ee1df41e8fb05f1e8db6280875eaa40f45860a2c..d1abbd6f4f83617f08b6bedf8dee91d153c650cc 100644 GIT binary patch delta 52 zcmZpvXsMXsUDU^hRb;$|L!z08~`#mPBI`T04Fo3{#^@@!^T_{FmMi3l_EWK0p+j~CjbBd delta 45 zcmZpvXsMXs¥U^hRb!e$?f|uJDUxvz#b1^JYPf&%zTMcmZ#1 B4^IF9 diff --git a/examples/src/main/java/druid/examples/HttpTest.java b/examples/src/main/java/druid/examples/HttpTest.java deleted file mode 100644 index ca39df38bf2..00000000000 --- a/examples/src/main/java/druid/examples/HttpTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package druid.examples; - -import org.apache.http.*; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.DefaultHttpClient; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLConnection; -import java.util.HashMap; - -/** - * Created with IntelliJ IDEA. - * User: dhruvparthasarathy - * Date: 6/20/13 - * Time: 12:13 PM - * To change this template use File | Settings | File Templates. - */ -public class HttpTest -{ - public static void main(String[] args) throws Exception{ - URL url = new URL("http://developer.usa.gov/1usagov"); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setDoInput(true); - connection.setDoOutput(true); - connection.setRequestMethod("GET"); - - BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); - String line; - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference> () {}; - try{ - while ((line = reader.readLine())!= null){ - try{ - HashMap map=mapper.readValue(line, typeRef); - System.out.println(map); - } - catch (Exception e){ - System.out.println(e); - } - } - } - catch (IOException e){ - - } - } -} diff --git a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java b/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java index 4cb267f571f..54d4a492584 100644 --- a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java +++ b/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java @@ -13,6 +13,7 @@ import com.metamx.druid.loading.DataSegmentPusher; import com.metamx.druid.log.LogLevelAdjuster; import com.metamx.druid.realtime.RealtimeNode; import com.metamx.druid.realtime.SegmentPublisher; +import druid.examples.webStream.WebFirehoseFactory; import druid.examples.flights.FlightsFirehoseFactory; import druid.examples.rand.RandomFirehoseFactory; import druid.examples.twitter.TwitterSpritzerFirehoseFactory; @@ -41,7 +42,9 @@ public class RealtimeStandaloneMain rn.registerJacksonSubtype( new NamedType(TwitterSpritzerFirehoseFactory.class, "twitzer"), new NamedType(FlightsFirehoseFactory.class, "flights"), - new NamedType(RandomFirehoseFactory.class, "rand") + new NamedType(RandomFirehoseFactory.class, "rand"), + new NamedType(WebFirehoseFactory.class, "webstream") + ); // Create dummy objects for the various interfaces that interact with the DB, ZK and deep storage diff --git a/examples/src/main/java/druid/examples/twitter/Configuration.java b/examples/src/main/java/druid/examples/twitter/Configuration.java deleted file mode 100644 index 4c6cc76f202..00000000000 --- a/examples/src/main/java/druid/examples/twitter/Configuration.java +++ /dev/null @@ -1,59 +0,0 @@ -package druid.examples.twitter; - -import druid.examples.twitter.HttpClientWrapperConfiguration.HttpClientWrapperConfiguration; - -import java.util.Map; - - -public interface Configuration extends HttpClientConfiguration - , HttpClientWrapperConfiguration - , java.io.Serializable { - - - boolean isDebugEnabled(); - - - Map getRequestHeaders(); - - // methods for HttpClientConfiguration - - String getHttpProxyHost(); - - String getHttpProxyUser(); - - String getHttpProxyPassword(); - - int getHttpProxyPort(); - - int getHttpConnectionTimeout(); - - int getHttpReadTimeout(); - - int getHttpStreamingReadTimeout(); - - int getHttpRetryCount(); - - int getHttpRetryIntervalSeconds(); - - int getHttpMaxTotalConnections(); - - int getHttpDefaultMaxPerRoute(); - - - String getStreamBaseURL(); - - - boolean isJSONStoreEnabled(); - - - boolean isStallWarningsEnabled(); - - int getAsyncNumThreads(); - - long getContributingTo(); - - String getDispatcherImpl(); - - String getLoggerFactory(); - -} diff --git a/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java b/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java deleted file mode 100644 index 15ba09c7f6c..00000000000 --- a/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java +++ /dev/null @@ -1,57 +0,0 @@ -package druid.examples.twitter; - -/* -* Copyright 2007 Yusuke Yamamoto -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -/** -* Static factory of Configuration. This class wraps ConfigurationFactory implementations.
-* By default, twitter4j.conf.PropertyConfigurationFactory will be used and can be changed with -Dtwitter4j.configurationFactory system property. -* -*/ -public final class ConfigurationContext { - public static final String DEFAULT_CONFIGURATION_FACTORY = "twitter4j.conf.PropertyConfigurationFactory"; - public static final String CONFIGURATION_IMPL = "twitter4j.configurationFactory"; - private static final ConfigurationFactory factory; - - static { - String CONFIG_IMPL; - try { - CONFIG_IMPL = System.getProperty(CONFIGURATION_IMPL, DEFAULT_CONFIGURATION_FACTORY); - } catch (SecurityException ignore) { - // Unsigned applets are not allowed to access System properties - CONFIG_IMPL = DEFAULT_CONFIGURATION_FACTORY; - } - - try { - factory = (ConfigurationFactory) Class.forName(CONFIG_IMPL).newInstance(); - } catch (ClassNotFoundException cnfe) { - throw new AssertionError(cnfe); - } catch (InstantiationException ie) { - throw new AssertionError(ie); - } catch (IllegalAccessException iae) { - throw new AssertionError(iae); - } - } - - - public static Configuration getInstance() { - return factory.getInstance(); - } - - public static Configuration getInstance(String configTreePath) { - return factory.getInstance(configTreePath); - } -} diff --git a/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java b/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java deleted file mode 100644 index d4c1acdf69e..00000000000 --- a/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @author Yusuke Yamamoto - yusuke at mac.com - */ -public interface ConfigurationFactory { - /** - * returns the root configuration - * - * @return root configuration - */ - Configuration getInstance(); - - /** - * returns the configuration specified by the path - * - * @param configTreePath the path - * @return the configuratoin - */ - Configuration getInstance(String configTreePath); - - /** - * clean up resources acquired by this factory. - */ - void dispose(); -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java b/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java deleted file mode 100644 index 882ca03091a..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java +++ /dev/null @@ -1,46 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -public interface HttpClientConfiguration { - - String getHttpProxyHost(); - - int getHttpProxyPort(); - - String getHttpProxyUser(); - - String getHttpProxyPassword(); - - int getHttpConnectionTimeout(); - - int getHttpReadTimeout(); - - int getHttpRetryCount(); - - int getHttpRetryIntervalSeconds(); - - int getHttpMaxTotalConnections(); - - int getHttpDefaultMaxPerRoute(); - - boolean isPrettyDebugEnabled(); - - boolean isGZIPEnabled(); -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java b/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java deleted file mode 100644 index e419f6da6e0..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java +++ /dev/null @@ -1,13 +0,0 @@ -package druid.examples.twitter.HttpClientWrapperConfiguration; - -import druid.examples.twitter.HttpClientConfiguration; - -import java.util.Map; - -public interface HttpClientWrapperConfiguration extends HttpClientConfiguration -{ - /** - * @return request headers - */ - Map getRequestHeaders(); -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpParameter.java b/examples/src/main/java/druid/examples/twitter/HttpParameter.java deleted file mode 100644 index 0ca5643df1b..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpParameter.java +++ /dev/null @@ -1,288 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - -import java.io.File; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - -/** - * A data class representing HTTP Post parameter - * - * @author Yusuke Yamamoto - yusuke at mac.com - */ -public final class HttpParameter implements Comparable, java.io.Serializable { - private String name = null; - private String value = null; - private File file = null; - private InputStream fileBody = null; - private static final long serialVersionUID = -8708108746980739212L; - - public HttpParameter(String name, String value) { - this.name = name; - this.value = value; - } - - public HttpParameter(String name, File file) { - this.name = name; - this.file = file; - } - - public HttpParameter(String name, String fileName, InputStream fileBody) { - this.name = name; - this.file = new File(fileName); - this.fileBody = fileBody; - } - - public HttpParameter(String name, int value) { - this.name = name; - this.value = String.valueOf(value); - } - - public HttpParameter(String name, long value) { - this.name = name; - this.value = String.valueOf(value); - } - - public HttpParameter(String name, double value) { - this.name = name; - this.value = String.valueOf(value); - } - - public HttpParameter(String name, boolean value) { - this.name = name; - this.value = String.valueOf(value); - } - - public String getName() { - return name; - } - - public String getValue() { - return value; - } - - public File getFile() { - return file; - } - - public InputStream getFileBody() { - return fileBody; - } - - public boolean isFile() { - return file != null; - } - - public boolean hasFileBody() { - return fileBody != null; - } - - private static final String JPEG = "image/jpeg"; - private static final String GIF = "image/gif"; - private static final String PNG = "image/png"; - private static final String OCTET = "application/octet-stream"; - - /** - * @return content-type - */ - public String getContentType() { - if (!isFile()) { - throw new IllegalStateException("not a file"); - } - String contentType; - String extensions = file.getName(); - int index = extensions.lastIndexOf("."); - if (-1 == index) { - // no extension - contentType = OCTET; - } else { - extensions = extensions.substring(extensions.lastIndexOf(".") + 1).toLowerCase(); - if (extensions.length() == 3) { - if ("gif".equals(extensions)) { - contentType = GIF; - } else if ("png".equals(extensions)) { - contentType = PNG; - } else if ("jpg".equals(extensions)) { - contentType = JPEG; - } else { - contentType = OCTET; - } - } else if (extensions.length() == 4) { - if ("jpeg".equals(extensions)) { - contentType = JPEG; - } else { - contentType = OCTET; - } - } else { - contentType = OCTET; - } - } - return contentType; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof HttpParameter)) return false; - - HttpParameter that = (HttpParameter) o; - - if (file != null ? !file.equals(that.file) : that.file != null) - return false; - if (fileBody != null ? !fileBody.equals(that.fileBody) : that.fileBody != null) - return false; - if (!name.equals(that.name)) return false; - if (value != null ? !value.equals(that.value) : that.value != null) - return false; - - return true; - } - - public static boolean containsFile(HttpParameter[] params) { - boolean containsFile = false; - if (null == params) { - return false; - } - for (HttpParameter param : params) { - if (param.isFile()) { - containsFile = true; - break; - } - } - return containsFile; - } - - /*package*/ - static boolean containsFile(List params) { - boolean containsFile = false; - for (HttpParameter param : params) { - if (param.isFile()) { - containsFile = true; - break; - } - } - return containsFile; - } - - public static HttpParameter[] getParameterArray(String name, String value) { - return new HttpParameter[]{new HttpParameter(name, value)}; - } - - public static HttpParameter[] getParameterArray(String name, int value) { - return getParameterArray(name, String.valueOf(value)); - } - - public static HttpParameter[] getParameterArray(String name1, String value1 - , String name2, String value2) { - return new HttpParameter[]{new HttpParameter(name1, value1) - , new HttpParameter(name2, value2)}; - } - - public static HttpParameter[] getParameterArray(String name1, int value1 - , String name2, int value2) { - return getParameterArray(name1, String.valueOf(value1), name2, String.valueOf(value2)); - } - - @Override - public int hashCode() { - int result = name.hashCode(); - result = 31 * result + (value != null ? value.hashCode() : 0); - result = 31 * result + (file != null ? file.hashCode() : 0); - result = 31 * result + (fileBody != null ? fileBody.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "PostParameter{" + - "name='" + name + '\'' + - ", value='" + value + '\'' + - ", file=" + file + - ", fileBody=" + fileBody + - '}'; - } - - @Override - public int compareTo(Object o) { - int compared; - HttpParameter that = (HttpParameter) o; - compared = name.compareTo(that.name); - if (0 == compared) { - compared = value.compareTo(that.value); - } - return compared; - } - - public static String encodeParameters(HttpParameter[] httpParams) { - if (null == httpParams) { - return ""; - } - StringBuilder buf = new StringBuilder(); - for (int j = 0; j < httpParams.length; j++) { - if (httpParams[j].isFile()) { - throw new IllegalArgumentException("parameter [" + httpParams[j].name + "]should be text"); - } - if (j != 0) { - buf.append("&"); - } - buf.append(encode(httpParams[j].name)) - .append("=").append(encode(httpParams[j].value)); - } - return buf.toString(); - } - - /** - * @param value string to be encoded - * @return encoded string - * @see OAuth / TestCases - * @see Space encoding - OAuth | Google Groups - * @see RFC 3986 - Uniform Resource Identifier (URI): Generic Syntax - 2.1. Percent-Encoding - */ - public static String encode(String value) { - String encoded = null; - try { - encoded = URLEncoder.encode(value, "UTF-8"); - } catch (UnsupportedEncodingException ignore) { - } - StringBuilder buf = new StringBuilder(encoded.length()); - char focus; - for (int i = 0; i < encoded.length(); i++) { - focus = encoded.charAt(i); - if (focus == '*') { - buf.append("%2A"); - } else if (focus == '+') { - buf.append("%20"); - } else if (focus == '%' && (i + 1) < encoded.length() - && encoded.charAt(i + 1) == '7' && encoded.charAt(i + 2) == 'E') { - buf.append('~'); - i += 2; - } else { - buf.append(focus); - } - } - return buf.toString(); - } -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpRequest.java b/examples/src/main/java/druid/examples/twitter/HttpRequest.java deleted file mode 100644 index 9d233a423f8..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpRequest.java +++ /dev/null @@ -1,120 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.util.Arrays; -import java.util.Map; - - -import java.util.Arrays; -import java.util.Map; - -/** - * HTTP Request parameter object - * - * @author Yusuke Yamamoto - yusuke at mac.com - */ -public final class HttpRequest implements java.io.Serializable { - - private final RequestMethod method; - - private final String url; - - private final HttpParameter[] parameters; - - - private Map requestHeaders; - - private static final long serialVersionUID = -3463594029098858381L; - - - private static final HttpParameter[] NULL_PARAMETERS = new HttpParameter[0]; - - /** - * @param method Specifies the HTTP method - * @param url the request to request - * @param parameters parameters - * @param requestHeaders - */ - public HttpRequest(RequestMethod method, String url, HttpParameter[] parameters - , Map requestHeaders) { - this.method = method; - if (method != RequestMethod.POST && parameters != null && parameters.length != 0) { - this.url = url + "?" + HttpParameter.encodeParameters(parameters); - this.parameters = NULL_PARAMETERS; - } else { - this.url = url; - this.parameters = parameters; - } - this.requestHeaders = requestHeaders; - } - - public RequestMethod getMethod() { - return method; - } - - public HttpParameter[] getParameters() { - return parameters; - } - - public String getURL() { - return url; - } - - - public Map getRequestHeaders() { - return requestHeaders; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - HttpRequest that = (HttpRequest) o; - - if (!Arrays.equals(parameters, that.parameters)) return false; - if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) - return false; - if (method != null ? !method.equals(that.method) : that.method != null) - return false; - if (url != null ? !url.equals(that.url) : that.url != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = method != null ? method.hashCode() : 0; - result = 31 * result + (url != null ? url.hashCode() : 0); - result = 31 * result + (parameters != null ? Arrays.hashCode(parameters) : 0); - result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "HttpRequest{" + - "requestMethod=" + method + - ", url='" + url + '\'' + - ", postParams=" + (parameters == null ? null : Arrays.asList(parameters)) + - ", requestHeaders=" + requestHeaders + - '}'; - } -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponse.java b/examples/src/main/java/druid/examples/twitter/HttpResponse.java deleted file mode 100644 index ae2bc6f27d4..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpResponse.java +++ /dev/null @@ -1,239 +0,0 @@ -package druid.examples.twitter; - -/* -* Copyright 2007 Yusuke Yamamoto -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -import org.codehaus.jettison.json.JSONException; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.List; -import java.util.Map; - -import java.io.*; -import java.util.List; -import java.util.Map; - -/** -* A data class representing HTTP Response -* -* @author Yusuke Yamamoto - yusuke at mac.com -*/ -public abstract class HttpResponse { - private static final Logger logger = Logger.getLogger(HttpResponseImpl.class); - protected final HttpClientConfiguration CONF; - - HttpResponse() { - this.CONF = ConfigurationContext.getInstance(); - } - - public HttpResponse(HttpClientConfiguration conf) { - this.CONF = conf; - } - - protected int statusCode; - protected String responseAsString = null; - protected InputStream is; - private boolean streamConsumed = false; - - public int getStatusCode() { - return statusCode; - } - - public abstract String getResponseHeader(String name); - - public abstract Map> getResponseHeaderFields(); - - /** - * Returns the response stream.
- * This method cannot be called after calling asString() or asDcoument()
- * It is suggested to call disconnect() after consuming the stream. - *

- * Disconnects the internal HttpURLConnection silently. - * - * @return response body stream - * @see #disconnect() - */ - public InputStream asStream() { - if (streamConsumed) { - throw new IllegalStateException("Stream has already been consumed."); - } - return is; - } - - /** - * Returns the response body as string.
- * Disconnects the internal HttpURLConnection silently. - * - * @return response body - */ - public String asString() throws RuntimeException { - if (null == responseAsString) { - BufferedReader br = null; - InputStream stream = null; - try { - stream = asStream(); - if (null == stream) { - return null; - } - br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); - StringBuilder buf = new StringBuilder(); - String line; - while ((line = br.readLine()) != null) { - buf.append(line).append("\n"); - } - this.responseAsString = buf.toString(); - //logger.debug(responseAsString); - stream.close(); - streamConsumed = true; - } catch (IOException ioe) { - throw new RuntimeException(ioe.getMessage(), ioe); - } finally { - if (stream != null) { - try { - stream.close(); - } catch (IOException ignore) { - } - } - if (br != null) { - try { - br.close(); - } catch (IOException ignore) { - } - } - disconnectForcibly(); - } - } - return responseAsString; - } - - private JSONObject json = null; - - /** - * Returns the response body as twitter4j.internal.org.json.JSONObject.
- * Disconnects the internal HttpURLConnection silently. - * - * @return response body as JSONObject - */ - public JSONObject asJSONObject() throws RuntimeException { - if (json == null) { - Reader reader = null; - try { - if (responseAsString == null) { - reader = asReader(); - json = new JSONObject(new JSONTokener(reader)); - } else { - json = new JSONObject(responseAsString); - } - if (CONF.isPrettyDebugEnabled()) { - //logger.debug(json.toString(1)); - } else { - //logger.debug(responseAsString != null ? responseAsString : -// json.toString()); - } - } catch (JSONException jsone) { - if (responseAsString == null) { - throw new RuntimeException(jsone.getMessage(), jsone); - } else { - throw new RuntimeException(jsone.getMessage() + ":" + this.responseAsString, jsone); - } - } finally { - if (reader != null) { - try { - reader.close(); - } catch (IOException ignore) { - } - } - disconnectForcibly(); - } - } - return json; - } - - private JSONArray jsonArray = null; - - /** - * Returns the response body as twitter4j.internal.org.json.JSONArray.
- * Disconnects the internal HttpURLConnection silently. - * - * @return response body as twitter4j.internal.org.json.JSONArray - * @throws RuntimeException - */ - public JSONArray asJSONArray() throws RuntimeException { - if (jsonArray == null) { - Reader reader = null; - try { - if (responseAsString == null) { - reader = asReader(); - jsonArray = new JSONArray(new JSONTokener(reader)); - } else { - jsonArray = new JSONArray(responseAsString); - } - if (CONF.isPrettyDebugEnabled()) { - //logger.debug(jsonArray.toString(1)); - } else { - //logger.debug(responseAsString != null ? responseAsString : -// jsonArray.toString()); - } - } catch (JSONException jsone) { -// if (logger.isDebugEnabled()) { -// throw new RuntimeException(jsone.getMessage() + ":" + this.responseAsString, jsone); -// } else { -// throw new RuntimeException(jsone.getMessage(), jsone); -// } - } finally { - if (reader != null) { - try { - reader.close(); - } catch (IOException ignore) { - } - } - disconnectForcibly(); - } - } - return jsonArray; - } - - public Reader asReader() { - try { - return new BufferedReader(new InputStreamReader(is, "UTF-8")); - } catch (java.io.UnsupportedEncodingException uee) { - return new InputStreamReader(is); - } - } - - private void disconnectForcibly() { - try { - disconnect(); - } catch (Exception ignore) { - } - } - - public abstract void disconnect() throws IOException; - - @Override - public String toString() { - return "HttpResponse{" + - "statusCode=" + statusCode + - ", responseAsString='" + responseAsString + '\'' + - ", is=" + is + - ", streamConsumed=" + streamConsumed + - '}'; - } -} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java b/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java deleted file mode 100644 index 2f92bae3d7d..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java +++ /dev/null @@ -1,92 +0,0 @@ -package druid.examples.twitter; - -/* -* Copyright 2007 Yusuke Yamamoto -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -/** -* @author Andrew Hedges - andrew.hedges at gmail.com -*/ -public final class HttpResponseEvent { - - private HttpRequest request; - - private HttpResponse response; - - private RuntimeException e; - - HttpResponseEvent(HttpRequest request, HttpResponse response, RuntimeException e) { - this.request = request; - this.response = response; - this.e = e; - } - - /** - * returns the request associated with the event - * - * @return the request associated with the event - */ - public HttpRequest getRequest() { - return request; - } - - /** - * returns the response associated with the event - * - * @return the response associated with the event - */ - public HttpResponse getResponse() { - return response; - } - - /** - * returns the TwitterException associated with the event - * - * @return the TwitterException associated with the event - */ - public RuntimeException getRuntimeException() { - return this.e; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - HttpResponseEvent that = (HttpResponseEvent) o; - - if (request != null ? !request.equals(that.request) : that.request != null) - return false; - if (response != null ? !response.equals(that.response) : that.response != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = request != null ? request.hashCode() : 0; - result = 31 * result + (response != null ? response.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "HttpResponseEvent{" + - "request=" + request + - ", response=" + response + - '}'; - } -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java b/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java deleted file mode 100644 index 0faf59f50c9..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java +++ /dev/null @@ -1,75 +0,0 @@ -package druid.examples.twitter; - -/* -* Copyright 2007 Yusuke Yamamoto -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.util.List; -import java.util.Map; - - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.util.List; -import java.util.Map; - -/** -* @author Yusuke Yamamoto - yusuke at mac.com -* @since Twitter4J 2.1.2 -*/ -public class HttpResponseImpl extends HttpResponse -{ - private HttpURLConnection con; - - HttpResponseImpl(HttpURLConnection con, HttpClientConfiguration conf) throws IOException - { - super(conf); - this.con = con; - this.statusCode = con.getResponseCode(); - if (null == (is = con.getErrorStream())) { - is = con.getInputStream(); - } - if (is != null && "gzip".equals(con.getContentEncoding())) { - // the response is gzipped - is = new StreamingGZIPInputStream(is); - } - } - - // for test purpose - /*package*/ HttpResponseImpl(String content) { - super(); - this.responseAsString = content; - } - - @Override - public String getResponseHeader(String name) { - return con.getHeaderField(name); - } - - @Override - public Map> getResponseHeaderFields() { - return con.getHeaderFields(); - } - - /** - * {@inheritDoc} - */ - @Override - public void disconnect() { - con.disconnect(); - } -} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java b/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java deleted file mode 100644 index 5d1d639bcae..00000000000 --- a/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java +++ /dev/null @@ -1,10 +0,0 @@ -package druid.examples.twitter; - -/** - * @author Andrew Hedges - andrew.hedges at gmail.com - */ -public interface HttpResponseListener { - - public void httpResponseReceived(HttpResponseEvent event); - -} diff --git a/examples/src/main/java/druid/examples/twitter/JSONArray.java b/examples/src/main/java/druid/examples/twitter/JSONArray.java deleted file mode 100644 index 897d9825826..00000000000 --- a/examples/src/main/java/druid/examples/twitter/JSONArray.java +++ /dev/null @@ -1,726 +0,0 @@ -package druid.examples.twitter; - -import org.codehaus.jettison.json.JSONException; - -import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Array; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; - - -/* -Copyright (c) 2002 JSON.org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -The Software shall be used for Good, not Evil. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Array; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; - -/** - * A JSONArray is an ordered sequence of values. Its external text form is a - * string wrapped in square brackets with commas separating the values. The - * internal form is an object having get and opt - * methods for accessing the values by index, and put methods for - * adding or replacing values. The values can be any of these types: - * Boolean, JSONArray, JSONObject, - * Number, String, or the - * JSONObject.NULL object. - *

- * The constructor can convert a JSON text into a Java object. The - * toString method converts to JSON text. - *

- * A get method returns a value if one can be found, and throws an - * exception if one cannot be found. An opt method returns a - * default value instead of throwing an exception, and so is useful for - * obtaining optional values. - *

- * The generic get() and opt() methods return an - * object which you can cast or query for type. There are also typed - * get and opt methods that do type checking and type - * coercion for you. - *

- * The texts produced by the toString methods strictly conform to - * JSON syntax rules. The constructors are more forgiving in the texts they will - * accept: - *

    - *
  • An extra , (comma) may appear just - * before the closing bracket.
    • - *
  • The null value will be inserted when there - * is , (comma) elision.
    • - *
  • Strings may be quoted with ' (single - * quote).
    • - *
  • Strings do not need to be quoted at all if they do not begin with a quote - * or single quote, and if they do not contain leading or trailing spaces, - * and if they do not contain any of these characters: - * { } [ ] / \ : , = ; # and if they do not look like numbers - * and if they are not the reserved words true, - * false, or null.
    • - *
  • Values can be separated by ; (semicolon) as - * well as by , (comma).
    • - *
  • Numbers may have the - * 0x- (hex) prefix.
    • - *
- * - * @author JSON.org - * @version 2010-12-28 - */ -public class JSONArray { - - - /** - * The arrayList where the JSONArray's properties are kept. - */ - private ArrayList myArrayList; - - - /** - * Construct an empty JSONArray. - */ - public JSONArray() { - this.myArrayList = new ArrayList(); - } - - /** - * Construct a JSONArray from a JSONTokener. - * - * @param x A JSONTokener - * @throws JSONException If there is a syntax error. - */ - public JSONArray(JSONTokener x) throws JSONException - { - this(); - if (x.nextClean() != '[') { - throw x.syntaxError("A JSONArray text must start with '['"); - } - if (x.nextClean() != ']') { - x.back(); - for (; ; ) { - if (x.nextClean() == ',') { - x.back(); - this.myArrayList.add(twitter4j.internal.org.json.JSONObject.NULL); - } else { - x.back(); - this.myArrayList.add(x.nextValue()); - } - switch (x.nextClean()) { - case ';': - case ',': - if (x.nextClean() == ']') { - return; - } - x.back(); - break; - case ']': - return; - default: - throw x.syntaxError("Expected a ',' or ']'"); - } - } - } - } - - - /** - * Construct a JSONArray from a source JSON text. - * - * @param source A string that begins with - * [ (left bracket) - * and ends with ] (right bracket). - * @throws JSONException If there is a syntax error. - */ - public JSONArray(String source) throws JSONException { - this(new twitter4j.internal.org.json.JSONTokener(source)); - } - - - /** - * Construct a JSONArray from a Collection. - * - * @param collection A Collection. - */ - public JSONArray(Collection collection) { - this.myArrayList = new ArrayList(); - if (collection != null) { - for (Object aCollection : collection) { - this.myArrayList.add(twitter4j.internal.org.json.JSONObject.wrap(aCollection)); - } - } - } - - - /** - * Construct a JSONArray from an array - * - * @throws JSONException If not an array. - */ - public JSONArray(Object array) throws JSONException { - this(); - if (array.getClass().isArray()) { - int length = Array.getLength(array); - for (int i = 0; i < length; i += 1) { - this.put(twitter4j.internal.org.json.JSONObject.wrap(Array.get(array, i))); - } - } else { - throw new JSONException( - "JSONArray initial value should be a string or collection or array."); - } - } - - - /** - * Get the object value associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return An object value. - * @throws JSONException If there is no value for the index. - */ - public Object get(int index) throws JSONException { - Object object = opt(index); - if (object == null) { - throw new JSONException("JSONArray[" + index + "] not found."); - } - return object; - } - - - /** - * Get the boolean value associated with an index. - * The string values "true" and "false" are converted to boolean. - * - * @param index The index must be between 0 and length() - 1. - * @return The truth. - * @throws JSONException If there is no value for the index or if the - * value is not convertible to boolean. - */ - public boolean getBoolean(int index) throws JSONException { - Object object = get(index); - if (object.equals(Boolean.FALSE) || - (object instanceof String && - ((String) object).equalsIgnoreCase("false"))) { - return false; - } else if (object.equals(Boolean.TRUE) || - (object instanceof String && - ((String) object).equalsIgnoreCase("true"))) { - return true; - } - throw new JSONException("JSONArray[" + index + "] is not a boolean."); - } - - - /** - * Get the double value associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return The value. - * @throws JSONException If the key is not found or if the value cannot - * be converted to a number. - */ - public double getDouble(int index) throws JSONException { - Object object = get(index); - try { - return object instanceof Number ? - ((Number) object).doubleValue() : - Double.parseDouble((String) object); - } catch (Exception e) { - throw new JSONException("JSONArray[" + index + - "] is not a number."); - } - } - - - /** - * Get the int value associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return The value. - * @throws JSONException If the key is not found or if the value is not a number. - */ - public int getInt(int index) throws JSONException { - Object object = get(index); - try { - return object instanceof Number ? - ((Number) object).intValue() : - Integer.parseInt((String) object); - } catch (Exception e) { - throw new JSONException("JSONArray[" + index + - "] is not a number."); - } - } - - - /** - * Get the JSONArray associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return A JSONArray value. - * @throws JSONException If there is no value for the index. or if the - * value is not a JSONArray - */ - public JSONArray getJSONArray(int index) throws JSONException { - Object object = get(index); - if (object instanceof JSONArray) { - return (JSONArray) object; - } - throw new JSONException("JSONArray[" + index + - "] is not a JSONArray."); - } - - - /** - * Get the JSONObject associated with an index. - * - * @param index subscript - * @return A JSONObject value. - * @throws JSONException If there is no value for the index or if the - * value is not a JSONObject - */ - public twitter4j.internal.org.json.JSONObject getJSONObject(int index) throws JSONException { - Object object = get(index); - if (object instanceof twitter4j.internal.org.json.JSONObject) { - return (twitter4j.internal.org.json.JSONObject) object; - } - throw new JSONException("JSONArray[" + index + - "] is not a JSONObject."); - } - - - /** - * Get the long value associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return The value. - * @throws JSONException If the key is not found or if the value cannot - * be converted to a number. - */ - public long getLong(int index) throws JSONException { - Object object = get(index); - try { - return object instanceof Number ? - ((Number) object).longValue() : - Long.parseLong((String) object); - } catch (Exception e) { - throw new JSONException("JSONArray[" + index + - "] is not a number."); - } - } - - - /** - * Get the string associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return A string value. - * @throws JSONException If there is no value for the index. - */ - public String getString(int index) throws JSONException { - Object object = get(index); - return object == twitter4j.internal.org.json.JSONObject.NULL ? null : object.toString(); - } - - - /** - * Determine if the value is null. - * - * @param index The index must be between 0 and length() - 1. - * @return true if the value at the index is null, or if there is no value. - */ - public boolean isNull(int index) { - return twitter4j.internal.org.json.JSONObject.NULL.equals(opt(index)); - } - - - /** - * Make a string from the contents of this JSONArray. The - * separator string is inserted between each element. - * Warning: This method assumes that the data structure is acyclical. - * - * @param separator A string that will be inserted between the elements. - * @return a string. - * @throws JSONException If the array contains an invalid number. - */ - public String join(String separator) throws JSONException { - int len = length(); - StringBuilder sb = new StringBuilder(); - - for (int i = 0; i < len; i += 1) { - if (i > 0) { - sb.append(separator); - } - sb.append(JSONObject.valueToString(this.myArrayList.get(i))); - } - return sb.toString(); - } - - - /** - * Get the number of elements in the JSONArray, included nulls. - * - * @return The length (or size). - */ - public int length() { - return this.myArrayList.size(); - } - - - /** - * Get the optional object value associated with an index. - * - * @param index The index must be between 0 and length() - 1. - * @return An object value, or null if there is no - * object at that index. - */ - public Object opt(int index) { - return (index < 0 || index >= length()) ? - null : this.myArrayList.get(index); - } - - /** - * Append a boolean value. This increases the array's length by one. - * - * @param value A boolean value. - * @return this. - */ - public JSONArray put(boolean value) { - put(value ? Boolean.TRUE : Boolean.FALSE); - return this; - } - - - /** - * Put a value in the JSONArray, where the value will be a - * JSONArray which is produced from a Collection. - * - * @param value A Collection value. - * @return this. - */ - public JSONArray put(Collection value) { - put(new JSONArray(value)); - return this; - } - - /** - * Append an int value. This increases the array's length by one. - * - * @param value An int value. - * @return this. - */ - public JSONArray put(int value) { - put(new Integer(value)); - return this; - } - - - /** - * Append an long value. This increases the array's length by one. - * - * @param value A long value. - * @return this. - */ - public JSONArray put(long value) { - put(new Long(value)); - return this; - } - - - /** - * Put a value in the JSONArray, where the value will be a - * JSONObject which is produced from a Map. - * - * @param value A Map value. - * @return this. - */ - public JSONArray put(Map value) { - put(new twitter4j.internal.org.json.JSONObject(value)); - return this; - } - - - /** - * Append an object value. This increases the array's length by one. - * - * @param value An object value. The value should be a - * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the - * JSONObject.NULL object. - * @return this. - */ - public JSONArray put(Object value) { - this.myArrayList.add(value); - return this; - } - - - /** - * Put or replace a boolean value in the JSONArray. If the index is greater - * than the length of the JSONArray, then null elements will be added as - * necessary to pad it out. - * - * @param index The subscript. - * @param value A boolean value. - * @return this. - * @throws JSONException If the index is negative. - */ - public JSONArray put(int index, boolean value) throws JSONException { - put(index, value ? Boolean.TRUE : Boolean.FALSE); - return this; - } - - - /** - * Put a value in the JSONArray, where the value will be a - * JSONArray which is produced from a Collection. - * - * @param index The subscript. - * @param value A Collection value. - * @return this. - * @throws JSONException If the index is negative or if the value is - * not finite. - */ - public JSONArray put(int index, Collection value) throws JSONException { - put(index, new JSONArray(value)); - return this; - } - - - /** - * Put or replace a double value. If the index is greater than the length of - * the JSONArray, then null elements will be added as necessary to pad - * it out. - * - * @param index The subscript. - * @param value A double value. - * @return this. - * @throws JSONException If the index is negative or if the value is - * not finite. - */ - public JSONArray put(int index, double value) throws JSONException { - put(index, new Double(value)); - return this; - } - - - /** - * Put or replace an int value. If the index is greater than the length of - * the JSONArray, then null elements will be added as necessary to pad - * it out. - * - * @param index The subscript. - * @param value An int value. - * @return this. - * @throws JSONException If the index is negative. - */ - public JSONArray put(int index, int value) throws JSONException { - put(index, new Integer(value)); - return this; - } - - - /** - * Put or replace a long value. If the index is greater than the length of - * the JSONArray, then null elements will be added as necessary to pad - * it out. - * - * @param index The subscript. - * @param value A long value. - * @return this. - * @throws JSONException If the index is negative. - */ - public JSONArray put(int index, long value) throws JSONException { - put(index, new Long(value)); - return this; - } - - - /** - * Put a value in the JSONArray, where the value will be a - * JSONObject which is produced from a Map. - * - * @param index The subscript. - * @param value The Map value. - * @return this. - * @throws JSONException If the index is negative or if the the value is - * an invalid number. - */ - public JSONArray put(int index, Map value) throws JSONException { - put(index, new twitter4j.internal.org.json.JSONObject(value)); - return this; - } - - - /** - * Put or replace an object value in the JSONArray. If the index is greater - * than the length of the JSONArray, then null elements will be added as - * necessary to pad it out. - * - * @param index The subscript. - * @param value The value to put into the array. The value should be a - * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the - * JSONObject.NULL object. - * @return this. - * @throws JSONException If the index is negative or if the the value is - * an invalid number. - */ - public JSONArray put(int index, Object value) throws JSONException { - JSONObject.testValidity(value); - if (index < 0) { - throw new JSONException("JSONArray[" + index + "] not found."); - } - if (index < length()) { - this.myArrayList.set(index, value); - } else { - while (index != length()) { - put(twitter4j.internal.org.json.JSONObject.NULL); - } - put(value); - } - return this; - } - - /** - * Make a JSON text of this JSONArray. For compactness, no - * unnecessary whitespace is added. If it is not possible to produce a - * syntactically correct JSON text then null will be returned instead. This - * could occur if the array contains an invalid number. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @return a printable, displayable, transmittable - * representation of the array. - */ - public String toString() { - try { - return '[' + join(",") + ']'; - } catch (Exception e) { - return null; - } - } - - - /** - * Make a prettyprinted JSON text of this JSONArray. - * Warning: This method assumes that the data structure is acyclical. - * - * @param indentFactor The number of spaces to add to each level of - * indentation. - * @return a printable, displayable, transmittable - * representation of the object, beginning - * with [ (left bracket) and ending - * with ] (right bracket). - * @throws JSONException - */ - public String toString(int indentFactor) throws JSONException { - return toString(indentFactor, 0); - } - - - /** - * Make a prettyprinted JSON text of this JSONArray. - * Warning: This method assumes that the data structure is acyclical. - * - * @param indentFactor The number of spaces to add to each level of - * indentation. - * @param indent The indention of the top level. - * @return a printable, displayable, transmittable - * representation of the array. - * @throws JSONException - */ - String toString(int indentFactor, int indent) throws JSONException { - int len = length(); - if (len == 0) { - return "[]"; - } - int i; - StringBuilder sb = new StringBuilder("["); - if (len == 1) { - sb.append( - JSONObject.valueToString( - this.myArrayList.get(0), - indentFactor, indent - )); - } else { - int newindent = indent + indentFactor; - sb.append('\n'); - for (i = 0; i < len; i += 1) { - if (i > 0) { - sb.append(",\n"); - } - for (int j = 0; j < newindent; j += 1) { - sb.append(' '); - } - sb.append( - JSONObject.valueToString( - this.myArrayList.get(i), - indentFactor, newindent - )); - } - sb.append('\n'); - for (i = 0; i < indent; i += 1) { - sb.append(' '); - } - } - sb.append(']'); - return sb.toString(); - } - - - /** - * Write the contents of the JSONArray as JSON text to a writer. - * For compactness, no whitespace is added. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @return The writer. - * @throws JSONException - */ - public Writer write(Writer writer) throws JSONException { - try { - boolean b = false; - int len = length(); - - writer.write('['); - - for (int i = 0; i < len; i += 1) { - if (b) { - writer.write(','); - } - Object v = this.myArrayList.get(i); - if (v instanceof twitter4j.internal.org.json.JSONObject) { - ((JSONObject) v).write(writer); - } else if (v instanceof JSONArray) { - ((JSONArray) v).write(writer); - } else { - writer.write(JSONObject.valueToString(v)); - } - b = true; - } - writer.write(']'); - return writer; - } catch (IOException e) { - throw new JSONException(e); - } - } -} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/JSONObject.java b/examples/src/main/java/druid/examples/twitter/JSONObject.java deleted file mode 100644 index 373dcd10726..00000000000 --- a/examples/src/main/java/druid/examples/twitter/JSONObject.java +++ /dev/null @@ -1,1285 +0,0 @@ -package druid.examples.twitter; - - -import org.codehaus.jettison.json.JSONException; - -import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Locale; -import java.util.Map; -import java.util.ResourceBundle; -import java.util.TreeSet; - - -/* -Copyright (c) 2002 JSON.org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -The Software shall be used for Good, not Evil. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.*; - -/** - * A JSONObject is an unordered collection of name/value pairs. Its - * external form is a string wrapped in curly braces with colons between the - * names and values, and commas between the values and names. The internal form - * is an object having get and opt methods for - * accessing the values by name, and put methods for adding or - * replacing values by name. The values can be any of these types: - * Boolean, JSONArray, JSONObject, - * Number, String, or the JSONObject.NULL - * object. A JSONObject constructor can be used to convert an external form - * JSON text into an internal form whose values can be retrieved with the - * get and opt methods, or to convert values into a - * JSON text using the put and toString methods. - * A get method returns a value if one can be found, and throws an - * exception if one cannot be found. An opt method returns a - * default value instead of throwing an exception, and so is useful for - * obtaining optional values. - *

- * The generic get() and opt() methods return an - * object, which you can cast or query for type. There are also typed - * get and opt methods that do type checking and type - * coercion for you. The opt methods differ from the get methods in that they - * do not throw. Instead, they return a specified value, such as null. - *

- * The put methods add or replace values in an object. For example, - * 

myString = new JSONObject().put("JSON", "Hello, World!").toString();
- * produces the string {"JSON": "Hello, World"}. - *

- * The texts produced by the toString methods strictly conform to - * the JSON syntax rules. - * The constructors are more forgiving in the texts they will accept: - *

    - *
  • An extra , (comma) may appear just - * before the closing brace.
    • - *
  • Strings may be quoted with ' (single - * quote).
    • - *
  • Strings do not need to be quoted at all if they do not begin with a quote - * or single quote, and if they do not contain leading or trailing spaces, - * and if they do not contain any of these characters: - * { } [ ] / \ : , = ; # and if they do not look like numbers - * and if they are not the reserved words true, - * false, or null.
    • - *
  • Keys can be followed by = or => as well as - * by :.
    • - *
  • Values can be followed by ; (semicolon) as - * well as by , (comma).
    • - *
  • Numbers may have the 0x- (hex) prefix.
    • - *
- * - * @author JSON.org - * @version 2010-12-28 - */ -public class JSONObject { - - /** - * JSONObject.NULL is equivalent to the value that JavaScript calls null, - * whilst Java's null is equivalent to the value that JavaScript calls - * undefined. - */ - private static final class Null { - - /** - * There is only intended to be a single instance of the NULL object, - * so the clone method returns itself. - * - * @return NULL. - */ - protected final Object clone() { - return this; - } - - /** - * A Null object is equal to the null value and to itself. - * - * @param object An object to test for nullness. - * @return true if the object parameter is the JSONObject.NULL object - * or null. - */ - public boolean equals(Object object) { - return object == null || object == this; - } - - /** - * Get the "null" string value. - * - * @return The string "null". - */ - public String toString() { - return "null"; - } - } - - - /** - * The map where the JSONObject's properties are kept. - */ - private Map map; - - - /** - * It is sometimes more convenient and less ambiguous to have a - * NULL object than to use Java's null value. - * JSONObject.NULL.equals(null) returns true. - * JSONObject.NULL.toString() returns "null". - */ - public static final Object NULL = new Null(); - - - /** - * Construct an empty JSONObject. - */ - public JSONObject() { - this.map = new HashMap(); - } - - - /** - * Construct a JSONObject from a subset of another JSONObject. - * An array of strings is used to identify the keys that should be copied. - * Missing keys are ignored. - * - * @param jo A JSONObject. - * @param names An array of strings. - */ - public JSONObject(JSONObject jo, String[] names) { - this(); - for (int i = 0; i < names.length; i += 1) { - try { - putOnce(names[i], jo.opt(names[i])); - } catch (Exception ignore) { - } - } - } - - - /** - * Construct a JSONObject from a JSONTokener. - * - * @param x A JSONTokener object containing the source string. - * @throws twitter4j.internal.org.json.JSONException If there is a syntax error in the source string - * or a duplicated key. - */ - public JSONObject(JSONTokener x) throws JSONException - { - this(); - char c; - String key; - - if (x.nextClean() != '{') { - throw x.syntaxError("A JSONObject text must begin with '{' found:" + x.nextClean()); - } - for (; ; ) { - c = x.nextClean(); - switch (c) { - case 0: - throw x.syntaxError("A JSONObject text must end with '}'"); - case '}': - return; - default: - x.back(); - key = x.nextValue().toString(); - } - -// The key is followed by ':'. We will also tolerate '=' or '=>'. - - c = x.nextClean(); - if (c == '=') { - if (x.next() != '>') { - x.back(); - } - } else if (c != ':') { - throw x.syntaxError("Expected a ':' after a key"); - } - putOnce(key, x.nextValue()); - -// Pairs are separated by ','. We will also tolerate ';'. - - switch (x.nextClean()) { - case ';': - case ',': - if (x.nextClean() == '}') { - return; - } - x.back(); - break; - case '}': - return; - default: - throw x.syntaxError("Expected a ',' or '}'"); - } - } - } - - - /** - * Construct a JSONObject from a Map. - * - * @param map A map object that can be used to initialize the contents of - * the JSONObject. - * @throws JSONException - */ - public JSONObject(Map map) { - this.map = new HashMap(); - if (map != null) { - Iterator i = map.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - Object value = e.getValue(); - if (value != null) { - this.map.put(e.getKey(), wrap(value)); - } - } - } - } - - - /** - * Construct a JSONObject from an Object using bean getters. - * It reflects on all of the public methods of the object. - * For each of the methods with no parameters and a name starting - * with "get" or "is" followed by an uppercase letter, - * the method is invoked, and a key and the value returned from the getter method - * are put into the new JSONObject. - *

- * The key is formed by removing the "get" or "is" prefix. - * If the second remaining character is not upper case, then the first - * character is converted to lower case. - *

- * For example, if an object has a method named "getName", and - * if the result of calling object.getName() is "Larry Fine", - * then the JSONObject will contain "name": "Larry Fine". - * - * @param bean An object that has getter methods that should be used - * to make a JSONObject. - */ - public JSONObject(Object bean) { - this(); - populateMap(bean); - } - - - /** - * Construct a JSONObject from an Object, using reflection to find the - * public members. The resulting JSONObject's keys will be the strings - * from the names array, and the values will be the field values associated - * with those keys in the object. If a key is not found or not visible, - * then it will not be copied into the new JSONObject. - * - * @param object An object that has fields that should be used to make a - * JSONObject. - * @param names An array of strings, the names of the fields to be obtained - * from the object. - */ - public JSONObject(Object object, String names[]) { - this(); - Class c = object.getClass(); - for (int i = 0; i < names.length; i += 1) { - String name = names[i]; - try { - putOpt(name, c.getField(name).get(object)); - } catch (Exception ignore) { - } - } - } - - - /** - * Construct a JSONObject from a source JSON text string. - * This is the most commonly used JSONObject constructor. - * - * @param source A string beginning - * with { (left brace) and ending - * with } (right brace). - * @throws JSONException If there is a syntax error in the source - * string or a duplicated key. - */ - public JSONObject(String source) throws JSONException { - this(new twitter4j.internal.org.json.JSONTokener(source)); - } - - - /** - * Construct a JSONObject from a ResourceBundle. - * - * @param baseName The ResourceBundle base name. - * @param locale The Locale to load the ResourceBundle for. - * @throws JSONException If any JSONExceptions are detected. - */ - public JSONObject(String baseName, Locale locale) throws JSONException { - this(); - ResourceBundle r = ResourceBundle.getBundle(baseName, locale, - Thread.currentThread().getContextClassLoader()); - -// Iterate through the keys in the bundle. - - Enumeration keys = r.getKeys(); - while (keys.hasMoreElements()) { - Object key = keys.nextElement(); - if (key instanceof String) { - -// Go through the path, ensuring that there is a nested JSONObject for each -// segment except the last. Add the value using the last segment's name into -// the deepest nested JSONObject. - - String[] path = ((String) key).split("\\."); - int last = path.length - 1; - JSONObject target = this; - for (int i = 0; i < last; i += 1) { - String segment = path[i]; - Object object = target.opt(segment); - JSONObject nextTarget = object instanceof JSONObject ? (JSONObject) object : null; - if (nextTarget == null) { - nextTarget = new JSONObject(); - target.put(segment, nextTarget); - } - target = nextTarget; - } - target.put(path[last], r.getString((String) key)); - } - } - } - - /** - * Append values to the array under a key. If the key does not exist in the - * JSONObject, then the key is put in the JSONObject with its value being a - * JSONArray containing the value parameter. If the key was already - * associated with a JSONArray, then the value parameter is appended to it. - * - * @param key A key string. - * @param value An object to be accumulated under the key. - * @return this. - * @throws JSONException If the key is null or if the current value - * associated with the key is not a JSONArray. - */ - public JSONObject append(String key, Object value) throws JSONException { - testValidity(value); - Object object = opt(key); - if (object == null) { - put(key, new JSONArray().put(value)); - } else if (object instanceof JSONArray) { - put(key, ((JSONArray) object).put(value)); - } else { - throw new JSONException("JSONObject[" + key + - "] is not a JSONArray."); - } - return this; - } - - /** - * Get the value object associated with a key. - * - * @param key A key string. - * @return The object associated with the key. - * @throws JSONException if the key is not found. - */ - public Object get(String key) throws JSONException { - if (key == null) { - throw new JSONException("Null key."); - } - Object object = opt(key); - if (object == null) { - throw new JSONException("JSONObject[" + quote(key) + - "] not found."); - } - return object; - } - - - /** - * Get the boolean value associated with a key. - * - * @param key A key string. - * @return The truth. - * @throws JSONException if the value is not a Boolean or the String "true" or "false". - */ - public boolean getBoolean(String key) throws JSONException { - Object object = get(key); - if (object.equals(Boolean.FALSE) || - (object instanceof String && - ((String) object).equalsIgnoreCase("false"))) { - return false; - } else if (object.equals(Boolean.TRUE) || - (object instanceof String && - ((String) object).equalsIgnoreCase("true"))) { - return true; - } - throw new JSONException("JSONObject[" + quote(key) + - "] is not a Boolean."); - } - - /** - * Get the int value associated with a key. - * - * @param key A key string. - * @return The integer value. - * @throws JSONException if the key is not found or if the value cannot - * be converted to an integer. - */ - public int getInt(String key) throws JSONException { - Object object = get(key); - try { - return object instanceof Number ? - ((Number) object).intValue() : - Integer.parseInt((String) object); - } catch (Exception e) { - throw new JSONException("JSONObject[" + quote(key) + - "] is not an int."); - } - } - - - /** - * Get the JSONArray value associated with a key. - * - * @param key A key string. - * @return A JSONArray which is the value. - * @throws JSONException if the key is not found or - * if the value is not a JSONArray. - */ - public JSONArray getJSONArray(String key) throws JSONException { - Object object = get(key); - if (object instanceof JSONArray) { - return (JSONArray) object; - } - throw new JSONException("JSONObject[" + quote(key) + - "] is not a JSONArray."); - } - - - /** - * Get the JSONObject value associated with a key. - * - * @param key A key string. - * @return A JSONObject which is the value. - * @throws JSONException if the key is not found or - * if the value is not a JSONObject. - */ - public JSONObject getJSONObject(String key) throws JSONException { - Object object = get(key); - if (object instanceof JSONObject) { - return (JSONObject) object; - } - throw new JSONException("JSONObject[" + quote(key) + - "] is not a JSONObject."); - } - - - /** - * Get the long value associated with a key. - * - * @param key A key string. - * @return The long value. - * @throws JSONException if the key is not found or if the value cannot - * be converted to a long. - */ - public long getLong(String key) throws JSONException { - Object object = get(key); - try { - return object instanceof Number ? - ((Number) object).longValue() : - Long.parseLong((String) object); - } catch (Exception e) { - throw new JSONException("JSONObject[" + quote(key) + - "] is not a long."); - } - } - - /** - * Get the string associated with a key. - * - * @param key A key string. - * @return A string which is the value. - * @throws JSONException if the key is not found. - */ - public String getString(String key) throws JSONException { - Object object = get(key); - return object == NULL ? null : object.toString(); - } - - - /** - * Determine if the JSONObject contains a specific key. - * - * @param key A key string. - * @return true if the key exists in the JSONObject. - */ - public boolean has(String key) { - return this.map.containsKey(key); - } - - /** - * Determine if the value associated with the key is null or if there is - * no value. - * - * @param key A key string. - * @return true if there is no value associated with the key or if - * the value is the JSONObject.NULL object. - */ - public boolean isNull(String key) { - return JSONObject.NULL.equals(opt(key)); - } - - - /** - * Get an enumeration of the keys of the JSONObject. - * - * @return An iterator of the keys. - */ - public Iterator keys() { - return this.map.keySet().iterator(); - } - - - /** - * Get the number of keys stored in the JSONObject. - * - * @return The number of keys in the JSONObject. - */ - public int length() { - return this.map.size(); - } - - - /** - * Produce a JSONArray containing the names of the elements of this - * JSONObject. - * - * @return A JSONArray containing the key strings, or null if the JSONObject - * is empty. - */ - public JSONArray names() { - JSONArray ja = new JSONArray(); - Iterator keys = keys(); - while (keys.hasNext()) { - ja.put(keys.next()); - } - return ja.length() == 0 ? null : ja; - } - - /** - * Produce a string from a Number. - * - * @param number A Number - * @return A String. - * @throws JSONException If n is a non-finite number. - */ - public static String numberToString(Number number) - throws JSONException { - if (number == null) { - throw new JSONException("Null pointer"); - } - testValidity(number); - -// Shave off trailing zeros and decimal point, if possible. - - String string = number.toString(); - if (string.indexOf('.') > 0 && string.indexOf('e') < 0 && - string.indexOf('E') < 0) { - while (string.endsWith("0")) { - string = string.substring(0, string.length() - 1); - } - if (string.endsWith(".")) { - string = string.substring(0, string.length() - 1); - } - } - return string; - } - - - /** - * Get an optional value associated with a key. - * - * @param key A key string. - * @return An object which is the value, or null if there is no value. - */ - public Object opt(String key) { - return key == null ? null : this.map.get(key); - } - - - private void populateMap(Object bean) { - Class klass = bean.getClass(); - -// If klass is a System class then set includeSuperClass to false. - - boolean includeSuperClass = klass.getClassLoader() != null; - - Method[] methods = (includeSuperClass) ? - klass.getMethods() : klass.getDeclaredMethods(); - for (int i = 0; i < methods.length; i += 1) { - try { - Method method = methods[i]; - if (Modifier.isPublic(method.getModifiers())) { - String name = method.getName(); - String key = ""; - if (name.startsWith("get")) { - if (name.equals("getClass") || - name.equals("getDeclaringClass")) { - key = ""; - } else { - key = name.substring(3); - } - } else if (name.startsWith("is")) { - key = name.substring(2); - } - if (key.length() > 0 && - Character.isUpperCase(key.charAt(0)) && - method.getParameterTypes().length == 0) { - if (key.length() == 1) { - key = key.toLowerCase(); - } else if (!Character.isUpperCase(key.charAt(1))) { - key = key.substring(0, 1).toLowerCase() + - key.substring(1); - } - - Object result = method.invoke(bean, (Object[]) null); - if (result != null) { - map.put(key, wrap(result)); - } - } - } - } catch (Exception ignore) { - } - } - } - - - /** - * Put a key/boolean pair in the JSONObject. - * - * @param key A key string. - * @param value A boolean which is the value. - * @return this. - * @throws JSONException If the key is null. - */ - public JSONObject put(String key, boolean value) throws JSONException { - put(key, value ? Boolean.TRUE : Boolean.FALSE); - return this; - } - - - /** - * Put a key/value pair in the JSONObject, where the value will be a - * JSONArray which is produced from a Collection. - * - * @param key A key string. - * @param value A Collection value. - * @return this. - * @throws JSONException - */ - public JSONObject put(String key, Collection value) throws JSONException { - put(key, new JSONArray(value)); - return this; - } - - - /** - * Put a key/double pair in the JSONObject. - * - * @param key A key string. - * @param value A double which is the value. - * @return this. - * @throws JSONException If the key is null or if the number is invalid. - */ - public JSONObject put(String key, double value) throws JSONException { - put(key, new Double(value)); - return this; - } - - - /** - * Put a key/int pair in the JSONObject. - * - * @param key A key string. - * @param value An int which is the value. - * @return this. - * @throws JSONException If the key is null. - */ - public JSONObject put(String key, int value) throws JSONException { - put(key, new Integer(value)); - return this; - } - - - /** - * Put a key/long pair in the JSONObject. - * - * @param key A key string. - * @param value A long which is the value. - * @return this. - * @throws JSONException If the key is null. - */ - public JSONObject put(String key, long value) throws JSONException { - put(key, new Long(value)); - return this; - } - - - /** - * Put a key/value pair in the JSONObject, where the value will be a - * JSONObject which is produced from a Map. - * - * @param key A key string. - * @param value A Map value. - * @return this. - * @throws JSONException - */ - public JSONObject put(String key, Map value) throws JSONException { - put(key, new JSONObject(value)); - return this; - } - - - /** - * Put a key/value pair in the JSONObject. If the value is null, - * then the key will be removed from the JSONObject if it is present. - * - * @param key A key string. - * @param value An object which is the value. It should be of one of these - * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, - * or the JSONObject.NULL object. - * @return this. - * @throws JSONException If the value is non-finite number - * or if the key is null. - */ - public JSONObject put(String key, Object value) throws JSONException { - if (key == null) { - throw new JSONException("Null key."); - } - if (value != null) { - testValidity(value); - this.map.put(key, value); - } else { - remove(key); - } - return this; - } - - - /** - * Put a key/value pair in the JSONObject, but only if the key and the - * value are both non-null, and only if there is not already a member - * with that name. - * - * @param key - * @param value - * @return his. - * @throws JSONException if the key is a duplicate - */ - public JSONObject putOnce(String key, Object value) throws JSONException { - if (key != null && value != null) { - if (opt(key) != null) { - throw new JSONException("Duplicate key \"" + key + "\""); - } - put(key, value); - } - return this; - } - - - /** - * Put a key/value pair in the JSONObject, but only if the - * key and the value are both non-null. - * - * @param key A key string. - * @param value An object which is the value. It should be of one of these - * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, - * or the JSONObject.NULL object. - * @return this. - * @throws JSONException If the value is a non-finite number. - */ - public JSONObject putOpt(String key, Object value) throws JSONException { - if (key != null && value != null) { - put(key, value); - } - return this; - } - - - /** - * Produce a string in double quotes with backslash sequences in all the - * right places. A backslash will be inserted within = '\u0080' && c < '\u00a0') || - (c >= '\u2000' && c < '\u2100')) { - hhhh = "000" + Integer.toHexString(c); - sb.append("\\u").append(hhhh.substring(hhhh.length() - 4)); - } else { - sb.append(c); - } - } - } - sb.append('"'); - return sb.toString(); - } - - /** - * Remove a name and its value, if present. - * - * @param key The name to be removed. - * @return The value that was associated with the name, - * or null if there was no value. - */ - public Object remove(String key) { - return this.map.remove(key); - } - - /** - * Get an enumeration of the keys of the JSONObject. - * The keys will be sorted alphabetically. - * - * @return An iterator of the keys. - */ - public Iterator sortedKeys() { - return new TreeSet(this.map.keySet()).iterator(); - } - - /** - * Try to convert a string into a number, boolean, or null. If the string - * can't be converted, return the string. - * - * @param string A String. - * @return A simple JSON value. - */ - public static Object stringToValue(String string) { - if (string.equals("")) { - return string; - } - if (string.equalsIgnoreCase("true")) { - return Boolean.TRUE; - } - if (string.equalsIgnoreCase("false")) { - return Boolean.FALSE; - } - if (string.equalsIgnoreCase("null")) { - return JSONObject.NULL; - } - - /* - * If it might be a number, try converting it. - * We support the non-standard 0x- convention. - * If a number cannot be produced, then the value will just - * be a string. Note that the 0x-, plus, and implied string - * conventions are non-standard. A JSON parser may accept - * non-JSON forms as long as it accepts all correct JSON forms. - */ - - char b = string.charAt(0); - if ((b >= '0' && b <= '9') || b == '.' || b == '-' || b == '+') { - if (b == '0' && string.length() > 2 && - (string.charAt(1) == 'x' || string.charAt(1) == 'X')) { - try { - return Integer.parseInt(string.substring(2), 16); - } catch (Exception ignore) { - } - } - try { - if (string.indexOf('.') > -1 || - string.indexOf('e') > -1 || string.indexOf('E') > -1) { - return Double.valueOf(string); - } else { - Long myLong = new Long(string); - if (myLong == myLong.intValue()) { - return myLong.intValue(); - } else { - return myLong; - } - } - } catch (Exception ignore) { - } - } - return string; - } - - - /** - * Throw an exception if the object is a NaN or infinite number. - * - * @param o The object to test. - * @throws JSONException If o is a non-finite number. - */ - public static void testValidity(Object o) throws JSONException { - if (o != null) { - if (o instanceof Double) { - if (((Double) o).isInfinite() || ((Double) o).isNaN()) { - throw new JSONException( - "JSON does not allow non-finite numbers."); - } - } else if (o instanceof Float) { - if (((Float) o).isInfinite() || ((Float) o).isNaN()) { - throw new JSONException( - "JSON does not allow non-finite numbers."); - } - } - } - } - - /** - * Make a JSON text of this JSONObject. For compactness, no whitespace - * is added. If this would not result in a syntactically correct JSON text, - * then null will be returned instead. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @return a printable, displayable, portable, transmittable - * representation of the object, beginning - * with { (left brace) and ending - * with } (right brace). - */ - public String toString() { - try { - Iterator keys = keys(); - StringBuilder sb = new StringBuilder("{"); - - while (keys.hasNext()) { - if (sb.length() > 1) { - sb.append(','); - } - Object o = keys.next(); - sb.append(quote(o.toString())); - sb.append(':'); - sb.append(valueToString(this.map.get(o))); - } - sb.append('}'); - return sb.toString(); - } catch (Exception e) { - return null; - } - } - - - /** - * Make a prettyprinted JSON text of this JSONObject. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @param indentFactor The number of spaces to add to each level of - * indentation. - * @return a printable, displayable, portable, transmittable - * representation of the object, beginning - * with { (left brace) and ending - * with } (right brace). - * @throws JSONException If the object contains an invalid number. - */ - public String toString(int indentFactor) throws JSONException { - return toString(indentFactor, 0); - } - - - /** - * Make a prettyprinted JSON text of this JSONObject. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @param indentFactor The number of spaces to add to each level of - * indentation. - * @param indent The indentation of the top level. - * @return a printable, displayable, transmittable - * representation of the object, beginning - * with { (left brace) and ending - * with } (right brace). - * @throws JSONException If the object contains an invalid number. - */ - String toString(int indentFactor, int indent) throws JSONException { - int i; - int length = this.length(); - if (length == 0) { - return "{}"; - } - Iterator keys = sortedKeys(); - int newindent = indent + indentFactor; - Object object; - StringBuilder sb = new StringBuilder("{"); - if (length == 1) { - object = keys.next(); - sb.append(quote(object.toString())); - sb.append(": "); - sb.append(valueToString(this.map.get(object), indentFactor, - indent)); - } else { - while (keys.hasNext()) { - object = keys.next(); - if (sb.length() > 1) { - sb.append(",\n"); - } else { - sb.append('\n'); - } - for (i = 0; i < newindent; i += 1) { - sb.append(' '); - } - sb.append(quote(object.toString())); - sb.append(": "); - sb.append(valueToString(this.map.get(object), indentFactor, - newindent)); - } - if (sb.length() > 1) { - sb.append('\n'); - for (i = 0; i < indent; i += 1) { - sb.append(' '); - } - } - } - sb.append('}'); - return sb.toString(); - } - - - /** - * Make a JSON text of an Object value. If the object has an - * value.toJSONString() method, then that method will be used to produce - * the JSON text. The method is required to produce a strictly - * conforming text. If the object does not contain a toJSONString - * method (which is the most common case), then a text will be - * produced by other means. If the value is an array or Collection, - * then a JSONArray will be made from it and its toJSONString method - * will be called. If the value is a MAP, then a JSONObject will be made - * from it and its toJSONString method will be called. Otherwise, the - * value's toString method will be called, and the result will be quoted. - *

- *

- * Warning: This method assumes that the data structure is acyclical. - * - * @param value The value to be serialized. - * @return a printable, displayable, transmittable - * representation of the object, beginning - * with { (left brace) and ending - * with } (right brace). - * @throws JSONException If the value is or contains an invalid number. - */ - public static String valueToString(Object value) throws JSONException { - if (value == null || value.equals(null)) { - return "null"; - } - if (value instanceof Number) { - return numberToString((Number) value); - } - if (value instanceof Boolean || value instanceof JSONObject || - value instanceof JSONArray) { - return value.toString(); - } - if (value instanceof Map) { - return new JSONObject((Map) value).toString(); - } - if (value instanceof Collection) { - return new JSONArray((Collection) value).toString(); - } - if (value.getClass().isArray()) { - return new JSONArray(value).toString(); - } - return quote(value.toString()); - } - - - /** - * Make a prettyprinted JSON text of an object value. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @param value The value to be serialized. - * @param indentFactor The number of spaces to add to each level of - * indentation. - * @param indent The indentation of the top level. - * @return a printable, displayable, transmittable - * representation of the object, beginning - * with { (left brace) and ending - * with } (right brace). - * @throws JSONException If the object contains an invalid number. - */ - static String valueToString(Object value, int indentFactor, int indent) - throws JSONException { - if (value == null || value.equals(null)) { - return "null"; - } - if (value instanceof Number) { - return numberToString((Number) value); - } - if (value instanceof Boolean) { - return value.toString(); - } - if (value instanceof JSONObject) { - return ((JSONObject) value).toString(indentFactor, indent); - } - if (value instanceof JSONArray) { - return ((JSONArray) value).toString(indentFactor, indent); - } - if (value instanceof Map) { - return new JSONObject((Map) value).toString(indentFactor, indent); - } - if (value instanceof Collection) { - return new JSONArray((Collection) value).toString(indentFactor, indent); - } - if (value.getClass().isArray()) { - return new JSONArray(value).toString(indentFactor, indent); - } - return quote(value.toString()); - } - - - /** - * Wrap an object, if necessary. If the object is null, return the NULL - * object. If it is an array or collection, wrap it in a JSONArray. If - * it is a map, wrap it in a JSONObject. If it is a standard property - * (Double, String, et al) then it is already wrapped. Otherwise, if it - * comes from one of the java packages, turn it into a string. And if - * it doesn't, try to wrap it in a JSONObject. If the wrapping fails, - * then null is returned. - * - * @param object The object to wrap - * @return The wrapped value - */ - public static Object wrap(Object object) { - try { - if (object == null) { - return NULL; - } - if (object instanceof JSONObject || object instanceof JSONArray || - NULL.equals(object) || - object instanceof Byte || object instanceof Character || - object instanceof Short || object instanceof Integer || - object instanceof Long || object instanceof Boolean || - object instanceof Float || object instanceof Double || - object instanceof String) { - return object; - } - - if (object instanceof Collection) { - return new JSONArray((Collection) object); - } - if (object.getClass().isArray()) { - return new JSONArray(object); - } - if (object instanceof Map) { - return new JSONObject((Map) object); - } - Package objectPackage = object.getClass().getPackage(); - String objectPackageName = (objectPackage != null ? objectPackage.getName() : ""); - if (objectPackageName.startsWith("java.") || - objectPackageName.startsWith("javax.") || - object.getClass().getClassLoader() == null) { - return object.toString(); - } - return new JSONObject(object); - } catch (Exception exception) { - return null; - } - } - - - /** - * Write the contents of the JSONObject as JSON text to a writer. - * For compactness, no whitespace is added. - *

- * Warning: This method assumes that the data structure is acyclical. - * - * @return The writer. - * @throws JSONException - */ - public Writer write(Writer writer) throws JSONException { - try { - boolean commanate = false; - Iterator keys = keys(); - writer.write('{'); - - while (keys.hasNext()) { - if (commanate) { - writer.write(','); - } - Object key = keys.next(); - writer.write(quote(key.toString())); - writer.write(':'); - Object value = this.map.get(key); - if (value instanceof JSONObject) { - ((JSONObject) value).write(writer); - } else if (value instanceof JSONArray) { - ((JSONArray) value).write(writer); - } else { - writer.write(valueToString(value)); - } - commanate = true; - } - writer.write('}'); - return writer; - } catch (IOException exception) { - throw new JSONException(exception); - } - } -} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/JSONTokener.java b/examples/src/main/java/druid/examples/twitter/JSONTokener.java deleted file mode 100644 index db694a335a1..00000000000 --- a/examples/src/main/java/druid/examples/twitter/JSONTokener.java +++ /dev/null @@ -1,359 +0,0 @@ -package druid.examples.twitter; - -import org.codehaus.jettison.json.JSONException; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringReader; - - -import java.io.*; - -/* -Copyright (c) 2002 JSON.org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -The Software shall be used for Good, not Evil. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -/** - * A JSONTokener takes a source string and extracts characters and tokens from - * it. It is used by the JSONObject and JSONArray constructors to parse - * JSON source strings. - * - * @author JSON.org - * @version 2010-12-24 - */ -public class JSONTokener { - - private int character; - private boolean eof; - private int index; - private int line; - private char previous; - private Reader reader; - private boolean usePrevious; - - - /** - * Construct a JSONTokener from a Reader. - * - * @param reader A reader. - */ - public JSONTokener(Reader reader) { - this.reader = reader.markSupported() ? - reader : new BufferedReader(reader); - this.eof = false; - this.usePrevious = false; - this.previous = 0; - this.index = 0; - this.character = 1; - this.line = 1; - } - - - /** - * Construct a JSONTokener from an InputStream. - */ - public JSONTokener(InputStream inputStream) throws JSONException - { - this(new InputStreamReader(inputStream)); - } - - - /** - * Construct a JSONTokener from a string. - * - * @param s A source string. - */ - public JSONTokener(String s) { - this(new StringReader(s)); - } - - - /** - * Back up one character. This provides a sort of lookahead capability, - * so that you can test for a digit or letter before attempting to parse - * the next number or identifier. - */ - public void back() throws JSONException { - if (usePrevious || index <= 0) { - throw new JSONException("Stepping back two steps is not supported"); - } - this.index -= 1; - this.character -= 1; - this.usePrevious = true; - this.eof = false; - } - - - public boolean end() { - return eof && !usePrevious; - } - - - /** - * Determine if the source string still contains characters that next() - * can consume. - * - * @return true if not yet at the end of the source. - */ - public boolean more() throws JSONException { - next(); - if (end()) { - return false; - } - back(); - return true; - } - - - /** - * Get the next character in the source string. - * - * @return The next character, or 0 if past the end of the source string. - */ - public char next() throws JSONException { - int c; - if (this.usePrevious) { - this.usePrevious = false; - c = this.previous; - } else { - try { - c = this.reader.read(); - } catch (IOException exception) { - throw new JSONException(exception); - } - - if (c <= 0) { // End of stream - this.eof = true; - c = 0; - } - } - this.index += 1; - if (this.previous == '\r') { - this.line += 1; - this.character = c == '\n' ? 0 : 1; - } else if (c == '\n') { - this.line += 1; - this.character = 0; - } else { - this.character += 1; - } - this.previous = (char) c; - return this.previous; - } - - - /** - * Consume the next character, and check that it matches a specified - * character. - * - * @param c The character to match. - * @return The character. - * @throws JSONException if the character does not match. - */ - public char next(char c) throws JSONException { - char n = next(); - if (n != c) { - throw syntaxError("Expected '" + c + "' and instead saw '" + - n + "'"); - } - return n; - } - - - /** - * Get the next n characters. - * - * @param n The number of characters to take. - * @return A string of n characters. - * @throws JSONException Substring bounds error if there are not - * n characters remaining in the source string. - */ - public String next(int n) throws JSONException { - if (n == 0) { - return ""; - } - - char[] chars = new char[n]; - int pos = 0; - - while (pos < n) { - chars[pos] = next(); - if (end()) { - throw syntaxError("Substring bounds error"); - } - pos += 1; - } - return new String(chars); - } - - - /** - * Get the next char in the string, skipping whitespace. - * - * @return A character, or 0 if there are no more characters. - * @throws JSONException - */ - public char nextClean() throws JSONException { - for (; ; ) { - char c = next(); - if (c == 0 || c > ' ') { - return c; - } - } - } - - - /** - * Return the characters up to the next close quote character. - * Backslash processing is done. The formal JSON format does not - * allow strings in single quotes, but an implementation is allowed to - * accept them. - * - * @param quote The quoting character, either - * " (double quote) or - * ' (single quote). - * @return A String. - * @throws JSONException Unterminated string. - */ - public String nextString(char quote) throws JSONException { - char c; - StringBuilder sb = new StringBuilder(); - for (; ; ) { - c = next(); - switch (c) { - case 0: - case '\n': - case '\r': - throw syntaxError("Unterminated string"); - case '\\': - c = next(); - switch (c) { - case 'b': - sb.append('\b'); - break; - case 't': - sb.append('\t'); - break; - case 'n': - sb.append('\n'); - break; - case 'f': - sb.append('\f'); - break; - case 'r': - sb.append('\r'); - break; - case 'u': - sb.append((char) Integer.parseInt(next(4), 16)); - break; - case '"': - case '\'': - case '\\': - case '/': - sb.append(c); - break; - default: - throw syntaxError("Illegal escape."); - } - break; - default: - if (c == quote) { - return sb.toString(); - } - sb.append(c); - } - } - } - - /** - * Get the next value. The value can be a Boolean, Double, Integer, - * JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object. - * - * @return An object. - * @throws JSONException If syntax error. - */ - public Object nextValue() throws JSONException { - char c = nextClean(); - String string; - - switch (c) { - case '"': - case '\'': - return nextString(c); - case '{': - back(); - return new JSONObject(this); - case '[': - back(); - return new JSONArray(this); - } - - /* - * Handle unquoted text. This could be the values true, false, or - * null, or it can be a number. An implementation (such as this one) - * is allowed to also accept non-standard forms. - * - * Accumulate characters until we reach the end of the text or a - * formatting character. - */ - - StringBuilder sb = new StringBuilder(); - while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) { - sb.append(c); - c = next(); - } - back(); - - string = sb.toString().trim(); - if (string.equals("")) { - throw syntaxError("Missing value"); - } - return JSONObject.stringToValue(string); - } - - - - /** - * Make a JSONException to signal a syntax error. - * - * @param message The error message. - * @return A JSONException object, suitable for throwing - */ - public JSONException syntaxError(String message) { - return new JSONException(message + toString()); - } - - - /** - * Make a printable string of this JSONTokener. - * - * @return " at {index} [character {character} line {line}]" - */ - public String toString() { - return " at " + index + " [character " + this.character + " line " + - this.line + "]"; - } -} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/Logger.java b/examples/src/main/java/druid/examples/twitter/Logger.java deleted file mode 100644 index 0f9a4eb3769..00000000000 --- a/examples/src/main/java/druid/examples/twitter/Logger.java +++ /dev/null @@ -1,180 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - - - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -/** - * @author Yusuke Yamamoto - yusuke at mac.com - * @since Twitter4J 2.1.0 - */ -public abstract class Logger { - private static final LoggerFactory LOGGER_FACTORY; - private static final String LOGGER_FACTORY_IMPLEMENTATION = "twitter4j.loggerFactory"; - - static { - LoggerFactory loggerFactory = null; - // -Dtwitter4j.debug=true -Dtwitter4j.loggerFactory=twitter4j.internal.logging.StdOutLoggerFactory - String loggerFactoryImpl = System.getProperty(LOGGER_FACTORY_IMPLEMENTATION); - if (loggerFactoryImpl != null) { - loggerFactory = getLoggerFactoryIfAvailable(loggerFactoryImpl, loggerFactoryImpl); - } - - Configuration conf = ConfigurationContext.getInstance(); - // configuration in twitter4j.properties - // loggerFactory=twitter4j.internal.logging.StdOutLoggerFactory - loggerFactoryImpl = conf.getLoggerFactory(); - if (loggerFactoryImpl != null) { - loggerFactory = getLoggerFactoryIfAvailable(loggerFactoryImpl, loggerFactoryImpl); - } - // use SLF4J if it's found in the classpath - if (null == loggerFactory) { - loggerFactory = getLoggerFactoryIfAvailable("org.slf4j.impl.StaticLoggerBinder", "twitter4j.internal.logging.SLF4JLoggerFactory"); - } - // otherwise, use commons-logging if it's found in the classpath - if (null == loggerFactory) { - loggerFactory = getLoggerFactoryIfAvailable("org.apache.commons.logging.Log", "twitter4j.internal.logging.CommonsLoggingLoggerFactory"); - } - // otherwise, use log4j if it's found in the classpath - if (null == loggerFactory) { - loggerFactory = getLoggerFactoryIfAvailable("org.apache.log4j.Logger", "twitter4j.internal.logging.Log4JLoggerFactory"); - } - // on Google App Engine, use java.util.logging - if (null == loggerFactory) { - loggerFactory = getLoggerFactoryIfAvailable("com.google.appengine.api.urlfetch.URLFetchService", "twitter4j.internal.logging.JULLoggerFactory"); - } - // otherwise, use the default logger -// if (null == loggerFactory) { -// loggerFactory = new StdOutLoggerFactory(); -// } - LOGGER_FACTORY = loggerFactory; - - try { - Method method = conf.getClass().getMethod("dumpConfiguration", new Class[]{}); - method.setAccessible(true); - method.invoke(conf); - } catch (IllegalAccessException ignore) { - } catch (InvocationTargetException ignore) { - } catch (NoSuchMethodException ignore) { - } - } - - private static LoggerFactory getLoggerFactoryIfAvailable(String checkClassName, String implementationClass) { - try { - Class.forName(checkClassName); - return (LoggerFactory) Class.forName(implementationClass).newInstance(); - } catch (ClassNotFoundException ignore) { - } catch (InstantiationException e) { - throw new AssertionError(e); - } catch (SecurityException ignore) { - // Unsigned applets are not allowed to access System properties - } catch (IllegalAccessException e) { - throw new AssertionError(e); - } - return null; - } - - /** - * Returns a Logger instance associated with the specified class. - * - * @param clazz class - * @return logger instance - */ - public static Logger getLogger(Class clazz) { - return LOGGER_FACTORY.getLogger(clazz); - } - - /** - * tests if debug level logging is enabled - * - * @return if debug level logging is enabled - */ - public abstract boolean isDebugEnabled(); - - /** - * tests if info level logging is enabled - * - * @return if info level logging is enabled - */ - public abstract boolean isInfoEnabled(); - - /** - * tests if warn level logging is enabled - * - * @return if warn level logging is enabled - */ - public abstract boolean isWarnEnabled(); - - /** - * tests if error level logging is enabled - * - * @return if error level logging is enabled - */ - public abstract boolean isErrorEnabled(); - - /** - * @param message message - */ - public abstract void debug(String message); - - /** - * @param message message - * @param message2 message2 - */ - public abstract void debug(String message, String message2); - - /** - * @param message message - */ - public abstract void info(String message); - - /** - * @param message message - * @param message2 message2 - */ - public abstract void info(String message, String message2); - - /** - * @param message message - */ - public abstract void warn(String message); - - /** - * @param message message - * @param message2 message2 - */ - public abstract void warn(String message, String message2); - - /** - * @param message message - */ - public abstract void error(String message); - - /** - * @param message message - * @param th throwable - */ - public abstract void error(String message, Throwable th); - -} diff --git a/examples/src/main/java/druid/examples/twitter/LoggerFactory.java b/examples/src/main/java/druid/examples/twitter/LoggerFactory.java deleted file mode 100644 index 5dece70aedc..00000000000 --- a/examples/src/main/java/druid/examples/twitter/LoggerFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import twitter4j.internal.logging.*; - -/** - * @author Yusuke Yamamoto - yusuke at mac.com - * @since Twitter4J 2.1.1 - */ -public abstract class LoggerFactory { - - /** - * Returns a logger associated with the specified class. - * - * @param clazz class - * @return a logger instance - */ - public abstract Logger getLogger(Class clazz); -} diff --git a/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java b/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java deleted file mode 100644 index a69da55a7e1..00000000000 --- a/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java +++ /dev/null @@ -1,325 +0,0 @@ -package druid.examples.twitter; -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectStreamException; -import java.util.ArrayList; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import java.io.*; -import java.util.ArrayList; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -/** - * @author Yusuke Yamamoto - yusuke at mac.com - */ -public final class PropertyConfiguration extends WebConfigurationBase implements java.io.Serializable { - - public static final String DEBUG = "debug"; - public static final String HTTP_USER_AGENT = "http.userAgent"; - - public static final String HTTP_USE_SSL = "http.useSSL"; - public static final String HTTP_PRETTY_DEBUG = "http.prettyDebug"; - public static final String HTTP_GZIP = "http.gzip"; - public static final String HTTP_PROXY_HOST = "http.proxyHost"; - public static final String HTTP_PROXY_HOST_FALLBACK = "http.proxyHost"; - public static final String HTTP_PROXY_USER = "http.proxyUser"; - public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword"; - public static final String HTTP_PROXY_PORT = "http.proxyPort"; - public static final String HTTP_PROXY_PORT_FALLBACK = "http.proxyPort"; - public static final String HTTP_CONNECTION_TIMEOUT = "http.connectionTimeout"; - public static final String HTTP_READ_TIMEOUT = "http.readTimeout"; - - public static final String HTTP_STREAMING_READ_TIMEOUT = "http.streamingReadTimeout"; - - public static final String HTTP_RETRY_COUNT = "http.retryCount"; - public static final String HTTP_RETRY_INTERVAL_SECS = "http.retryIntervalSecs"; - - public static final String HTTP_MAX_TOTAL_CONNECTIONS = "http.maxTotalConnections"; - public static final String HTTP_DEFAULT_MAX_PER_ROUTE = "http.defaultMaxPerRoute"; - - - public static final String STREAM_BASE_URL = "streamBaseURL"; - - public static final String ASYNC_NUM_THREADS = "async.numThreads"; - public static final String CONTRIBUTING_TO = "contributingTo"; - public static final String ASYNC_DISPATCHER_IMPL = "async.dispatcherImpl"; - public static final String LOGGER_FACTORY = "loggerFactory"; - public static final String JSON_STORE_ENABLED = "jsonStoreEnabled"; - public static final String STREAM_STALL_WARNINGS_ENABLED = "stream.enableStallWarnings"; - - // hidden portion - private static final long serialVersionUID = 6458764415636588373L; - - public PropertyConfiguration(InputStream is) { - super(); - Properties props = new Properties(); - loadProperties(props, is); - setFieldsWithTreePath(props, "/"); - } - - public PropertyConfiguration(Properties props) { - this(props, "/"); - } - - public PropertyConfiguration(Properties props, String treePath) { - super(); - setFieldsWithTreePath(props, treePath); - } - - PropertyConfiguration(String treePath) { - super(); - Properties props; - // load from system properties - try { - props = (Properties) System.getProperties().clone(); - try { - Map envMap = System.getenv(); - for(String key :envMap.keySet()){ - props.setProperty(key, envMap.get(key)); - } - }catch(SecurityException ignore){} - normalize(props); - } catch (SecurityException ignore) { - // Unsigned applets are not allowed to access System properties - props = new Properties(); - } -// final String TWITTER4J_PROPERTIES = "twitter4j.properties"; -// // override System properties with ./twitter4j.properties in the classpath -// loadProperties(props, "." + File.separatorChar + TWITTER4J_PROPERTIES); -// // then, override with /twitter4j.properties in the classpath -// loadProperties(props, twitter4j.conf.Configuration.class.getResourceAsStream("/" + TWITTER4J_PROPERTIES)); -// // then, override with /WEB/INF/twitter4j.properties in the classpath -// loadProperties(props, twitter4j.conf.Configuration.class.getResourceAsStream("/WEB-INF/" + TWITTER4J_PROPERTIES)); -// // for Google App Engine -// try { -// loadProperties(props, new FileInputStream("WEB-INF/" + TWITTER4J_PROPERTIES)); -// } catch (SecurityException ignore) { -// } catch (FileNotFoundException ignore) { -// } - - setFieldsWithTreePath(props, treePath); - } - - /** - * Creates a root PropertyConfiguration. This constructor is equivalent to new PropertyConfiguration("/"). - */ - PropertyConfiguration() { - this("/"); - } - - private boolean notNull(Properties props, String prefix, String name) { - return props.getProperty(prefix + name) != null; - } - - private boolean loadProperties(Properties props, String path) { - FileInputStream fis = null; - try { - File file = new File(path); - if (file.exists() && file.isFile()) { - fis = new FileInputStream(file); - props.load(fis); - normalize(props); - return true; - } - } catch (Exception ignore) { - } finally { - try { - if (fis != null) { - fis.close(); - } - } catch (IOException ignore) { - - } - } - return false; - } - - private boolean loadProperties(Properties props, InputStream is) { - try { - props.load(is); - normalize(props); - return true; - } catch (Exception ignore) { - } - return false; - } - - private void normalize(Properties props) { - Set keys = props.keySet(); - ArrayList toBeNormalized = new ArrayList(10); - for (Object key : keys) { - String keyStr = (String) key; - if (-1 != (keyStr.indexOf("twitter4j."))) { - toBeNormalized.add(keyStr); - } - } - for (String keyStr : toBeNormalized) { - String property = props.getProperty(keyStr); - int index = keyStr.indexOf("twitter4j."); - String newKey = keyStr.substring(0, index) + keyStr.substring(index + 10); - props.setProperty(newKey, property); - } - } - - /** - * passing "/foo/bar" as treePath will result:
- * 1. load [twitter4j.]restBaseURL
- * 2. override the value with foo.[twitter4j.]restBaseURL
- * 3. override the value with foo.bar.[twitter4j.]restBaseURL
- * - * @param props properties to be loaded - * @param treePath the path - */ - private void setFieldsWithTreePath(Properties props, String treePath) { - setFieldsWithPrefix(props, ""); - String[] splitArray = z_InternalStringUtil.split(treePath, "/"); - String prefix = null; - for (String split : splitArray) { - if (!"".equals(split)) { - if (null == prefix) { - prefix = split + "."; - } else { - prefix += split + "."; - } - setFieldsWithPrefix(props, prefix); - } - } - } - - private void setFieldsWithPrefix(Properties props, String prefix) { - if (notNull(props, prefix, DEBUG)) { - setDebug(getBoolean(props, prefix, DEBUG)); - } - - if (notNull(props, prefix, HTTP_USE_SSL)) { - setUseSSL(getBoolean(props, prefix, HTTP_USE_SSL)); - } - if (notNull(props, prefix, HTTP_PRETTY_DEBUG)) { - setPrettyDebugEnabled(getBoolean(props, prefix, HTTP_PRETTY_DEBUG)); - } - if (notNull(props, prefix, HTTP_GZIP)) { - setGZIPEnabled(getBoolean(props, prefix, HTTP_GZIP)); - } - if (notNull(props, prefix, HTTP_PROXY_HOST)) { - setHttpProxyHost(getString(props, prefix, HTTP_PROXY_HOST)); - } else if (notNull(props, prefix, HTTP_PROXY_HOST_FALLBACK)) { - setHttpProxyHost(getString(props, prefix, HTTP_PROXY_HOST_FALLBACK)); - } - if (notNull(props, prefix, HTTP_PROXY_USER)) { - setHttpProxyUser(getString(props, prefix, HTTP_PROXY_USER)); - } - if (notNull(props, prefix, HTTP_PROXY_PASSWORD)) { - setHttpProxyPassword(getString(props, prefix, HTTP_PROXY_PASSWORD)); - } - if (notNull(props, prefix, HTTP_PROXY_PORT)) { - setHttpProxyPort(getIntProperty(props, prefix, HTTP_PROXY_PORT)); - } else if (notNull(props, prefix, HTTP_PROXY_PORT_FALLBACK)) { - setHttpProxyPort(getIntProperty(props, prefix, HTTP_PROXY_PORT_FALLBACK)); - } - if (notNull(props, prefix, HTTP_CONNECTION_TIMEOUT)) { - setHttpConnectionTimeout(getIntProperty(props, prefix, HTTP_CONNECTION_TIMEOUT)); - } - if (notNull(props, prefix, HTTP_READ_TIMEOUT)) { - setHttpReadTimeout(getIntProperty(props, prefix, HTTP_READ_TIMEOUT)); - } - if (notNull(props, prefix, HTTP_STREAMING_READ_TIMEOUT)) { - setHttpStreamingReadTimeout(getIntProperty(props, prefix, HTTP_STREAMING_READ_TIMEOUT)); - } - if (notNull(props, prefix, HTTP_RETRY_COUNT)) { - setHttpRetryCount(getIntProperty(props, prefix, HTTP_RETRY_COUNT)); - } - if (notNull(props, prefix, HTTP_RETRY_INTERVAL_SECS)) { - setHttpRetryIntervalSeconds(getIntProperty(props, prefix, HTTP_RETRY_INTERVAL_SECS)); - } - if (notNull(props, prefix, HTTP_MAX_TOTAL_CONNECTIONS)) { - setHttpMaxTotalConnections(getIntProperty(props, prefix, HTTP_MAX_TOTAL_CONNECTIONS)); - } - if (notNull(props, prefix, HTTP_DEFAULT_MAX_PER_ROUTE)) { - setHttpDefaultMaxPerRoute(getIntProperty(props, prefix, HTTP_DEFAULT_MAX_PER_ROUTE)); - } - if (notNull(props, prefix, ASYNC_NUM_THREADS)) { - setAsyncNumThreads(getIntProperty(props, prefix, ASYNC_NUM_THREADS)); - } - if (notNull(props, prefix, CONTRIBUTING_TO)) { - setContributingTo(getLongProperty(props, prefix, CONTRIBUTING_TO)); - } - if (notNull(props, prefix, ASYNC_DISPATCHER_IMPL)) { - setDispatcherImpl(getString(props, prefix, ASYNC_DISPATCHER_IMPL)); - } - if (notNull(props, prefix, HTTP_USER_AGENT)) { - setUserAgent(getString(props, prefix, HTTP_USER_AGENT)); - } - - - if (notNull(props, prefix, STREAM_BASE_URL)) { - setStreamBaseURL(getString(props, prefix, STREAM_BASE_URL)); - } - if (notNull(props, prefix, LOGGER_FACTORY)) { - setLoggerFactory(getString(props, prefix, LOGGER_FACTORY)); - } - if (notNull(props, prefix, JSON_STORE_ENABLED)) { - setJSONStoreEnabled(getBoolean(props, prefix, JSON_STORE_ENABLED)); - } - if (notNull(props, prefix, STREAM_STALL_WARNINGS_ENABLED)) { - setStallWarningsEnabled(getBoolean(props, prefix, STREAM_STALL_WARNINGS_ENABLED)); - } - - cacheInstance(); - } - - protected boolean getBoolean(Properties props, String prefix, String name) { - String value = props.getProperty(prefix + name); - return Boolean.valueOf(value); - } - - protected int getIntProperty(Properties props, String prefix, String name) { - String value = props.getProperty(prefix + name); - try { - return Integer.parseInt(value); - } catch (NumberFormatException nfe) { - return -1; - } - } - - protected long getLongProperty(Properties props, String prefix, String name) { - String value = props.getProperty(prefix + name); - try { - return Long.parseLong(value); - } catch (NumberFormatException nfe) { - return -1L; - } - } - - protected String getString(Properties props, String prefix, String name) { - return props.getProperty(prefix + name); - } - - // assures equality after deserialization - protected Object readResolve() throws ObjectStreamException - { - return super.readResolve(); - } -} diff --git a/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java b/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java deleted file mode 100644 index 7926874be2a..00000000000 --- a/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java +++ /dev/null @@ -1,61 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * ConfigurationFactory implementation for PropertyConfiguration. - * Currently getInstance calls concrete constructor each time. No caching at all. - * - * @author Yusuke Yamamoto - yusuke at mac.com - */ -class PropertyConfigurationFactory implements ConfigurationFactory { - private static final PropertyConfiguration ROOT_CONFIGURATION; - - static { - ROOT_CONFIGURATION = new PropertyConfiguration(); - // calling ROOT_CONFIGURATION.dumpConfiguration() will cause ExceptionInInitializerError as Logger has not been initialized. - // as a quick and dirty solution, static initializer of twitter4j.internal.logging.Logger will call dumpConfiguration() on behalf. - } - - /** - * {@inheritDoc} - */ - @Override - public Configuration getInstance() { - return ROOT_CONFIGURATION; - } - - // It may be preferable to cache the config instance - - /** - * {@inheritDoc} - */ - @Override - public Configuration getInstance(String configTreePath) { - PropertyConfiguration conf = new PropertyConfiguration(configTreePath); - conf.dumpConfiguration(); - return conf; - } - - /** - * {@inheritDoc} - */ - @Override - public void dispose() { - // nothing to do for property based configuration - } -} diff --git a/examples/src/main/java/druid/examples/twitter/RequestMethod.java b/examples/src/main/java/druid/examples/twitter/RequestMethod.java deleted file mode 100644 index 16fea4c21b0..00000000000 --- a/examples/src/main/java/druid/examples/twitter/RequestMethod.java +++ /dev/null @@ -1,26 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/** - * @author Dan Checkoway - dcheckoway at gmail.com - */ -public enum RequestMethod { - GET, POST, DELETE, HEAD, PUT -} - diff --git a/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java b/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java deleted file mode 100644 index ecbd9e9b1f0..00000000000 --- a/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java +++ /dev/null @@ -1,53 +0,0 @@ -package druid.examples.twitter; -/* - * Copyright 2007 Yusuke Yamamoto - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.InputStream; -import java.util.zip.GZIPInputStream; - -import java.io.IOException; -import java.io.InputStream; -import java.util.zip.GZIPInputStream; - -final class StreamingGZIPInputStream extends GZIPInputStream -{ - - private final InputStream wrapped; - - public StreamingGZIPInputStream(InputStream is) throws IOException - { - super(is); - wrapped = is; - } - - /** - * Overrides behavior of GZIPInputStream which assumes we have all the data available - * which is not true for streaming. We instead rely on the underlying stream to tell us - * how much data is available. - *

- * Programs should not count on this method to return the actual number - * of bytes that could be read without blocking. - * - * @return - whatever the wrapped InputStream returns - * @throws IOException if an I/O error occurs. - */ - @Override - public int available() throws IOException { - return wrapped.available(); - } -} diff --git a/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java deleted file mode 100644 index 7f880d23959..00000000000 --- a/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java +++ /dev/null @@ -1,173 +0,0 @@ -//package druid.examples.twitter; -// -//import com.fasterxml.jackson.annotation.JsonCreator; -//import com.google.common.collect.Maps; -//import com.metamx.common.logger.Logger; -//import com.metamx.druid.input.InputRow; -//import com.metamx.druid.input.MapBasedInputRow; -//import com.metamx.druid.realtime.firehose.Firehose; -//import com.metamx.druid.realtime.firehose.FirehoseFactory; -//import org.codehaus.jackson.annotate.JsonProperty; -//import org.codehaus.jackson.annotate.JsonTypeName; -//import twitter4j.ConnectionLifeCycleListener; -//import twitter4j.Status; -// -//import java.io.IOException; -//import java.util.LinkedList; -//import java.util.Map; -//import java.util.concurrent.ArrayBlockingQueue; -//import java.util.concurrent.BlockingQueue; -//import java.util.concurrent.TimeUnit; -// -//import static java.lang.Thread.sleep; -// -///** -//* Created with IntelliJ IDEA. -//* User: dhruvparthasarathy -//* Date: 6/18/13 -//* Time: 4:52 PM -//* To change this template use File | Settings | File Templates. -//*/ -//@JsonTypeName("Gov") -// -//public class USGovFirehoseFactory implements FirehoseFactory -//{ -// private static final Logger log = new Logger(TwitterSpritzerFirehoseFactory.class); -// -// private final int maxEventCount; -// -// private final int rowCount; -// -// @JsonCreator -// public USGovFirehoseFactory( -// @JsonProperty("maxEventCount") Integer maxEventCount, -// @JsonProperty("rowCount") Integer rowCount -// ) -// { -// this.maxEventCount=maxEventCount; -// this.rowCount=rowCount; -// log.info("maxEventCount=" + ((maxEventCount <= 0) ? "no limit" : maxEventCount)); -// log.info("rowCount=" + ((rowCount <= 0) ? "no limit" : rowCount)); -// } -// -// @Override -// public Firehose connect() throws IOException -// { -// final LinkedList dimensions = new LinkedList(); -// final int QUEUE_SIZE = 2000; -// final BlockingQueue queue = new ArrayBlockingQueue(QUEUE_SIZE); -// dimensions.add("device"); -// dimensions.add("country_code"); -// dimensions.add("known_user"); -// dimensions.add("base_url"); -// dimensions.add("referring_url"); -// dimensions.add("full_url"); -// dimensions.add("timestamp"); -// dimensions.add("city"); -// dimensions.add("tz"); -// -// WebListener listener = new WebListener() { -// @Override -// public void onUpdate(Update update) -// { -// if (Thread.currentThread().isInterrupted()) { -// throw new RuntimeException("Interrupted, time to stop"); -// } -// try { -// boolean success = queue.offer(update, 15L, TimeUnit.SECONDS); -// if (!success){ -// log.warn("queue too slow!"); -// } -// } -// catch (InterruptedException e){ -// throw new RuntimeException("InterruptedException", e); -// } -// } -// }; -// return new Firehose() -// { -// final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { -// @Override -// public void onConnect() -// { -// log.info("Connected_to_stream"); -// } -// -// @Override -// public void onDisconnect() -// { -// log.info("Disconnected_from_stream"); -// } -// -// @Override -// public void onCleanUp() -// { -// log.info("Cleanup_stream"); -// } -// }; -// private final Runnable doNothingRunnable = new Runnable() { -// public void run() -// { -// } -// }; -// private boolean waitIfmax = true; -// @Override -// public boolean hasMore() -// { -// if (maxEventCount >=0 && rowCount >= maxEventCount){ -// return waitIfmax; -// } -// else -// { -// return true; -// } -// } -// -// @Override -// public InputRow nextRow() -// { -// if (maxEventCount >=0 && rowCount >=maxEventCount && waitIfmax){ -// try { -// sleep(2000000000L); -// } -// catch (InterruptedException e) { -// throw new RuntimeException("InterruptedException"); -// } -// } -// Update udpate; -// try{ -// update=queue.take(); -// } -// catch (InterruptedException e) { -// throw new RuntimeException("InterruptedException", e); -// } -// final Map theMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); -// theMap.put("device", update.device()); -// theMap.put("country_code", update.country()); -// theMap.put("known_user", update.knownUser()); -// theMap.put("base_url", update.baseUrl()); -// theMap.put("referring_url", update.referringUrl()); -// theMap.put("full_url", update.fullUrl()); -// theMap.put("timestamp", update.timestamp()); -// theMap.put("city", update.city()); -// theMap.put("tz", update.tz()); -// -// return new MapBasedInputRow(update.timestamp,dimensions,theMap); //To change body of implemented methods use File | Settings | File Templates. -// } -// -// @Override -// public Runnable commit() -// { -// return doNothingRunnable; //To change body of implemented methods use File | Settings | File Templates. -// } -// -// @Override -// public void close() throws IOException -// { -// //To change body of implemented methods use File | Settings | File Templates. -// } -// -// }; -// -// } -//} diff --git a/examples/src/main/java/druid/examples/twitter/WebBase.java b/examples/src/main/java/druid/examples/twitter/WebBase.java deleted file mode 100644 index 4e223b8ebd9..00000000000 --- a/examples/src/main/java/druid/examples/twitter/WebBase.java +++ /dev/null @@ -1,15 +0,0 @@ -package druid.examples.twitter; - -/** - * Created with IntelliJ IDEA. - * User: dhruvparthasarathy - * Date: 6/19/13 - * Time: 4:15 PM - * To change this template use File | Settings | File Templates. - */ -public interface WebBase -{ - Configuration getConfiguration(); - - void shutdown(); -} diff --git a/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java b/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java deleted file mode 100644 index 30310e89739..00000000000 --- a/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java +++ /dev/null @@ -1,170 +0,0 @@ -//package druid.examples.twitter; -// -///* -// * Copyright 2007 Yusuke Yamamoto -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// -//import java.io.IOException; -//import java.io.ObjectInputStream; -//import java.util.ArrayList; -//import java.util.List; -// -// -//import java.io.IOException; -//import java.io.ObjectInputStream; -//import java.util.ArrayList; -//import java.util.List; -// -///** -// * Base class of Twitter / AsyncTwitter / TwitterStream supports OAuth. -// * -// * @author Yusuke Yamamoto - yusuke at mac.com -// */ -//abstract class WebBaseImpl implements WebBase, java.io.Serializable, HttpResponseListener -//{ -// protected Configuration conf; -// protected transient String screenName = null; -// protected transient long id = 0; -// -// protected transient HttpClientWrapper http; -// -// protected z_T4JInternalFactory factory; -// -// private static final long serialVersionUID = -3812176145960812140L; -// -// /*package*/ WebBaseImpl(Configuration conf) { -// this.conf = conf; -// init(); -// } -// -// private void init() { -// -// http = new HttpClientWrapper(conf); -// http.setHttpResponseListener(this); -// setFactory(); -// } -// -// protected void setFactory() { -// factory = new z_T4JInternalJSONImplFactory(conf); -// } -// -// /** -// * {@inheritDoc} -// */ -// -// @Override -// public void httpResponseReceived(HttpResponseEvent event) { -// if (rateLimitStatusListeners.size() != 0) { -// HttpResponse res = event.getResponse(); -// TwitterException te = event.getTwitterException(); -// RateLimitStatus rateLimitStatus; -// int statusCode; -// if (te != null) { -// rateLimitStatus = te.getRateLimitStatus(); -// statusCode = te.getStatusCode(); -// } else { -// rateLimitStatus = z_T4JInternalJSONImplFactory.createRateLimitStatusFromResponseHeader(res); -// statusCode = res.getStatusCode(); -// } -// if (rateLimitStatus != null) { -// RateLimitStatusEvent statusEvent -// = new RateLimitStatusEvent(this, rateLimitStatus, event.isAuthenticated()); -// if (statusCode == ENHANCE_YOUR_CLAIM -// || statusCode == SERVICE_UNAVAILABLE) { -// // EXCEEDED_RATE_LIMIT_QUOTA is returned by Rest API -// // SERVICE_UNAVAILABLE is returned by Search API -// for (RateLimitStatusListener listener : rateLimitStatusListeners) { -// listener.onRateLimitStatus(statusEvent); -// listener.onRateLimitReached(statusEvent); -// } -// } else { -// for (RateLimitStatusListener listener : rateLimitStatusListeners) { -// listener.onRateLimitStatus(statusEvent); -// } -// } -// } -// } -// } -// -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public Configuration getConfiguration() { -// return this.conf; -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void shutdown() { -// if (http != null) http.shutdown(); -// } -// -// -// private void writeObject(java.io.ObjectOutputStream out) throws IOException -// { -// // http://docs.oracle.com/javase/6/docs/platform/serialization/spec/output.html#861 -// out.putFields(); -// out.writeFields(); -// -// out.writeObject(conf); -// } -// -// private void readObject(ObjectInputStream stream) -// throws IOException, ClassNotFoundException { -// // http://docs.oracle.com/javase/6/docs/platform/serialization/spec/input.html#2971 -// stream.readFields(); -// -// conf = (Configuration) stream.readObject(); -// http = new HttpClientWrapper(conf); -// http.setHttpResponseListener(this); -// setFactory(); -// } -// -// -// -// @Override -// public boolean equals(Object o) { -// if (this == o) return true; -// if (!(o instanceof WebBaseImpl)) return false; -// -// WebBaseImpl that = (WebBaseImpl) o; -// -// if (!conf.equals(that.conf)) return false; -// if (http != null ? !http.equals(that.http) : that.http != null) -// return false; -// -// return true; -// } -// -// @Override -// public int hashCode() { -// int result = conf.hashCode(); -// result = 31 * result + (http != null ? http.hashCode() : 0); -// return result; -// } -// -// @Override -// public String toString() { -// return "TwitterBase{" + -// "conf=" + conf + -// ", http=" + http + -// '}'; -// } -//} diff --git a/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java b/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java deleted file mode 100644 index 64854151f31..00000000000 --- a/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java +++ /dev/null @@ -1,494 +0,0 @@ -package druid.examples.twitter; -/** - * Created with IntelliJ IDEA. - * User: dhruvparthasarathy - * Date: 6/19/13 - * Time: 2:02 PM - * To change this template use File | Settings | File Templates. - */ - -import druid.examples.twitter.Configuration; -import twitter4j.internal.util.z_T4JInternalStringUtil; - -import java.io.ObjectStreamException; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - - -import java.io.ObjectStreamException; -import java.lang.reflect.Field; -import java.util.*; -import java.util.logging.Logger; - -/** - * Configuration base class with default settings. - * - * @author Yusuke Yamamoto - yusuke at mac.com - */ -class WebConfigurationBase implements Configuration, java.io.Serializable { - private boolean debug; - private boolean useSSL; - private boolean prettyDebug; - private boolean gzipEnabled; - private String httpProxyHost; - private String httpProxyUser; - private String httpProxyPassword; - private String userAgent; - private int httpProxyPort; - private int httpConnectionTimeout; - private int httpReadTimeout; - - private int httpStreamingReadTimeout; - private int httpRetryCount; - private int httpRetryIntervalSeconds; - private int maxTotalConnections; - private int defaultMaxPerRoute; - - private String streamBaseURL; - - private String dispatcherImpl; - private String loggerFactory; - - private int asyncNumThreads; - - private long contributingTo; - private boolean includeEntitiesEnabled = true; - - private boolean jsonStoreEnabled; - - - private boolean stallWarningsEnabled; - - - - private static final String DEFAULT_STREAM_BASE_URL = "http://developer.usa.gov/1usagov/"; - - private static final long serialVersionUID = -6610497517837844232L; - - - protected WebConfigurationBase() { - setDebug(false); - setUseSSL(false); - setPrettyDebugEnabled(false); - setGZIPEnabled(true); - setHttpProxyHost(null); - setHttpProxyUser(null); - setHttpProxyPassword(null); - setUserAgent(null); - setHttpProxyPort(-1); - setHttpConnectionTimeout(20000); - setHttpReadTimeout(120000); - setHttpStreamingReadTimeout(40 * 1000); - setHttpRetryCount(0); - setHttpRetryIntervalSeconds(5); - setHttpMaxTotalConnections(20); - setHttpDefaultMaxPerRoute(2); - setAsyncNumThreads(1); - setContributingTo(-1L); - - setJSONStoreEnabled(false); - - - setStreamBaseURL(DEFAULT_STREAM_BASE_URL); - - setDispatcherImpl("twitter4j.internal.async.DispatcherImpl"); - setLoggerFactory(null); - - setStallWarningsEnabled(true); - - } - - public void dumpConfiguration() { - if (debug) { -// Field[] fields = WebConfigurationBase.class.getDeclaredFields(); -// for (Field field : fields) { -// try { -// Object value = field.get(this); -// String strValue = String.valueOf(value); -// if (value != null && field.getName().matches("oAuthConsumerSecret|oAuthAccessTokenSecret|password")) { -// strValue = z_T4JInternalStringUtil.maskString(String.valueOf(value)); -// } -// log.debug(field.getName() + ": " + strValue); -// } catch (IllegalAccessException ignore) { -// } -// } - } - } - - @Override - public final boolean isDebugEnabled() { - return debug; - } - - protected final void setDebug(boolean debug) { - this.debug = debug; - } - - - @Override - public boolean isPrettyDebugEnabled() { - return prettyDebug; - } - - protected final void setUseSSL(boolean useSSL) { - this.useSSL = useSSL; - } - - protected final void setPrettyDebugEnabled(boolean prettyDebug) { - this.prettyDebug = prettyDebug; - } - - protected final void setGZIPEnabled(boolean gzipEnabled) { - this.gzipEnabled = gzipEnabled; - initRequestHeaders(); - } - - @Override - public boolean isGZIPEnabled() { - return gzipEnabled; - } - - // method for HttpRequestFactoryConfiguration - Map requestHeaders; - - private void initRequestHeaders() { -// requestHeaders = new HashMap(); -// requestHeaders.put("X-Twitter-Client-Version", getClientVersion()); -// requestHeaders.put("X-Twitter-Client-URL", getClientURL()); -// requestHeaders.put("X-Twitter-Client", "Twitter4J"); -// -// requestHeaders.put("User-Agent", getUserAgent()); -// if (gzipEnabled) { -// requestHeaders.put("Accept-Encoding", "gzip"); -// } -// if (IS_DALVIK) { -// requestHeaders.put("Connection", "close"); -// } - } - - @Override - public Map getRequestHeaders() { - return requestHeaders; - } - - // methods for HttpClientConfiguration - - @Override - public final String getHttpProxyHost() { - return httpProxyHost; - } - - protected final void setHttpProxyHost(String proxyHost) { - this.httpProxyHost = proxyHost; - } - - @Override - public final String getHttpProxyUser() { - return httpProxyUser; - } - - protected final void setHttpProxyUser(String proxyUser) { - this.httpProxyUser = proxyUser; - } - - @Override - public final String getHttpProxyPassword() { - return httpProxyPassword; - } - - protected final void setHttpProxyPassword(String proxyPassword) { - this.httpProxyPassword = proxyPassword; - } - - @Override - public final int getHttpProxyPort() { - return httpProxyPort; - } - - protected final void setHttpProxyPort(int proxyPort) { - this.httpProxyPort = proxyPort; - } - - @Override - public final int getHttpConnectionTimeout() { - return httpConnectionTimeout; - } - - protected final void setHttpConnectionTimeout(int connectionTimeout) { - this.httpConnectionTimeout = connectionTimeout; - } - - @Override - public final int getHttpReadTimeout() { - return httpReadTimeout; - } - - protected final void setHttpReadTimeout(int readTimeout) { - this.httpReadTimeout = readTimeout; - } - - @Override - public int getHttpStreamingReadTimeout() { - return httpStreamingReadTimeout; - } - - protected final void setHttpStreamingReadTimeout(int httpStreamingReadTimeout) { - this.httpStreamingReadTimeout = httpStreamingReadTimeout; - } - - - @Override - public final int getHttpRetryCount() { - return httpRetryCount; - } - - protected final void setHttpRetryCount(int retryCount) { - this.httpRetryCount = retryCount; - } - - @Override - public final int getHttpRetryIntervalSeconds() { - return httpRetryIntervalSeconds; - } - - protected final void setHttpRetryIntervalSeconds(int retryIntervalSeconds) { - this.httpRetryIntervalSeconds = retryIntervalSeconds; - } - - @Override - public final int getHttpMaxTotalConnections() { - return maxTotalConnections; - } - - protected final void setHttpMaxTotalConnections(int maxTotalConnections) { - this.maxTotalConnections = maxTotalConnections; - } - - @Override - public final int getHttpDefaultMaxPerRoute() { - return defaultMaxPerRoute; - } - - protected final void setHttpDefaultMaxPerRoute(int defaultMaxPerRoute) { - this.defaultMaxPerRoute = defaultMaxPerRoute; - } - - - @Override - public final int getAsyncNumThreads() { - return asyncNumThreads; - } - - protected final void setAsyncNumThreads(int asyncNumThreads) { - this.asyncNumThreads = asyncNumThreads; - } - - @Override - public final long getContributingTo() { - return contributingTo; - } - - protected final void setContributingTo(long contributingTo) { - this.contributingTo = contributingTo; - } - - - @Override - public String getStreamBaseURL() { - return streamBaseURL; - } - - protected final void setStreamBaseURL(String streamBaseURL) { - this.streamBaseURL = streamBaseURL; - } - - - @Override - public String getDispatcherImpl() { - return dispatcherImpl; - } - - protected final void setDispatcherImpl(String dispatcherImpl) { - this.dispatcherImpl = dispatcherImpl; - } - - protected final void setUserAgent(String userAgent) { - this.userAgent= userAgent; - } - - @Override - public String getLoggerFactory() { - return loggerFactory; - } - - - protected final void setLoggerFactory(String loggerImpl) { - this.loggerFactory = loggerImpl; - } - - - public boolean isJSONStoreEnabled() { - return this.jsonStoreEnabled; - } - - protected final void setJSONStoreEnabled(boolean enabled) { - this.jsonStoreEnabled = enabled; - } - - @Override - public boolean isStallWarningsEnabled() { - return stallWarningsEnabled; - } - - protected final void setStallWarningsEnabled(boolean stallWarningsEnabled) { - this.stallWarningsEnabled = stallWarningsEnabled; - } - - static String fixURL(boolean useSSL, String url) { - if (null == url) { - return null; - } - int index = url.indexOf("://"); - if (-1 == index) { - throw new IllegalArgumentException("url should contain '://'"); - } - String hostAndLater = url.substring(index + 3); - if (useSSL) { - return "https://" + hostAndLater; - } else { - return "http://" + hostAndLater; - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - WebConfigurationBase that = (WebConfigurationBase) o; - - if (asyncNumThreads != that.asyncNumThreads) return false; - if (contributingTo != that.contributingTo) return false; - if (debug != that.debug) return false; - if (defaultMaxPerRoute != that.defaultMaxPerRoute) return false; - if (gzipEnabled != that.gzipEnabled) return false; - if (httpConnectionTimeout != that.httpConnectionTimeout) return false; - if (httpProxyPort != that.httpProxyPort) return false; - if (httpReadTimeout != that.httpReadTimeout) return false; - if (httpRetryCount != that.httpRetryCount) return false; - if (httpRetryIntervalSeconds != that.httpRetryIntervalSeconds) return false; - if (httpStreamingReadTimeout != that.httpStreamingReadTimeout) return false; - if (includeEntitiesEnabled != that.includeEntitiesEnabled) return false; - if (jsonStoreEnabled != that.jsonStoreEnabled) return false; - if (maxTotalConnections != that.maxTotalConnections) return false; - if (prettyDebug != that.prettyDebug) return false; - if (stallWarningsEnabled != that.stallWarningsEnabled) return false; - if (useSSL != that.useSSL) return false; - if (dispatcherImpl != null ? !dispatcherImpl.equals(that.dispatcherImpl) : that.dispatcherImpl != null) - return false; - if (httpProxyHost != null ? !httpProxyHost.equals(that.httpProxyHost) : that.httpProxyHost != null) - return false; - if (httpProxyPassword != null ? !httpProxyPassword.equals(that.httpProxyPassword) : that.httpProxyPassword != null) - return false; - if (httpProxyUser != null ? !httpProxyUser.equals(that.httpProxyUser) : that.httpProxyUser != null) - return false; - if (loggerFactory != null ? !loggerFactory.equals(that.loggerFactory) : that.loggerFactory != null) - return false; - if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) - return false; - if (streamBaseURL != null ? !streamBaseURL.equals(that.streamBaseURL) : that.streamBaseURL != null) - return false; - return true; - } - - @Override - public int hashCode() { - int result = (debug ? 1 : 0); - result = 31 * result + (useSSL ? 1 : 0); - result = 31 * result + (prettyDebug ? 1 : 0); - result = 31 * result + (gzipEnabled ? 1 : 0); - result = 31 * result + (httpProxyHost != null ? httpProxyHost.hashCode() : 0); - result = 31 * result + (httpProxyUser != null ? httpProxyUser.hashCode() : 0); - result = 31 * result + (httpProxyPassword != null ? httpProxyPassword.hashCode() : 0); - result = 31 * result + httpProxyPort; - result = 31 * result + httpConnectionTimeout; - result = 31 * result + httpReadTimeout; - result = 31 * result + httpStreamingReadTimeout; - result = 31 * result + httpRetryCount; - result = 31 * result + httpRetryIntervalSeconds; - result = 31 * result + maxTotalConnections; - result = 31 * result + defaultMaxPerRoute; - result = 31 * result + (streamBaseURL != null ? streamBaseURL.hashCode() : 0); - result = 31 * result + (dispatcherImpl != null ? dispatcherImpl.hashCode() : 0); - result = 31 * result + (loggerFactory != null ? loggerFactory.hashCode() : 0); - result = 31 * result + asyncNumThreads; - result = 31 * result + (int) (contributingTo ^ (contributingTo >>> 32)); - result = 31 * result + (includeEntitiesEnabled ? 1 : 0); - result = 31 * result + (jsonStoreEnabled ? 1 : 0); - result = 31 * result + (stallWarningsEnabled ? 1 : 0); - result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "ConfigurationBase{" + - "debug=" + debug + - ", useSSL=" + useSSL + - ", prettyDebug=" + prettyDebug + - ", gzipEnabled=" + gzipEnabled + - ", httpProxyHost='" + httpProxyHost + '\'' + - ", httpProxyUser='" + httpProxyUser + '\'' + - ", httpProxyPassword='" + httpProxyPassword + '\'' + - ", httpProxyPort=" + httpProxyPort + - ", httpConnectionTimeout=" + httpConnectionTimeout + - ", httpReadTimeout=" + httpReadTimeout + - ", httpStreamingReadTimeout=" + httpStreamingReadTimeout + - ", httpRetryCount=" + httpRetryCount + - ", httpRetryIntervalSeconds=" + httpRetryIntervalSeconds + - ", maxTotalConnections=" + maxTotalConnections + - ", defaultMaxPerRoute=" + defaultMaxPerRoute + - ", streamBaseURL='" + streamBaseURL + '\'' + - ", dispatcherImpl='" + dispatcherImpl + '\'' + - ", loggerFactory='" + loggerFactory + '\'' + - ", asyncNumThreads=" + asyncNumThreads + - ", contributingTo=" + contributingTo + - ", includeEntitiesEnabled=" + includeEntitiesEnabled + - ", jsonStoreEnabled=" + jsonStoreEnabled + - ", stallWarningsEnabled=" + stallWarningsEnabled + - ", requestHeaders=" + requestHeaders + - '}'; - } - - private static final List instances = new ArrayList(); - - private static void cacheInstance(WebConfigurationBase conf) { - if (!instances.contains(conf)) { - instances.add(conf); - } - } - - protected void cacheInstance() { - cacheInstance(this); - } - - private static WebConfigurationBase getInstance(WebConfigurationBase configurationBase) { - int index; - if ((index = instances.indexOf(configurationBase)) == -1) { - instances.add(configurationBase); - return configurationBase; - } else { - return instances.get(index); - } - } - - // assures equality after deserializedation - protected Object readResolve() throws ObjectStreamException - { - return getInstance(this); - } -} diff --git a/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java b/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java deleted file mode 100644 index 747fd9bb7b7..00000000000 --- a/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java +++ /dev/null @@ -1,176 +0,0 @@ -//package druid.examples.twitter; -// -///* -// * Copyright 2007 Yusuke Yamamoto -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// -// -//import org.mortbay.jetty.servlet.Dispatcher; -// -//import java.io.BufferedReader; -//import java.io.IOException; -//import java.io.InputStream; -//import java.io.InputStreamReader; -// -///** -// * @author Yusuke Yamamoto - yusuke at mac.com -// * @since Twitter4J 2.1.8 -// */ -//abstract class WebStatusStreamBase implements WebStatusStream -//{ -// protected static final twitter4j.internal.logging.Logger logger = twitter4j.internal -// .logging -// .Logger -// .getLogger(WebStatusStreamImpl.class); -// -// private boolean streamAlive = true; -// private BufferedReader br; -// private InputStream is; -// private HttpResponse response; -// protected final Dispatcher dispatcher; -// protected final Configuration CONF; -// protected z_InternalFactory factory; -// -// /*package*/ -// -// StatusStreamBase(Dispatcher dispatcher, InputStream stream, twitter4j.conf.Configuration conf) throws IOException -// { -// this.is = stream; -// this.br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); -// this.dispatcher = dispatcher; -// this.CONF = conf; -// this.factory = new z_T4JInternalJSONImplFactory(conf); -// } -// /*package*/ -// -// StatusStreamBase(Dispatcher dispatcher, HttpResponse response, twitter4j.conf.Configuration conf) throws IOException { -// this(dispatcher, response.asStream(), conf); -// this.response = response; -// } -// -// protected String parseLine(String line) { -// return line; -// } -// -// abstract class StreamEvent implements Runnable { -// String line; -// -// StreamEvent(String line) { -// this.line = line; -// } -// } -// -// protected void handleNextElement(final StreamListener[] listeners, -// final RawStreamListener[] rawStreamListeners) throws TwitterException -// { -// if (!streamAlive) { -// throw new IllegalStateException("Stream already closed."); -// } -// try { -// String line = br.readLine(); -// if (null == line) { -// //invalidate this status stream -// throw new IOException("the end of the stream has been reached"); -// } -// dispatcher.invokeLater(new StreamEvent(line) { -// public void run() { -// try { -// if (rawStreamListeners.length > 0) { -// onMessage(line, rawStreamListeners); -// } -// // SiteStreamsImpl will parse "forUser" attribute -// line = parseLine(line); -// if (line != null && line.length() > 0) { -// // parsing JSON is an expensive process and can be avoided when all listeners are instanceof RawStreamListener -// if (listeners.length > 0) { -// if (CONF.isJSONStoreEnabled()) { -// DataObjectFactoryUtil.clearThreadLocalMap(); -// } -// twitter4j.internal.org.json.JSONObject json = new twitter4j.internal.org.json.JSONObject(line); -// JSONObjectType.Type event = JSONObjectType.determine(json); -// if (logger.isDebugEnabled()) { -// logger.debug("Received:", CONF.isPrettyDebugEnabled() ? json.toString(1) : json.toString()); -// } -// switch (event) { -// case STATUS: -// onStatus(json, listeners); -// break; -// default: -// logger.warn("Received unknown event:", CONF.isPrettyDebugEnabled() ? json.toString(1) : json.toString()); -// } -// } -// } -// } catch (Exception ex) { -// onException(ex, listeners); -// } -// } -// }); -// -// } catch (IOException ioe) { -// try { -// is.close(); -// } catch (IOException ignore) { -// } -// boolean isUnexpectedException = streamAlive; -// streamAlive = false; -// if (isUnexpectedException) { -// throw new RuntimeException("Stream closed.", ioe); -// } -// } -// } -// -// -// -// protected void onStatus(twitter4j.internal.org.json.JSONObject json, StreamListener[] listeners) throws TwitterException { -// logger.warn("Unhandled event: onStatus"); -// } -// -// -// protected void onException(Exception e, StreamListener[] listeners) { -// logger.warn("Unhandled event: ", e.getMessage()); -// } -// -// public void close() throws IOException { -// streamAlive = false; -// is.close(); -// br.close(); -// if (response != null) { -// response.disconnect(); -// } -// } -// -// protected Status asStatus(twitter4j.internal.org.json.JSONObject json) throws TwitterException { -// Status status = factory.createStatus(json); -// if (CONF.isJSONStoreEnabled()) { -// DataObjectFactoryUtil.registerJSONObject(status, json); -// } -// return status; -// } -// -// -// public abstract void next(StatusListener listener) throws TwitterException; -// -// public abstract void next(StreamListener[] listeners, RawStreamListener[] rawStreamListeners) throws TwitterException; -// -// public void onException(Exception e, StreamListener[] listeners, RawStreamListener[] rawStreamListeners) { -// for (StreamListener listener : listeners) { -// listener.onException(e); -// } -// for (RawStreamListener listener : rawStreamListeners) { -// listener.onException(e); -// } -// } -//} diff --git a/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java b/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java deleted file mode 100644 index 53ef525f33b..00000000000 --- a/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java +++ /dev/null @@ -1,111 +0,0 @@ -//package druid.examples.twitter; -// -///* -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// -///** -// * An instance of this class is completely thread safe and can be re-used and used concurrently.
-// * Note that TwitterStream is NOT compatible with Google App Engine as GAE is not capable of handling requests longer than 30 seconds. -// * -// */ -//public final class WebStreamFactory implements java.io.Serializable { -// private static final long serialVersionUID = 8146074704915782233L; -// private final Configuration conf; -// private static final WebStream SINGLETON; -// -// static { -// SINGLETON = new TwitterStreamImpl(ConfigurationContext.getInstance(), TwitterFactory.DEFAULT_AUTHORIZATION); -// } -// -// /** -// * Creates a TwitterStreamFactory with the root configuration. -// */ -// public TwitterStreamFactory() { -// this(ConfigurationContext.getInstance()); -// } -// -// /** -// * Creates a TwitterStreamFactory with the given configuration. -// * -// * @param conf the configuration to use -// * @since Twitter4J 2.1.1 -// */ -// public WebStreamFactory(Configuration conf) { -// this.conf = conf; -// } -// -// /** -// * Creates a TwitterStreamFactory with a specified config tree. -// * -// * @param configTreePath the path -// */ -// public WebStreamFactory(String configTreePath) { -// this(ConfigurationContext.getInstance(configTreePath)); -// } -// -// // implementations for BasicSupportFactory -// -// /** -// * Returns a instance associated with the configuration bound to this factory. -// * -// * @return default instance -// */ -// public WebStream getInstance() { -// return getInstance(AuthorizationFactory.getInstance(conf)); -// } -// -// /** -// * Returns a OAuth Authenticated instance.
-// * consumer key and consumer Secret must be provided by twitter4j.properties, or system properties. -// * Unlike {@link TwitterStream#setOAuthAccessToken(twitter4j.auth.AccessToken)}, this factory method potentially returns a cached instance. -// * -// * @param accessToken access token -// * @return an instance -// */ -// public TwitterStream getInstance(AccessToken accessToken) { -// String consumerKey = conf.getOAuthConsumerKey(); -// String consumerSecret = conf.getOAuthConsumerSecret(); -// if (null == consumerKey && null == consumerSecret) { -// throw new IllegalStateException("Consumer key and Consumer secret not supplied."); -// } -// OAuthAuthorization oauth = new OAuthAuthorization(conf); -// oauth.setOAuthAccessToken(accessToken); -// return getInstance(conf, oauth); -// } -// -// /** -// * Returns a instance. -// * -// * @return an instance -// */ -// public TwitterStream getInstance(Authorization auth) { -// return getInstance(conf, auth); -// } -// -// private TwitterStream getInstance(twitter4j.conf.Configuration conf, Authorization auth) { -// return new TwitterStreamImpl(conf, auth); -// } -// -// /** -// * Returns default singleton TwitterStream instance. -// * -// * @return default singleton TwitterStream instance -// * @since Twitter4J 2.2.4 -// */ -// public static TwitterStream getSingleton() { -// return SINGLETON; -// } -//} diff --git a/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java b/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java deleted file mode 100644 index 330ff6b5f95..00000000000 --- a/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java +++ /dev/null @@ -1,750 +0,0 @@ -//package druid.examples.twitter; -// -///* -// * Copyright 2007 Yusuke Yamamoto -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//import java.io.IOException; -//import java.io.InputStream; -//import java.util.ArrayList; -//import java.util.HashMap; -//import java.util.List; -//import java.util.Map; -// -// -// -//import java.io.IOException; -//import java.io.InputStream; -//import java.util.ArrayList; -//import java.util.HashMap; -//import java.util.List; -//import java.util.Map; -// -///** -// * A java representation of the Streaming API: Methods
-// * Note that this class is NOT compatible with Google App Engine as GAE is not capable of handling requests longer than 30 seconds. -// * -// * @author Yusuke Yamamoto - yusuke at mac.com -// * @since Twitter4J 2.0.4 -// */ -//class WebStreamImpl extends TwitterBaseImpl implements WebStream -//{ -// private static final long serialVersionUID = 5529611191443189901L; -// private final HttpClientWrapper http; -// private static final Logger logger = Logger.getLogger(TwitterStreamImpl.class); -// -// private List lifeCycleListeners = new ArrayList(0); -// private TwitterStreamConsumer handler = null; -// -// private String stallWarningsGetParam; -// private HttpParameter stallWarningsParam; -// -// /*package*/ -// TwitterStreamImpl(twitter4j.conf.Configuration conf, Authorization auth) { -// super(conf, auth); -// http = new HttpClientWrapper(new StreamingReadTimeoutConfiguration(conf)); -// stallWarningsGetParam = "stall_warnings=" + (conf.isStallWarningsEnabled() ? "true" : "false"); -// stallWarningsParam = new HttpParameter("stall_warnings", conf.isStallWarningsEnabled()); -// } -// -// /* Streaming API */ -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void firehose(final int count) { -// ensureAuthorizationEnabled(); -// ensureStatusStreamListenerIsSet(); -// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException -// { -// return getFirehoseStream(count); -// } -// }); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public StatusStream getFirehoseStream(int count) throws TwitterException { -// ensureAuthorizationEnabled(); -// return getCountStream("statuses/firehose.json", count); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void links(final int count) { -// ensureAuthorizationEnabled(); -// ensureStatusStreamListenerIsSet(); -// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException { -// return getLinksStream(count); -// } -// }); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public StatusStream getLinksStream(int count) throws TwitterException { -// ensureAuthorizationEnabled(); -// return getCountStream("statuses/links.json", count); -// } -// -// private StatusStream getCountStream(String relativeUrl, int count) throws TwitterException { -// ensureAuthorizationEnabled(); -// try { -// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() + relativeUrl -// , new HttpParameter[]{new HttpParameter("count", String.valueOf(count)) -// , stallWarningsParam}, auth), conf); -// } catch (IOException e) { -// throw new TwitterException(e); -// } -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void retweet() { -// ensureAuthorizationEnabled(); -// ensureStatusStreamListenerIsSet(); -// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException { -// return getRetweetStream(); -// } -// }); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public StatusStream getRetweetStream() throws TwitterException { -// ensureAuthorizationEnabled(); -// try { -// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() + "statuses/retweet.json" -// , new HttpParameter[]{stallWarningsParam}, auth), conf); -// } catch (IOException e) { -// throw new TwitterException(e); -// } -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void sample() { -// ensureAuthorizationEnabled(); -// ensureStatusStreamListenerIsSet(); -// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException { -// return getSampleStream(); -// } -// }); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public StatusStream getSampleStream() throws TwitterException { -// ensureAuthorizationEnabled(); -// try { -// return new StatusStreamImpl(getDispatcher(), http.get(conf.getStreamBaseURL() + "statuses/sample.json?" -// + stallWarningsGetParam, auth), conf); -// } catch (IOException e) { -// throw new TwitterException(e); -// } -// } -// -// /** -// * {@inheritDoc} -// */ -// public void user() { -// user(null); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void user(final String[] track) { -// ensureAuthorizationEnabled(); -// ensureUserStreamListenerIsSet(); -// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException { -// return getUserStream(track); -// } -// }); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public UserStream getUserStream() throws TwitterException { -// return getUserStream(null); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public UserStream getUserStream(String[] track) throws TwitterException { -// ensureAuthorizationEnabled(); -// try { -// List params = new ArrayList(); -// params.add(stallWarningsParam); -// if (conf.isUserStreamRepliesAllEnabled()) { -// params.add(new HttpParameter("replies", "all")); -// } -// if (track != null) { -// params.add(new HttpParameter("track", z_T4JInternalStringUtil.join(track))); -// } -// return new UserStreamImpl(getDispatcher(), http.post(conf.getUserStreamBaseURL() + "user.json" -// , params.toArray(new HttpParameter[params.size()]) -// , auth), conf); -// } catch (IOException e) { -// throw new TwitterException(e); -// } -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public StreamController site(final boolean withFollowings, final long[] follow) { -// ensureOAuthEnabled(); -// ensureSiteStreamsListenerIsSet(); -// final StreamController cs = new StreamController(http, auth); -// startHandler(new TwitterStreamConsumer(siteStreamsListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException { -// try { -// return new SiteStreamsImpl(getDispatcher(), getSiteStream(withFollowings, follow), conf, cs); -// } catch (IOException e) { -// throw new TwitterException(e); -// } -// } -// }); -// return cs; -// } -// -// private Dispatcher getDispatcher() { -// if (null == TwitterStreamImpl.dispatcher) { -// synchronized (TwitterStreamImpl.class) { -// if (null == TwitterStreamImpl.dispatcher) { -// // dispatcher is held statically, but it'll be instantiated with -// // the configuration instance associated with this TwitterStream -// // instance which invokes getDispatcher() on the first time. -// TwitterStreamImpl.dispatcher = new DispatcherFactory(conf).getInstance(); -// } -// } -// } -// return TwitterStreamImpl.dispatcher; -// } -// -// private static transient Dispatcher dispatcher; -// -// InputStream getSiteStream(boolean withFollowings, long[] follow) throws TwitterException { -// ensureOAuthEnabled(); -// return http.post(conf.getSiteStreamBaseURL() + "site.json", -// new HttpParameter[]{ -// new HttpParameter("with", withFollowings ? "followings" : "user") -// , new HttpParameter("follow", z_T4JInternalStringUtil.join(follow)) -// , stallWarningsParam}, auth).asStream(); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void filter(final FilterQuery query) { -// ensureAuthorizationEnabled(); -// ensureStatusStreamListenerIsSet(); -// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { -// @Override -// public StatusStream getStream() throws TwitterException { -// return getFilterStream(query); -// } -// }); -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public StatusStream getFilterStream(FilterQuery query) throws TwitterException { -// ensureAuthorizationEnabled(); -// try { -// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() -// + "statuses/filter.json" -// , query.asHttpParameterArray(stallWarningsParam), auth), conf); -// } catch (IOException e) { -// throw new TwitterException(e); -// } -// } -// -// -// /** -// * check if any listener is set. Throws IllegalStateException if no listener is set. -// * -// * @throws IllegalStateException when no listener is set. -// */ -// -// private void ensureStatusStreamListenerIsSet() { -// if (statusListeners.size() == 0 && rawStreamListeners.size() == 0) { -// throw new IllegalStateException("StatusListener is not set."); -// } -// } -// -// private void ensureUserStreamListenerIsSet() { -// if (userStreamListeners.size() == 0 && rawStreamListeners.size() == 0) { -// throw new IllegalStateException("UserStreamListener is not set."); -// } -// } -// -// private void ensureSiteStreamsListenerIsSet() { -// if (siteStreamsListeners.size() == 0 && rawStreamListeners.size() == 0) { -// throw new IllegalStateException("SiteStreamsListener is not set."); -// } -// } -// -// private static int numberOfHandlers = 0; -// -// private synchronized void startHandler(TwitterStreamConsumer handler) { -// cleanUp(); -// this.handler = handler; -// this.handler.start(); -// numberOfHandlers++; -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public synchronized void cleanUp() { -// if (handler != null) { -// handler.close(); -// numberOfHandlers--; -// } -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public synchronized void shutdown() { -// super.shutdown(); -// cleanUp(); -// synchronized (TwitterStreamImpl.class) { -// if (0 == numberOfHandlers) { -// if (dispatcher != null) { -// dispatcher.shutdown(); -// dispatcher = null; -// } -// } -// } -// } -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void addConnectionLifeCycleListener(ConnectionLifeCycleListener listener) { -// this.lifeCycleListeners.add(listener); -// } -// -// private List userStreamListeners = new ArrayList(0); -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void addListener(UserStreamListener listener) { -// statusListeners.add(listener); -// userStreamListeners.add(listener); -// } -// -// private List statusListeners = new ArrayList(0); -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void addListener(StatusListener listener) { -// statusListeners.add(listener); -// } -// -// private List siteStreamsListeners = new ArrayList(0); -// -// /** -// * {@inheritDoc} -// */ -// @Override -// public void addListener(SiteStreamsListener listener) { -// siteStreamsListeners.add(listener); -// } -// -// private List rawStreamListeners = new ArrayList(0); -// -// /** -// * {@inheritDoc} -// */ -// public void addListener(RawStreamListener listener) { -// rawStreamListeners.add(listener); -// } -// -// /* -// https://dev.twitter.com/docs/streaming-api/concepts#connecting -// When a network error (TCP/IP level) is encountered, back off linearly. Perhaps start at 250 milliseconds, double, and cap at 16 seconds -// When a HTTP error (> 200) is returned, back off exponentially. -// Perhaps start with a 10 second wait, double on each subsequent failure, and finally cap the wait at 240 seconds. Consider sending an alert to a human operator after multiple HTTP errors, as there is probably a client configuration issue that is unlikely to be resolved without human intervention. There's not much point in polling any faster in the face of HTTP error codes and your client is may run afoul of a rate limit. -// */ -// private static final int TCP_ERROR_INITIAL_WAIT = 250; -// private static final int TCP_ERROR_WAIT_CAP = 16 * 1000; -// -// private static final int HTTP_ERROR_INITIAL_WAIT = 10 * 1000; -// private static final int HTTP_ERROR_WAIT_CAP = 240 * 1000; -// -// private static final int NO_WAIT = 0; -// -// static int count = 0; -// -// abstract class TwitterStreamConsumer extends Thread { -// private StatusStreamBase stream = null; -// private final String NAME = "Twitter Stream consumer-" + (++count); -// private volatile boolean closed = false; -// private final StreamListener[] streamListeners; -// private final RawStreamListener[] rawStreamListeners; -// -// TwitterStreamConsumer(List streamListeners, List rawStreamListeners) { -// super(); -// setName(NAME + "[initializing]"); -// this.streamListeners = streamListeners.toArray(new StreamListener[streamListeners.size()]); -// this.rawStreamListeners = rawStreamListeners.toArray(new RawStreamListener[rawStreamListeners.size()]); -// } -// -// @Override -// public void run() { -// int timeToSleep = NO_WAIT; -// boolean connected = false; -// while (!closed) { -// try { -// if (!closed && null == stream) { -// // try establishing connection -// logger.info("Establishing connection."); -// setStatus("[Establishing connection]"); -// stream = (StatusStreamBase) getStream(); -// connected = true; -// logger.info("Connection established."); -// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { -// try { -// listener.onConnect(); -// } catch (Exception e) { -// logger.warn(e.getMessage()); -// } -// } -// // connection established successfully -// timeToSleep = NO_WAIT; -// logger.info("Receiving status stream."); -// setStatus("[Receiving stream]"); -// while (!closed) { -// try { -// stream.next(this.streamListeners, this.rawStreamListeners); -// } catch (IllegalStateException ise) { -// logger.warn(ise.getMessage()); -// break; -// } catch (TwitterException e) { -// logger.info(e.getMessage()); -// stream.onException(e, this.streamListeners, this.rawStreamListeners); -// throw e; -// } catch (Exception e) { -// logger.info(e.getMessage()); -// stream.onException(e, this.streamListeners, this.rawStreamListeners); -// closed = true; -// break; -// } -// } -// } -// } catch (TwitterException te) { -// logger.info(te.getMessage()); -// if (!closed) { -// if (NO_WAIT == timeToSleep) { -// if (te.getStatusCode() == FORBIDDEN) { -// logger.warn("This account is not in required role. ", te.getMessage()); -// closed = true; -// for (StreamListener statusListener : streamListeners) { -// statusListener.onException(te); -// } -// break; -// } -// if (te.getStatusCode() == NOT_ACCEPTABLE) { -// logger.warn("Parameter not accepted with the role. ", te.getMessage()); -// closed = true; -// for (StreamListener statusListener : streamListeners) { -// statusListener.onException(te); -// } -// break; -// } -// connected = false; -// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { -// try { -// listener.onDisconnect(); -// } catch (Exception e) { -// logger.warn(e.getMessage()); -// } -// } -// if (te.getStatusCode() > 200) { -// timeToSleep = HTTP_ERROR_INITIAL_WAIT; -// } else if (0 == timeToSleep) { -// timeToSleep = TCP_ERROR_INITIAL_WAIT; -// } -// } -// if (te.getStatusCode() > 200 && timeToSleep < HTTP_ERROR_INITIAL_WAIT) { -// timeToSleep = HTTP_ERROR_INITIAL_WAIT; -// } -// if (connected) { -// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { -// try { -// listener.onDisconnect(); -// } catch (Exception e) { -// logger.warn(e.getMessage()); -// } -// } -// } -// for (StreamListener statusListener : streamListeners) { -// statusListener.onException(te); -// } -// // there was a problem establishing the connection, or the connection closed by peer -// if (!closed) { -// // wait for a moment not to overload Twitter API -// logger.info("Waiting for " + (timeToSleep) + " milliseconds"); -// setStatus("[Waiting for " + (timeToSleep) + " milliseconds]"); -// try { -// Thread.sleep(timeToSleep); -// } catch (InterruptedException ignore) { -// } -// timeToSleep = Math.min(timeToSleep * 2, (te.getStatusCode() > 200) ? HTTP_ERROR_WAIT_CAP : TCP_ERROR_WAIT_CAP); -// } -// stream = null; -// logger.debug(te.getMessage()); -// connected = false; -// } -// } -// } -// if (this.stream != null && connected) { -// try { -// this.stream.close(); -// } catch (IOException ignore) { -// } catch (Exception e) { -// e.printStackTrace(); -// logger.warn(e.getMessage()); -// } finally { -// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { -// try { -// listener.onDisconnect(); -// } catch (Exception e) { -// logger.warn(e.getMessage()); -// } -// } -// } -// } -// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { -// try { -// listener.onCleanUp(); -// } catch (Exception e) { -// logger.warn(e.getMessage()); -// } -// } -// } -// -// public synchronized void close() { -// setStatus("[Disposing thread]"); -// try { -// if (stream != null) { -// try { -// stream.close(); -// } catch (IOException ignore) { -// } catch (Exception e) { -// e.printStackTrace(); -// logger.warn(e.getMessage()); -// } -// } -// } finally { -// closed = true; -// } -// } -// -// private void setStatus(String message) { -// String actualMessage = NAME + message; -// setName(actualMessage); -// logger.debug(actualMessage); -// } -// -// abstract StatusStream getStream() throws TwitterException; -// -// } -// -// @Override -// public boolean equals(Object o) { -// if (this == o) return true; -// if (o == null || getClass() != o.getClass()) return false; -// if (!super.equals(o)) return false; -// -// TwitterStreamImpl that = (TwitterStreamImpl) o; -// -// if (handler != null ? !handler.equals(that.handler) : that.handler != null) return false; -// if (http != null ? !http.equals(that.http) : that.http != null) return false; -// if (lifeCycleListeners != null ? !lifeCycleListeners.equals(that.lifeCycleListeners) : that.lifeCycleListeners != null) -// return false; -// if (rawStreamListeners != null ? !rawStreamListeners.equals(that.rawStreamListeners) : that.rawStreamListeners != null) -// return false; -// if (siteStreamsListeners != null ? !siteStreamsListeners.equals(that.siteStreamsListeners) : that.siteStreamsListeners != null) -// return false; -// if (stallWarningsGetParam != null ? !stallWarningsGetParam.equals(that.stallWarningsGetParam) : that.stallWarningsGetParam != null) -// return false; -// if (stallWarningsParam != null ? !stallWarningsParam.equals(that.stallWarningsParam) : that.stallWarningsParam != null) -// return false; -// if (statusListeners != null ? !statusListeners.equals(that.statusListeners) : that.statusListeners != null) -// return false; -// if (userStreamListeners != null ? !userStreamListeners.equals(that.userStreamListeners) : that.userStreamListeners != null) -// return false; -// -// return true; -// } -// -// @Override -// public int hashCode() { -// int result = super.hashCode(); -// result = 31 * result + (http != null ? http.hashCode() : 0); -// result = 31 * result + (lifeCycleListeners != null ? lifeCycleListeners.hashCode() : 0); -// result = 31 * result + (handler != null ? handler.hashCode() : 0); -// result = 31 * result + (stallWarningsGetParam != null ? stallWarningsGetParam.hashCode() : 0); -// result = 31 * result + (stallWarningsParam != null ? stallWarningsParam.hashCode() : 0); -// result = 31 * result + (userStreamListeners != null ? userStreamListeners.hashCode() : 0); -// result = 31 * result + (statusListeners != null ? statusListeners.hashCode() : 0); -// result = 31 * result + (siteStreamsListeners != null ? siteStreamsListeners.hashCode() : 0); -// result = 31 * result + (rawStreamListeners != null ? rawStreamListeners.hashCode() : 0); -// return result; -// } -// -// @Override -// public String toString() { -// return "TwitterStreamImpl{" + -// "http=" + http + -// ", lifeCycleListeners=" + lifeCycleListeners + -// ", handler=" + handler + -// ", stallWarningsGetParam='" + stallWarningsGetParam + '\'' + -// ", stallWarningsParam=" + stallWarningsParam + -// ", userStreamListeners=" + userStreamListeners + -// ", statusListeners=" + statusListeners + -// ", siteStreamsListeners=" + siteStreamsListeners + -// ", rawStreamListeners=" + rawStreamListeners + -// '}'; -// } -//} -// -//class StreamingReadTimeoutConfiguration implements HttpClientWrapperConfiguration -//{ -// twitter4j.conf.Configuration nestedConf; -// -// StreamingReadTimeoutConfiguration(twitter4j.conf.Configuration httpConf) { -// this.nestedConf = httpConf; -// } -// -// @Override -// public String getHttpProxyHost() { -// return nestedConf.getHttpProxyHost(); -// } -// -// @Override -// public int getHttpProxyPort() { -// return nestedConf.getHttpProxyPort(); -// } -// -// @Override -// public String getHttpProxyUser() { -// return nestedConf.getHttpProxyUser(); -// } -// -// @Override -// public String getHttpProxyPassword() { -// return nestedConf.getHttpProxyPassword(); -// } -// -// @Override -// public int getHttpConnectionTimeout() { -// return nestedConf.getHttpConnectionTimeout(); -// } -// -// @Override -// public int getHttpReadTimeout() { -// // this is the trick that overrides connection timeout -// return nestedConf.getHttpStreamingReadTimeout(); -// } -// -// @Override -// public int getHttpRetryCount() { -// return nestedConf.getHttpRetryCount(); -// } -// -// @Override -// public int getHttpRetryIntervalSeconds() { -// return nestedConf.getHttpRetryIntervalSeconds(); -// } -// -// @Override -// public int getHttpMaxTotalConnections() { -// return nestedConf.getHttpMaxTotalConnections(); -// } -// -// @Override -// public int getHttpDefaultMaxPerRoute() { -// return nestedConf.getHttpDefaultMaxPerRoute(); -// } -// -// @Override -// public Map getRequestHeaders() { -// // turning off keepalive connection explicitly because Streaming API doesn't need keepalive connection. -// // and this will reduce the shutdown latency of streaming api connection -// // see also - http://jira.twitter4j.org/browse/TFJ-556 -// Map headers = new HashMap(nestedConf.getRequestHeaders()); -// headers.put("Connection", "close"); -// return headers; -// } -// -// @Override -// public boolean isPrettyDebugEnabled() { -// return nestedConf.isPrettyDebugEnabled(); -// } -// -// @Override -// public boolean isGZIPEnabled() { -// return nestedConf.isGZIPEnabled(); -// } -//} -// diff --git a/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java b/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java deleted file mode 100644 index d214d755260..00000000000 --- a/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java +++ /dev/null @@ -1,111 +0,0 @@ -package druid.examples.twitter; - -/* - * Copyright 2007 Yusuke Yamamoto - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.ArrayList; -import java.util.List; - - -import java.util.ArrayList; -import java.util.List; - -/** - * @author Yusuke Yamamoto - yusuke at mac.com - * @since Twitter4J 2.1.4 - */ -public class z_InternalStringUtil { - private z_InternalStringUtil() { - throw new AssertionError(); - } - - public static String maskString(String str) { - StringBuilder buf = new StringBuilder(str.length()); - for (int i = 0; i < str.length(); i++) { - buf.append("*"); - } - return buf.toString(); - } - - // for JDK1.4 compatibility - - public static String[] split(String str, String separator) { - String[] returnValue; - int index = str.indexOf(separator); - if (index == -1) { - returnValue = new String[]{str}; - } else { - List strList = new ArrayList(); - int oldIndex = 0; - while (index != -1) { - String subStr = str.substring(oldIndex, index); - strList.add(subStr); - oldIndex = index + separator.length(); - index = str.indexOf(separator, oldIndex); - } - if (oldIndex != str.length()) { - strList.add(str.substring(oldIndex)); - } - returnValue = strList.toArray(new String[strList.size()]); - } - - return returnValue; - } - - public static String join(int[] follows) { - StringBuilder buf = new StringBuilder(11 * follows.length); - for (int follow : follows) { - if (0 != buf.length()) { - buf.append(","); - } - buf.append(follow); - } - return buf.toString(); - } - - public static String join(long[] follows) { - StringBuilder buf = new StringBuilder(11 * follows.length); - for (long follow : follows) { - if (0 != buf.length()) { - buf.append(","); - } - buf.append(follow); - } - return buf.toString(); - } - - public static String join(String[] track) { - StringBuilder buf = new StringBuilder(11 * track.length); - for (String str : track) { - if (0 != buf.length()) { - buf.append(","); - } - buf.append(str); - } - return buf.toString(); - } - - public static String join(List strs) { - StringBuilder buf = new StringBuilder(11 * strs.size()); - for (String str : strs) { - if (0 != buf.length()) { - buf.append(","); - } - buf.append(str); - } - return buf.toString(); - } -} diff --git a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java new file mode 100644 index 00000000000..3081b71928f --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java @@ -0,0 +1,27 @@ +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; + +public class TestCaseSupplier implements InputSupplier +{ + private final ArrayList inputList = new ArrayList(); + public TestCaseSupplier(String s){ + inputList.add(s); + } + @Override + public BufferedReader getInput() throws IOException + { + StringBuilder buffer = new StringBuilder(); + for (String current : inputList) { + buffer.append(current).append('\n'); + } + + BufferedReader br = new BufferedReader(new StringReader(buffer.toString())); + return br; + } +} diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java new file mode 100644 index 00000000000..84f11585539 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -0,0 +1,67 @@ +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; +import com.metamx.emitter.EmittingLogger; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.ProtocolException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +public class UpdateStream implements Runnable +{ + private static final EmittingLogger log = new EmittingLogger(UpdateStream.class); + private InputSupplier supplier; + private BlockingQueue> queue; + public UpdateStream(InputSupplier supplier,BlockingQueue> queue, String s){ + this.supplier=supplier; + this.queue=queue; + } + + @Override + public void run() throws RuntimeException + { + try{ + BufferedReader reader = (BufferedReader) supplier.getInput(); + String line; + ObjectMapper mapper = new ObjectMapper(); + TypeReference> typeRef = new TypeReference> () {}; + while ((line = reader.readLine())!= null){ + if(!line.equals("")){ + try{ + HashMap map=mapper.readValue(line, typeRef);; + queue.offer(map, 15L, TimeUnit.SECONDS); + log.info("Successfully added to queue"); + } + catch (JsonParseException e){ + System.out.println("Invalid JSON Stream. Please check if the url returns a proper JSON stream."); + throw new RuntimeException("Invalid JSON Stream"); + } + catch (Exception e){ + System.out.println(e); + return; + } + } + } + } + catch (MalformedURLException e){ + throw new RuntimeException("Malformed url"); + } + catch (ProtocolException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + throw new RuntimeException(e.getMessage()); + } + catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + throw new RuntimeException(e.getMessage()); + } + + } +} diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java new file mode 100644 index 00000000000..d816ce8f9f6 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -0,0 +1,139 @@ +package druid.examples.webStream; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.metamx.druid.input.InputRow; +import com.metamx.druid.input.MapBasedInputRow; +import com.metamx.druid.realtime.firehose.Firehose; +import com.metamx.druid.realtime.firehose.FirehoseFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +@JsonTypeName("webstream") +public class WebFirehoseFactory implements FirehoseFactory +{ + private final String url; + private final List dimensions; + private final String timeDimension; + private final List newDimensionNames; + + @JsonCreator + public WebFirehoseFactory( + @JsonProperty("url") String url, + @JsonProperty("dimensions") List dimensions, + @JsonProperty("newDimensionNames") List newDimensionNames, + @JsonProperty("timeDimension") String s + ) + { + this.url=url; + this.dimensions=dimensions; + this.timeDimension = s; + this.newDimensionNames=newDimensionNames; + } + + @Override + public Firehose connect() throws IOException + { + final int QUEUE_SIZE=2000; + final BlockingQueue> queue= new ArrayBlockingQueue>(QUEUE_SIZE); + + Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions,url),queue,timeDimension); + Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread th, Throwable ex) { + System.out.println("Uncaught exception: " + ex); + } + }; + final Thread t = new Thread(updateStream); + t.setUncaughtExceptionHandler(h); + t.start(); + + return new Firehose() { + private final Runnable doNothingRunnable = new Runnable() { + public void run(){ + + } + }; + + @Override + public boolean hasMore(){ + if (t.isAlive()){ + return true; + } + else{ + return false; + } + } + + + @Override + public InputRow nextRow() + { + if (Thread.currentThread().isInterrupted()) { + throw new RuntimeException("Interrupted, time to stop"); + } + Map update; + try{ + update=queue.take(); + } + catch (InterruptedException e) { + throw new RuntimeException("InterrutpedException", e); + } + Map processedMap = processMap(update); + return new MapBasedInputRow(((Integer) processedMap.get(timeDimension)).longValue()*1000,newDimensionNames,processedMap); + } + + private Map renameKeys (Map update){ + Map renamedMap = new HashMap(); + int iter=0; + while (iter map) + { + for (String key:newDimensionNames){ + if (map.get(key)==null){ + if (key.equals(timeDimension)){ + map.put(key,new Integer((int) System.currentTimeMillis()/1000)); + } + else{ + map.put(key, null); + } + } + } + } + + private Map processMap(Map map){ + Map renamedMap = renameKeys(map); + processNullDimensions(renamedMap); + return renamedMap; + } + + + + @Override + public Runnable commit() + { + // ephemera in, ephemera out. + return doNothingRunnable; // reuse the same object each time + } + + @Override + public void close() throws IOException + { + System.out.println("CLOSING!!!"); + } + + }; + } +} diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java new file mode 100644 index 00000000000..c495900a6e8 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -0,0 +1,172 @@ +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; +import com.metamx.druid.input.InputRow; +import com.metamx.druid.realtime.firehose.Firehose; +import com.metamx.druid.realtime.firehose.FirehoseFactory; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +public class WebFirehoseFactoryTest +{ + @Test + void testMalformedUrlConnect() throws Exception + { + List dimensions = new LinkedList(); + dimensions.add("g"); + dimensions.add("c"); + dimensions.add("a"); + dimensions.add("cy"); + dimensions.add("l"); + dimensions.add("hh"); + dimensions.add("hc"); + dimensions.add("h"); + dimensions.add("u"); + dimensions.add("tz"); + dimensions.add("t"); + dimensions.add("r"); + dimensions.add("gr"); + dimensions.add("nk"); + dimensions.add("al"); + + String invalidURL = "http://invalid.url"; + FirehoseFactory test = new WebFirehoseFactory(invalidURL,dimensions,dimensions,"t"); + Firehose returnedFirehose = test.connect(); + Thread.sleep(3000); + assert returnedFirehose.hasMore() == false; + } + + + @Test + public void testUrlWithNoJsonStreamConnect() throws Exception + { + List dimensions = new LinkedList(); + dimensions.add("g"); + dimensions.add("c"); + dimensions.add("a"); + dimensions.add("cy"); + dimensions.add("l"); + dimensions.add("hh"); + dimensions.add("hc"); + dimensions.add("h"); + dimensions.add("u"); + dimensions.add("tz"); + dimensions.add("t"); + dimensions.add("r"); + dimensions.add("gr"); + dimensions.add("nk"); + dimensions.add("al"); + + String nonJsonUrl = "http://google.com"; + FirehoseFactory test = new WebFirehoseFactory(nonJsonUrl,dimensions,dimensions,"t"); + Firehose returnedFirehose = test.connect(); + Thread.sleep(3000); + assert returnedFirehose.hasMore()== false; + } + + @Test + public void correctUrlCheck() throws Exception + { + List dimensions = new LinkedList(); + dimensions.add("g"); + dimensions.add("c"); + dimensions.add("a"); + dimensions.add("cy"); + dimensions.add("l"); + dimensions.add("hh"); + dimensions.add("hc"); + dimensions.add("h"); + dimensions.add("u"); + dimensions.add("tz"); + dimensions.add("t"); + dimensions.add("r"); + dimensions.add("gr"); + dimensions.add("nk"); + dimensions.add("al"); + + String url = "http://developer.usa.gov/1usagov"; + FirehoseFactory test = new WebFirehoseFactory(url,dimensions,dimensions,"t"); + Firehose returnedFirehose = test.connect(); + Thread.sleep(3000); + assert returnedFirehose.hasMore()== true; + } + + + @Test + public void basicIngestionCheck() throws Exception + { + final int QUEUE_SIZE=2000; + BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); + InputSupplier testCaseSupplier = new TestCaseSupplier("{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }"); + UpdateStream updateStream = new UpdateStream (testCaseSupplier, queue,"t"); + Thread t = new Thread(updateStream); + t.start(); + Map expectedAnswer = new HashMap(); + expectedAnswer.put("a","Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); + expectedAnswer.put("c","US"); + expectedAnswer.put("nk",1); + expectedAnswer.put ("tz", "America/New_York"); + expectedAnswer.put("gr", "NY"); + expectedAnswer.put("g","1Chgyj"); + expectedAnswer.put("h","15vMQjX"); + expectedAnswer.put("l","o_d63rn9enb"); + expectedAnswer.put("al","en-US,en;q=0.5"); + expectedAnswer.put("hh","1.usa.gov"); + expectedAnswer.put("r","http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX"); + expectedAnswer.put("u","http://www.spc.ncep.noaa.gov/"); + expectedAnswer.put("t",1372121562); + expectedAnswer.put("hc",1368193091); + expectedAnswer.put("cy","New York"); + expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); + Map insertedRow=queue.poll(10, TimeUnit.SECONDS); + assert expectedAnswer.equals(insertedRow); + } + + + + + + @Test + public void renameDimensionsCheck() throws Exception + { + List dimensions = new ArrayList(); + dimensions.add("bitly_hash"); + dimensions.add("country"); + dimensions.add("user"); + dimensions.add("city"); + dimensions.add("encoding_user_login"); + dimensions.add("short_url"); + dimensions.add("timestamp_hash"); + dimensions.add("user_bitly_hash"); + dimensions.add("url"); + dimensions.add("timezone"); + dimensions.add("time"); + dimensions.add("referring_url"); + dimensions.add("geo_region"); + dimensions.add("known_users"); + dimensions.add("accept_language"); + + WebFirehoseFactory webbie = new WebFirehoseFactory("http://developer.usa.gov/1usagov",dimensions,dimensions,"time"); + Firehose webbieHose = webbie.connect(); + final int QUEUE_SIZE=2000; + BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); + InputSupplier testCaseSupplier = new TestCaseSupplier("{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"kw\": \"spcnws\", \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }"); + UpdateStream updateStream = new UpdateStream (testCaseSupplier, queue,"time"); + Thread t = new Thread(updateStream); + t.start(); + InputRow row = webbieHose.nextRow(); + assert row.getDimensions().equals(dimensions); + + } + + +} diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java new file mode 100644 index 00000000000..dc559e3d68f --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -0,0 +1,37 @@ +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; +import com.metamx.common.Pair; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +public class WebJsonSupplier implements InputSupplier +{ + private BlockingQueue,Long>> queue; + private List dimensions; + private String urlString; + + public WebJsonSupplier(List dimensions, String urlString){ + this.dimensions=dimensions; + this.urlString=urlString; + } + @Override + public BufferedReader getInput() throws IOException + { + URL url = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setDoInput(true); + connection.setDoOutput(true); + connection.setRequestMethod("GET"); + + BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); + return reader; + } +} diff --git a/examples/twitter4j.properties b/examples/twitter4j.properties deleted file mode 100644 index 002d832143b..00000000000 --- a/examples/twitter4j.properties +++ /dev/null @@ -1,5 +0,0 @@ -debug=true -oauth.consumerKey=HOQS4c7WveN7NCxzl2aNgg -oauth.consumerSecret=47mKCGRfzml268RWDmRYGkFF6pusVrzoYrhRZPfELY -oauth.accessToken=19520065-aBZl60q698VYksk78oayY1eRFucofWo3FLI1tvekA -oauth.accessTokenSecret=Ul0bSghDzlYGKzN6lH7W6qZxW4bFCANfVXMs3KPfM diff --git a/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java b/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java deleted file mode 100644 index f57f5604f6b..00000000000 --- a/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.metamx.druid.realtime.firehose; - -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedInputRow; - -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -/** -* Created with IntelliJ IDEA. -* User: dhruvparthasarathy -* Date: 6/20/13 -* Time: 2:03 PM -* To change this template use File | Settings | File Templates. -*/ -public class WebFirehoseFactory implements FirehoseFactory -{ - - @Override - public Firehose connect() throws IOException - { - final UpdateListener updateListener; - final int QUEUE_SIZE=2000; - final BlockingQueue> queue= new ArrayBlockingQueue>(QUEUE_SIZE); - final LinkedList dimensions = new LinkedList(); - - dimensions.add("BITLY_HASH"); - dimensions.add("LATUTUDE-LONGITUDE"); - dimensions.add("COUNTRTY_CODE"); - dimensions.add("USER_AGENT"); - dimensions.add("CITY"); - dimensions.add("ENCODING_USER_LOGIN"); - dimensions.add("SHORT_URL_CNAME"); - dimensions.add("TIMESTAMP OF TIME HASH WAS CREATED"); - dimensions.add("ENCODING_USER_BITLY_HASH"); - dimensions.add("LONG_URL"); - dimensions.add("TIMEZONE"); - dimensions.add("TIMESTAMP OF TIME HASH WAS CREATED"); - dimensions.add("REFERRING_URL"); - dimensions.add("GEO_REGION"); - dimensions.add("KNOWN_USER"); - - UpdateStream updateStream = new UpdateStream(queue); - updateStream.start(); - - return new Firehose() { - private final Map theMap = new HashMap(); - - private final Runnable doNothingRunnable = new Runnable() { - public void run(){ - - } - }; - - @Override - public boolean hasMore(){ - if (queue.size()>0){ - return true; - } - return false; - } - - @Override - public InputRow nextRow() - { - if (Thread.currentThread().isInterrupted()) { - throw new RuntimeException("Interrupted, time to stop"); - } - Map update; - try{ - update=queue.take(); - } - catch (InterruptedException e) { - throw new RuntimeException("InterrutpedException", e); - } - - return new MapBasedInputRow(((Long) update.get("TIMESTAMP")).intValue(),dimensions,update); - } - }; - } -} From 0e1857f046f347508c8cb32a1d91eb28dc6b74ea Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 25 Jun 2013 14:48:02 -0700 Subject: [PATCH 03/25] Applied codestyle and optimized imports --- .../examples/webStream/TestCaseSupplier.java | 5 +- .../examples/webStream/UpdateStream.java | 31 ++++--- .../webStream/WebFirehoseFactory.java | 83 ++++++++++--------- .../webStream/WebFirehoseFactoryTest.java | 75 +++++++++-------- .../examples/webStream/WebJsonSupplier.java | 12 ++- 5 files changed, 115 insertions(+), 91 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java index 3081b71928f..ebaabf0f395 100644 --- a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java @@ -10,9 +10,12 @@ import java.util.ArrayList; public class TestCaseSupplier implements InputSupplier { private final ArrayList inputList = new ArrayList(); - public TestCaseSupplier(String s){ + + public TestCaseSupplier(String s) + { inputList.add(s); } + @Override public BufferedReader getInput() throws IOException { diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 84f11585539..1906bc212cf 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -19,39 +19,44 @@ public class UpdateStream implements Runnable { private static final EmittingLogger log = new EmittingLogger(UpdateStream.class); private InputSupplier supplier; - private BlockingQueue> queue; - public UpdateStream(InputSupplier supplier,BlockingQueue> queue, String s){ - this.supplier=supplier; - this.queue=queue; + private BlockingQueue> queue; + + public UpdateStream(InputSupplier supplier, BlockingQueue> queue) + { + this.supplier = supplier; + this.queue = queue; } @Override public void run() throws RuntimeException { - try{ + try { BufferedReader reader = (BufferedReader) supplier.getInput(); String line; ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference> () {}; - while ((line = reader.readLine())!= null){ - if(!line.equals("")){ - try{ - HashMap map=mapper.readValue(line, typeRef);; + TypeReference> typeRef = new TypeReference>() + { + }; + while ((line = reader.readLine()) != null) { + if (!line.equals("")) { + try { + HashMap map = mapper.readValue(line, typeRef); + ; queue.offer(map, 15L, TimeUnit.SECONDS); log.info("Successfully added to queue"); } - catch (JsonParseException e){ + catch (JsonParseException e) { System.out.println("Invalid JSON Stream. Please check if the url returns a proper JSON stream."); throw new RuntimeException("Invalid JSON Stream"); } - catch (Exception e){ + catch (Exception e) { System.out.println(e); return; } } } } - catch (MalformedURLException e){ + catch (MalformedURLException e) { throw new RuntimeException("Malformed url"); } catch (ProtocolException e) { diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index d816ce8f9f6..b9719a8f419 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -29,23 +29,25 @@ public class WebFirehoseFactory implements FirehoseFactory @JsonProperty("dimensions") List dimensions, @JsonProperty("newDimensionNames") List newDimensionNames, @JsonProperty("timeDimension") String s - ) + ) { - this.url=url; - this.dimensions=dimensions; + this.url = url; + this.dimensions = dimensions; this.timeDimension = s; - this.newDimensionNames=newDimensionNames; + this.newDimensionNames = newDimensionNames; } @Override public Firehose connect() throws IOException { - final int QUEUE_SIZE=2000; - final BlockingQueue> queue= new ArrayBlockingQueue>(QUEUE_SIZE); + final int QUEUE_SIZE = 2000; + final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions,url),queue,timeDimension); - Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread th, Throwable ex) { + Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions, url), queue); + Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() + { + public void uncaughtException(Thread th, Throwable ex) + { System.out.println("Uncaught exception: " + ex); } }; @@ -53,19 +55,22 @@ public class WebFirehoseFactory implements FirehoseFactory t.setUncaughtExceptionHandler(h); t.start(); - return new Firehose() { - private final Runnable doNothingRunnable = new Runnable() { - public void run(){ + return new Firehose() + { + private final Runnable doNothingRunnable = new Runnable() + { + public void run() + { } }; @Override - public boolean hasMore(){ - if (t.isAlive()){ + public boolean hasMore() + { + if (t.isAlive()) { return true; - } - else{ + } else { return false; } } @@ -77,50 +82,54 @@ public class WebFirehoseFactory implements FirehoseFactory if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } - Map update; - try{ - update=queue.take(); + Map update; + try { + update = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterrutpedException", e); } - Map processedMap = processMap(update); - return new MapBasedInputRow(((Integer) processedMap.get(timeDimension)).longValue()*1000,newDimensionNames,processedMap); + Map processedMap = processMap(update); + return new MapBasedInputRow( + ((Integer) processedMap.get(timeDimension)).longValue() * 1000, + newDimensionNames, + processedMap + ); } - private Map renameKeys (Map update){ - Map renamedMap = new HashMap(); - int iter=0; - while (iter renameKeys(Map update) + { + Map renamedMap = new HashMap(); + int iter = 0; + while (iter < dimensions.size()) { Object obj = update.get(dimensions.get(iter)); - renamedMap.put(newDimensionNames.get(iter),obj); + renamedMap.put(newDimensionNames.get(iter), obj); iter++; } return renamedMap; } - private void processNullDimensions(Map map) + private void processNullDimensions(Map map) { - for (String key:newDimensionNames){ - if (map.get(key)==null){ - if (key.equals(timeDimension)){ - map.put(key,new Integer((int) System.currentTimeMillis()/1000)); - } - else{ - map.put(key, null); + for (String key : newDimensionNames) { + if (map.get(key) == null) { + if (key.equals(timeDimension)) { + map.put(key, new Integer((int) System.currentTimeMillis() / 1000)); + } else { + map.put(key, null); } } } } - private Map processMap(Map map){ - Map renamedMap = renameKeys(map); + private Map processMap(Map map) + { + Map renamedMap = renameKeys(map); processNullDimensions(renamedMap); return renamedMap; } - @Override public Runnable commit() { diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index c495900a6e8..d62f8ddcdd2 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -39,7 +39,7 @@ public class WebFirehoseFactoryTest dimensions.add("al"); String invalidURL = "http://invalid.url"; - FirehoseFactory test = new WebFirehoseFactory(invalidURL,dimensions,dimensions,"t"); + FirehoseFactory test = new WebFirehoseFactory(invalidURL, dimensions, dimensions, "t"); Firehose returnedFirehose = test.connect(); Thread.sleep(3000); assert returnedFirehose.hasMore() == false; @@ -67,10 +67,10 @@ public class WebFirehoseFactoryTest dimensions.add("al"); String nonJsonUrl = "http://google.com"; - FirehoseFactory test = new WebFirehoseFactory(nonJsonUrl,dimensions,dimensions,"t"); + FirehoseFactory test = new WebFirehoseFactory(nonJsonUrl, dimensions, dimensions, "t"); Firehose returnedFirehose = test.connect(); Thread.sleep(3000); - assert returnedFirehose.hasMore()== false; + assert returnedFirehose.hasMore() == false; } @Test @@ -94,47 +94,49 @@ public class WebFirehoseFactoryTest dimensions.add("al"); String url = "http://developer.usa.gov/1usagov"; - FirehoseFactory test = new WebFirehoseFactory(url,dimensions,dimensions,"t"); + FirehoseFactory test = new WebFirehoseFactory(url, dimensions, dimensions, "t"); Firehose returnedFirehose = test.connect(); Thread.sleep(3000); - assert returnedFirehose.hasMore()== true; + assert returnedFirehose.hasMore() == true; } @Test public void basicIngestionCheck() throws Exception { - final int QUEUE_SIZE=2000; - BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - InputSupplier testCaseSupplier = new TestCaseSupplier("{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }"); - UpdateStream updateStream = new UpdateStream (testCaseSupplier, queue,"t"); + final int QUEUE_SIZE = 2000; + BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); + InputSupplier testCaseSupplier = new TestCaseSupplier( + "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" + ); + UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue); Thread t = new Thread(updateStream); t.start(); - Map expectedAnswer = new HashMap(); - expectedAnswer.put("a","Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); - expectedAnswer.put("c","US"); - expectedAnswer.put("nk",1); - expectedAnswer.put ("tz", "America/New_York"); + Map expectedAnswer = new HashMap(); + expectedAnswer.put("a", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); + expectedAnswer.put("c", "US"); + expectedAnswer.put("nk", 1); + expectedAnswer.put("tz", "America/New_York"); expectedAnswer.put("gr", "NY"); - expectedAnswer.put("g","1Chgyj"); - expectedAnswer.put("h","15vMQjX"); - expectedAnswer.put("l","o_d63rn9enb"); - expectedAnswer.put("al","en-US,en;q=0.5"); - expectedAnswer.put("hh","1.usa.gov"); - expectedAnswer.put("r","http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX"); - expectedAnswer.put("u","http://www.spc.ncep.noaa.gov/"); - expectedAnswer.put("t",1372121562); - expectedAnswer.put("hc",1368193091); - expectedAnswer.put("cy","New York"); + expectedAnswer.put("g", "1Chgyj"); + expectedAnswer.put("h", "15vMQjX"); + expectedAnswer.put("l", "o_d63rn9enb"); + expectedAnswer.put("al", "en-US,en;q=0.5"); + expectedAnswer.put("hh", "1.usa.gov"); + expectedAnswer.put( + "r", + "http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX" + ); + expectedAnswer.put("u", "http://www.spc.ncep.noaa.gov/"); + expectedAnswer.put("t", 1372121562); + expectedAnswer.put("hc", 1368193091); + expectedAnswer.put("cy", "New York"); expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); - Map insertedRow=queue.poll(10, TimeUnit.SECONDS); + Map insertedRow = queue.poll(10, TimeUnit.SECONDS); assert expectedAnswer.equals(insertedRow); } - - - @Test public void renameDimensionsCheck() throws Exception { @@ -155,12 +157,19 @@ public class WebFirehoseFactoryTest dimensions.add("known_users"); dimensions.add("accept_language"); - WebFirehoseFactory webbie = new WebFirehoseFactory("http://developer.usa.gov/1usagov",dimensions,dimensions,"time"); + WebFirehoseFactory webbie = new WebFirehoseFactory( + "http://developer.usa.gov/1usagov", + dimensions, + dimensions, + "time" + ); Firehose webbieHose = webbie.connect(); - final int QUEUE_SIZE=2000; - BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - InputSupplier testCaseSupplier = new TestCaseSupplier("{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"kw\": \"spcnws\", \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }"); - UpdateStream updateStream = new UpdateStream (testCaseSupplier, queue,"time"); + final int QUEUE_SIZE = 2000; + BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); + InputSupplier testCaseSupplier = new TestCaseSupplier( + "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"kw\": \"spcnws\", \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" + ); + UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue); Thread t = new Thread(updateStream); t.start(); InputRow row = webbieHose.nextRow(); diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java index dc559e3d68f..cfcfa52acdf 100644 --- a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -1,7 +1,6 @@ package druid.examples.webStream; import com.google.common.io.InputSupplier; -import com.metamx.common.Pair; import java.io.BufferedReader; import java.io.IOException; @@ -9,19 +8,18 @@ import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; public class WebJsonSupplier implements InputSupplier { - private BlockingQueue,Long>> queue; private List dimensions; private String urlString; - public WebJsonSupplier(List dimensions, String urlString){ - this.dimensions=dimensions; - this.urlString=urlString; + public WebJsonSupplier(List dimensions, String urlString) + { + this.dimensions = dimensions; + this.urlString = urlString; } + @Override public BufferedReader getInput() throws IOException { From 4e4a0a79532e8285d36a72b49c01906e36fa44ba Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 25 Jun 2013 14:58:12 -0700 Subject: [PATCH 04/25] removed DS_Store files --- .DS_Store | Bin 15364 -> 0 bytes .gitignore | 1 + common/.DS_Store | Bin 6148 -> 0 bytes common/src/.DS_Store | Bin 6148 -> 0 bytes common/src/main/.DS_Store | Bin 6148 -> 0 bytes common/src/main/java/.DS_Store | Bin 6148 -> 0 bytes common/src/main/java/com/.DS_Store | Bin 6148 -> 0 bytes common/src/main/java/com/metamx/.DS_Store | Bin 6148 -> 0 bytes examples/.DS_Store | Bin 6148 -> 0 bytes indexing-common/.DS_Store | Bin 6148 -> 0 bytes indexing-common/src/.DS_Store | Bin 6148 -> 0 bytes indexing-common/src/main/.DS_Store | Bin 6148 -> 0 bytes indexing-common/src/main/java/.DS_Store | Bin 6148 -> 0 bytes install/.DS_Store | Bin 6148 -> 0 bytes realtime/.DS_Store | Bin 6148 -> 0 bytes server/.DS_Store | Bin 6148 -> 0 bytes services/.DS_Store | Bin 6148 -> 0 bytes 17 files changed, 1 insertion(+) delete mode 100644 .DS_Store delete mode 100644 common/.DS_Store delete mode 100644 common/src/.DS_Store delete mode 100644 common/src/main/.DS_Store delete mode 100644 common/src/main/java/.DS_Store delete mode 100644 common/src/main/java/com/.DS_Store delete mode 100644 common/src/main/java/com/metamx/.DS_Store delete mode 100644 examples/.DS_Store delete mode 100644 indexing-common/.DS_Store delete mode 100644 indexing-common/src/.DS_Store delete mode 100644 indexing-common/src/main/.DS_Store delete mode 100644 indexing-common/src/main/java/.DS_Store delete mode 100644 install/.DS_Store delete mode 100644 realtime/.DS_Store delete mode 100644 server/.DS_Store delete mode 100644 services/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index d1abbd6f4f83617f08b6bedf8dee91d153c650cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15364 zcmeI1-A)rh6vxj%q1%X&8;ss9UU7k>l@hq{qI?-i)EG*_g&1tRm2R@_Hf{MxYPISs zc;Q>sk=3m41D3EYMh zZs#jeM$D6GNCkT$hvukFwGGd&B(&v$5D)@FKnMr{A@DyCz;`wm>sBD;E(C;t5Eu~P z^})u)vS4ICkb3FBO>Y4pQ+TWk>evTp8{5c&k^MkwQ}_&a56YM-v&B$>IofSjhb$P` z4`g6YDljKy-mJ_HMd`Q0&erOr3W1cn5D)@M0@HY8QJMD1B`>^xFMEC&Z9JWx;F>wn z1vR2^szBGJDmAFXd;O5^k0H7~)$#GE5!F?j=joOY+fC#5Te<1E?M^oPQ;m$K#>UfD zCSy(PY`YD|KV0$m8&3OGvF0BZU8hM1UntCwz_~};ZUGM#eKDv#f%t}PJ zgTDv&PpC1X!LSdCt%m0}_Sgv5C@}j}iNW}VmN4WDv|2hWe5m3Apck_MK6M-l%M`5* zELQP`7B%&~L>+``m7dWuJT>9>ZPb$v@Yj&1yCF{-2(p9sAK-G(pAtfUo-a;-sIRI? zmnM#esu^Vgsn!rR&U&Aku@@mH#|W^ty*=L|AL z|EHi%PM9{uv_tuE4Huo*U}Eu4V*%K})Lo|+RHXGNMjh{?P`uN6@u&}}{@7XiS0AKY4&Mfk^7+g7W=IS_7N&8% zN^-U%=Z6I87-&)kO+@$Vv0UyK=IJEIuwzK90TLH0R$|%L*~nN%2N`qf%jjQkG>PHI z@L=Rpgn$qb0zyCtT%5pYplb8^|N7a#|6kmpXbJ%#aP0`F)LLHV9(Tb0N&)RKV zZ*Xy8y?!9I32v;y&-HOU{@~;I8S^a5k_*}kM)m`#&GB5DLk6e$|Nh4S+5ZpOT84M@ yvj1=WIT^l*`+uH)QiOmI5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDpKu1bzd*#X#i% diff --git a/.gitignore b/.gitignore index 31985c78113..845a2ddc007 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ target examples/rand/RealtimeNode.out examples/twitter/RealtimeNode.out *.log +*.DS_Store diff --git a/common/.DS_Store b/common/.DS_Store deleted file mode 100644 index 3e05fc0a0239b1d753108e06558e41f99c476523..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z<*_H>i+{(BtB{NJ=gB;30$x9)%Qou%d|(8wg2BQhTV;;4}ChzKzf0 z%M5lQ@mCrI#(zsJQZH+0u`r`!L9_ z6)%HwQFuYFJeNT^1CQe4EK7h-P4py=%Gr(GS*`kx)0Mr!pz6xia0vXQe_B8qkQ5*VNC8rS6i^D7148VP z9;5&%Kni?Q0QUz81<^HFYgAhYRCs+xe;W}EIKCwir9szVtr0w6T&Du+RBoOaT&F|7 zG;yxMTBA;9T+IyQn30tS5T5&#X{M>@)>{SrCExdv;EI12WwbU?ZY MC_$(q1%5$+PkJjxzW@LL diff --git a/common/src/.DS_Store b/common/src/.DS_Store deleted file mode 100644 index 73307bf9352f15b2d73524ac7d4be4c294bd50f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!AiqG5Z!H~Ca92$(BtB{NJ=gB;30$x9)%Qou%ZbO8;D71QhTV;;5Yam{*B+` z%0Qb%X6wBNT7SeMO6f^KhKF+cP_*6$v;;5Ki+nx2g?>Jr28w|>>SPzH5kNYQOX_=;F zznTY!N2llG;IVvqu9S`(WR|wCv4$5QpT~Ic7ip5EcQ9AS*KvfT04YEUkOHJYeL0xB zqy;HJ3XlTd6u{z9KtpsjRtjZZ2_2Tt=r<72fa6;NQD}5ERtmub##JhyO6B^A!BslU z3mxZbtQ4wr#^uhij^4R`yl}ZY%nKRLxT}zQQh*d_D$rI#8}I)!_{nU1kp*nQg0J5nPLnt<9Os=kn#RuVo@rU;!FUv>QMUB5c^c){{v=!aadaOB zvm42apqS@gFq59EpqPM1@?n-Gz{fgz6i3D6)^4p&)r&(b7IA7HMIuj2?w0aAbzAO%Q)`f@P$ zNefbd6d(n@DS*YJfQIO5tQ5++5;`oO(QhE40mrulqR{AStQ3L=jH^^YmCE%KgR69y z7dp<>SSeKLjLV&29ldk?c;Rw)m=`jfaaSSrqyQ7k-wQjBT|4A z_-6`mW8e=wIF!Cye;g0*S^@128XCq`NWg$^6ag^6|B;q-D!)JuajwQnA+Cb+G98c# N0VxP|q`)sI@By;bM|=PP diff --git a/common/src/main/java/.DS_Store b/common/src/main/java/.DS_Store deleted file mode 100644 index 3c38389f4b6ce453a087e350f406df53d2a5baad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-O8Ca92$(BtB{NJ=gB;30$x9)%Qou%d|(8;IG`r1nsw!DsM2d>fy~ zncWRo?O8--VE6lWXJ$A1LH36+#{EUyWz1oWSx^u;k`0F48(j?(jL30}d={lLg4TO9 zGrqqL`0Y)0%OrbUe^4C+SMGBAt z|4aei8U{lj7G=)XAIrnDRzSIdf`)Mw5-{LvmjGzsKGIfB?H9-)&NEnP#8I$cr32DM NKnX$>Dewyld;;U3My&t< diff --git a/common/src/main/java/com/.DS_Store b/common/src/main/java/com/.DS_Store deleted file mode 100644 index 7274933556578f5532ba45ef704dae81dded9531..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~v94NJ=gB;30$x9)%Qou%ZbmHjpNzNv%+$!DsM2d>fy~ zncWQ-%vr?F!0b0WKfBovvOkP5?yQ3zV+LbP14ZO$)Cjt3Lk*RT$i)~7_T+EUr5`1J z%|L(Ah2P$03pQgJ^8mkp|A)Vf!Zfq2Pu^&@_S*ZJu4{+WNf-yo#!1$3klwiSWaEax z%=cEe!p^*GojTr1M6SJT4iRCGlOzIrs!0Vj zu4%gQt{tD8UR;g6=ltchko<9=TF8;Y65hb5EalN%#ZeMJfWNFMa|ww7Vt^PR26mVM zbtq^DJInzsofsen>KVZO!30IL6&4c3)&UJ(pE2G-L;)M$5{RZkTVWv)A|PCr0?JaZ zo)}z~gWptnw!%W9EN5KI4CAPo%f|~BvxDCh>5SVFsU-%8fhq${*){O|zl2|=_K{yL zp&l_n4E!?&cx&X299Wb(TYoGM&sr1o9ux)h3N%2#u3Q43gZoHF1ZCVr8}e+0g+v?$ S{h}O@E&_rOYKVbfVBiZyhe?V6 diff --git a/common/src/main/java/com/metamx/.DS_Store b/common/src/main/java/com/metamx/.DS_Store deleted file mode 100644 index de97bb0c5e9924efedd9e0823865ec36191fac9d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOH0E*5Z>)W6I94W=rMRMl2S`OcnG0_McZy)Zn86s+KoL0_4TA2*P)j8vaxq5Wud<~dC4R#& zf76BE-emzxS;0cK`Tbx1G77WYvc7q%-RT}2YPzl+&u3vABr7K=;vlKUQI<0S5Z-NTO;I5ip~uB@krt(`;30$x9)%Qou%ZbmZ6G8iN$sIVgU{f5_%=R| zGrOBZFlP}v1GC@k{Oo2w$o??KxV`ZQjG2rv0~C>?Q6uQC4K;K!B9~(zyQ7DRr$wc}0Xr}xe(-a3K5 z@Z97<+L@beQrk^rodk%h$vv=TLMvNv@{k9Ap*ixDWEFl z`ia3+IrxRnvosb8RXO8wW*A4$Ts~g7oE`i^rZa9Sq?Q;U2C57+RoBGx{|0`U+DCr1 zgnGmPG4RhA;H`->v0+jAZ2hr3JZlB$6DSJi6=;BfUAY862ltV#463+58}clTg+d$! S{jwa8E&`GeYKVbfVBj0ZI7x*7 diff --git a/indexing-common/.DS_Store b/indexing-common/.DS_Store deleted file mode 100644 index feca8613504bcd7430524202412f605a970dd548..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~JqiLr422VS3&Cbf%V|7-HyA`u-~~i21wpZ&qx41vI)z6#m+=T zbn`f`MFtU>!%bynVPcAXCwIBY>3lz3j@RpDteEA>YT$!Ro{xoGkN^pg011!)3H%HJ zJGWu;MJOW)kN^pc1nmD%;HEXTh5D}p!AAgSkF*=sK1)E0C7?C6g(3sfXoW_r`WRw) zZ-`o{NKSp&Hs}YZb^Uy{)~Y3 z+x>QfkIJ+4?ei>u$gHg!9O~r=FP8vp>?mHs-SE8F0$NjBC^9f!1RMhc34E2n6Mwc5 Aa{vGU diff --git a/indexing-common/src/.DS_Store b/indexing-common/src/.DS_Store deleted file mode 100644 index 9d6a56620545250bb9a34aa5ef87a586fdec28fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~J#ND=428+mgMlm_Giu2JdIP~oPS6W9ZI%FTJOtf!^gjNeK$Xr#5j+CqCsHB| zzriv^MCZ@*v&bYOBitx^3!N$QK|bUp7rCBq@5|e|E_r5?v@`HYy8YZHr~nn90#tws zP=SA>fZf}$aT{c$0#twsd@EqzhXOa&WDEMI1HnfCa6s7&Yo8^+Vg;}!TM!kPMk^St z>SKu2y&Wuht|nVB+C_8t(0sDm6a&*}7cEF&S{)2jfC_X9JVid){eOi2Hve}mOsN1B zcrXQYIJ_KYyj0$;cduvl*R0yQ!9hQc@Z%={iCx7zxEuD1Ex?*=K~!M;5pWq8sKB2p F@C6gX5~u(G diff --git a/indexing-common/src/main/.DS_Store b/indexing-common/src/main/.DS_Store deleted file mode 100644 index d38176301af6a9db8e9ffa5dc7916732489a254b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~JqiLr422WjLa^D=avBfd4F=H@cme+^5iCUOIl3=D2(H#5@&d^>$xK-G6+0Ud z(dF&95a~r^0yoOs!oU=HCnwp-U3SOo;k>_G&Ut2(v@`Ht2K%{9Pys4H1*iZOpaMUn zfZf}$$u!7F1*iZOcq(AuhXOa&WDEMI1HnfCutV7mYo8^+Vg;}!TM!kPMk^St>SKu2 zy&Wuht|nVB+C_8t(0sDm6a&*}7cEF&S{)2jfC>y07)RdQ{lA8Pn*RqaOsN1B_%j8x zTWwZLyj0$;Z?9+dZB}jF;GiEzc=-uHVps7D?uPwh3$P|z5EU4I1Y8CND)3bW9?Vn{ Ar2qf` diff --git a/indexing-common/src/main/java/.DS_Store b/indexing-common/src/main/java/.DS_Store deleted file mode 100644 index 2b615a767f6da08a02d4f0c4241f455170fff868..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~JqiLr422W55Nx)zoW=uqgF*BJUO;q9Wec&Nqx41vI)z6#m+=T zboc1DBAtjV;ij^&Ffqk`E*BZ(*55DJ^Dtb;idn9#2Hwf!`B=yW36KB@kN^pgz|Rn{ za~rl?g)))=36Q`@!2S;fZdy}YsQ)?;d<1|lkaok`X9;Mr1hl5MP-I{lt-v z{c*p;N9Ec2_IZ{+W!BaW4)t<`mrDRPb`-DSZg^g70j;Sm6d4#V0*--!1inh(38@+p AY5)KL diff --git a/install/.DS_Store b/install/.DS_Store deleted file mode 100644 index 33eea665350de4626480e3f764234d3db87f9fef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyGjH>5UkbK950p*sT?z=0;xb>fo<u)Yv~NVO8`4siub?d6>T%G3hW$>&T*p?^CFJ5&P${lz9-`!Pzc zdXER<*ZLdQANehF!|JN4*Q=^tk*{K1(of&a7y(CkLQVc0&n)}wmmYhB9cNs3$CT?H z>kti<++xF3G(GNXkIQOUC=d#S0--=C@FN9yW~)u6j!}jJp+G3`setYeiLO{=Y#nXu zU|~-HqF$rT80+;|b5h14W9!Hsns_SFQzbt!#M9ZHGOx(kI(j-JKRzVS{PB1Zo1Nn) zqeBYEC_{lzpsT>ht1~_SFZjz0KJvRMQ40k^fq$lejF&gd1wSgEt=~RR&)P)4rK>5g p)quvha|vKY_mLCp^!}tVc}2$7QAg2sODD!fzyygR6!--N-T@|8LGJ(n diff --git a/server/.DS_Store b/server/.DS_Store deleted file mode 100644 index 8c136e4f0413485ac8547313f579d9bbd85218f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%Sr=55Ue&q1G((c;~c#h@DG*{1+V%8nyg4*WhExT<9?tY;NNI<_o(c;2Tvl> z4Kp>5>fV`xotXw8^IdfY6aXeHilU7H)8WCXgG?S1#bQ)A#S`wZ!G0jnUmVi47pS=6 z9oEFJ_3JKX9D6{0SvRX?-7Jj@To6y*%xp_Xctk_q49^4XmmYgWg*I4qW6D*Jb%-r) z$-AeT+aC9|$F)U76$*p`p+G1Q3j9z3-q~uCsbkckKqwFjd@7*lLt;@ZGq#R)bg;4% zfT+)CGqyFCP@j~s%-A||h9>SxbXUnKhPXS&Q|6T!TSs??=D$M8m25C${{!mFx+#}+vm{@|x?nkZ%SI14z#|&+W_TW8zx3E6=3S%g=9H@* zYad(Ol6TK}Y7d{yOug zOotSPQHKJdKv#j0HK%(1pYxX)eB^gSq7@2+0{=__885CEb1ury)^E$xvo^8Zu&61o pRfk5ua|vKW_mSf~XyZwJ@(Rb+P_t-%4kyM%APW*zDDVpkyaPdALL2}9 From 039b06d64676e41b70b7e6c88955ccf29dd4b593 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 25 Jun 2013 15:01:19 -0700 Subject: [PATCH 05/25] made sure only edited files were pushed --- .../com/metamx/druid/client/DruidServer.java | 1 - examples/query.body | 12 ----- examples/search_query.body | 13 ------ .../TwitterSpritzerFirehoseFactory.java | 2 +- examples/twitter_realtime.spec | 44 ------------------- 5 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 examples/query.body delete mode 100644 examples/search_query.body delete mode 100644 examples/twitter_realtime.spec diff --git a/client/src/main/java/com/metamx/druid/client/DruidServer.java b/client/src/main/java/com/metamx/druid/client/DruidServer.java index 79bb8e3e0c2..ebd078e9383 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidServer.java +++ b/client/src/main/java/com/metamx/druid/client/DruidServer.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; - /** */ public class DruidServer implements Comparable diff --git a/examples/query.body b/examples/query.body deleted file mode 100644 index e0607aa1554..00000000000 --- a/examples/query.body +++ /dev/null @@ -1,12 +0,0 @@ -{ - "queryType": "groupBy", - "dataSource": "twitterstream", - "granularity": "all", - "dimensions": ["lang", "utc_offset"], - "aggregations":[ - { "type": "count", "name": "rows"}, - { "type": "doubleSum", "fieldName": "tweets", "name": "tweets"} - ], - "filter": { "type": "selector", "dimension": "lang", "value": "en" }, - "intervals":["2012-10-01T00:00/2020-01-01T00"] -} diff --git a/examples/search_query.body b/examples/search_query.body deleted file mode 100644 index f2feb27d4fe..00000000000 --- a/examples/search_query.body +++ /dev/null @@ -1,13 +0,0 @@ -{ - "queryType": "search", - "dataSource": "twitterstream", - "granularity": "all", - "searchDimensions": ["htags"], - "limit": 1, - "query": { - "type": "fragment", - "values": ["men"], - "sort": { "type": "strlen" } - }, - "intervals":["2012-10-01T00:00/2020-01-01T00"] -} diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java index ea1d8331924..e87b5e44de2 100644 --- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java @@ -139,7 +139,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { public void onStatus(Status status) { // time to stop? - if (Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { diff --git a/examples/twitter_realtime.spec b/examples/twitter_realtime.spec deleted file mode 100644 index f17aab07f17..00000000000 --- a/examples/twitter_realtime.spec +++ /dev/null @@ -1,44 +0,0 @@ -[{ - "schema": { - "dataSource": "twitterstream", - "aggregators": [ - {"type": "count", "name": "tweets"}, - {"type": "doubleSum", "fieldName": "follower_count", "name": "total_follower_count"}, - {"type": "doubleSum", "fieldName": "retweet_count", "name": "total_retweet_count" }, - {"type": "doubleSum", "fieldName": "friends_count", "name": "total_friends_count" }, - {"type": "doubleSum", "fieldName": "statuses_count", "name": "total_statuses_count"}, - - {"type": "min", "fieldName": "follower_count", "name": "min_follower_count"}, - {"type": "max", "fieldName": "follower_count", "name": "max_follower_count"}, - - {"type": "min", "fieldName": "friends_count", "name": "min_friends_count"}, - {"type": "max", "fieldName": "friends_count", "name": "max_friends_count"}, - - {"type": "min", "fieldName": "statuses_count", "name": "min_statuses_count"}, - {"type": "max", "fieldName": "statuses_count", "name": "max_statuses_count"}, - - {"type": "min", "fieldName": "retweet_count", "name": "min_retweet_count"}, - {"type": "max", "fieldName": "retweet_count", "name": "max_retweet_count"} - ], - "indexGranularity": "minute", - "shardSpec": {"type": "none"} - }, - - "config": { - "maxRowsInMemory": 50000, - "intermediatePersistPeriod": "PT2m" - }, - - "firehose": { - "type": "twitzer", - "maxEventCount": 500000, - "maxRunMinutes": 120 - }, - - "plumber": { - "type": "realtime", - "windowPeriod": "PT3m", - "segmentGranularity": "hour", - "basePersistDirectory": "/tmp/example/twitter_realtime/basePersist" - } -}] From e3f0b8770bb7930ff7492697de68c501b091dce0 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Wed, 26 Jun 2013 15:05:52 -0700 Subject: [PATCH 06/25] updated with Fangjin comments --- .../examples/webStream/NoopRunnable.java | 16 ++++++ .../examples/webStream/TestCaseSupplier.java | 11 ++-- .../examples/webStream/UpdateStream.java | 39 +++++++------ .../webStream/WebFirehoseFactory.java | 56 +++++++++---------- .../examples/webStream/WebJsonSupplier.java | 17 +++++- 5 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 examples/src/main/java/druid/examples/webStream/NoopRunnable.java diff --git a/examples/src/main/java/druid/examples/webStream/NoopRunnable.java b/examples/src/main/java/druid/examples/webStream/NoopRunnable.java new file mode 100644 index 00000000000..7001b28a436 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/NoopRunnable.java @@ -0,0 +1,16 @@ +package druid.examples.webStream; + +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/26/13 + * Time: 2:40 PM + * To change this template use File | Settings | File Templates. + */ +public class NoopRunnable implements Runnable +{ + @Override + public void run() + { + } +} diff --git a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java index ebaabf0f395..4198ade1fa8 100644 --- a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java @@ -1,5 +1,6 @@ package druid.examples.webStream; +import com.google.common.collect.Lists; import com.google.common.io.InputSupplier; import java.io.BufferedReader; @@ -9,21 +10,17 @@ import java.util.ArrayList; public class TestCaseSupplier implements InputSupplier { - private final ArrayList inputList = new ArrayList(); - + private final String s; public TestCaseSupplier(String s) { - inputList.add(s); + this.s=s; } @Override public BufferedReader getInput() throws IOException { StringBuilder buffer = new StringBuilder(); - for (String current : inputList) { - buffer.append(current).append('\n'); - } - + buffer.append(s); BufferedReader br = new BufferedReader(new StringReader(buffer.toString())); return br; } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 1906bc212cf..a266fd37f01 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -1,5 +1,6 @@ package druid.examples.webStream; +import com.google.common.base.Throwables; import com.google.common.io.InputSupplier; import com.metamx.emitter.EmittingLogger; import org.codehaus.jackson.JsonParseException; @@ -8,6 +9,7 @@ import org.codehaus.jackson.type.TypeReference; import java.io.BufferedReader; import java.io.IOException; +import java.lang.reflect.Type; import java.net.MalformedURLException; import java.net.ProtocolException; import java.util.HashMap; @@ -20,11 +22,21 @@ public class UpdateStream implements Runnable private static final EmittingLogger log = new EmittingLogger(UpdateStream.class); private InputSupplier supplier; private BlockingQueue> queue; + private ObjectMapper mapper; + private final TypeReference> typeRef; public UpdateStream(InputSupplier supplier, BlockingQueue> queue) { this.supplier = supplier; this.queue = queue; + this.mapper = new ObjectMapper(); + this.typeRef= new TypeReference>() + { + }; + } + + private boolean isValid(String s){ + return !(s.isEmpty()); } @Override @@ -33,39 +45,26 @@ public class UpdateStream implements Runnable try { BufferedReader reader = (BufferedReader) supplier.getInput(); String line; - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() - { - }; while ((line = reader.readLine()) != null) { - if (!line.equals("")) { + if (isValid(line)) { try { HashMap map = mapper.readValue(line, typeRef); - ; queue.offer(map, 15L, TimeUnit.SECONDS); log.info("Successfully added to queue"); } catch (JsonParseException e) { - System.out.println("Invalid JSON Stream. Please check if the url returns a proper JSON stream."); - throw new RuntimeException("Invalid JSON Stream"); + log.info("Invalid JSON Stream. Please check if the url returns a proper JSON stream."); + Throwables.propagate(e); } catch (Exception e) { - System.out.println(e); - return; + Throwables.propagate(e); } } } } - catch (MalformedURLException e) { - throw new RuntimeException("Malformed url"); - } - catch (ProtocolException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - throw new RuntimeException(e.getMessage()); - } - catch (IOException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - throw new RuntimeException(e.getMessage()); + catch (Exception e) { + e.printStackTrace(); + Throwables.propagate(e); } } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index b9719a8f419..843679956e4 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -7,6 +7,7 @@ import com.metamx.druid.input.InputRow; import com.metamx.druid.input.MapBasedInputRow; import com.metamx.druid.realtime.firehose.Firehose; import com.metamx.druid.realtime.firehose.FirehoseFactory; +import com.metamx.emitter.EmittingLogger; import java.io.IOException; import java.util.HashMap; @@ -14,27 +15,31 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @JsonTypeName("webstream") public class WebFirehoseFactory implements FirehoseFactory { + private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); private final String url; private final List dimensions; private final String timeDimension; - private final List newDimensionNames; + private final List renamedDimensions; @JsonCreator public WebFirehoseFactory( @JsonProperty("url") String url, @JsonProperty("dimensions") List dimensions, - @JsonProperty("newDimensionNames") List newDimensionNames, + @JsonProperty("renamedDimensions") List renamedDimensions, @JsonProperty("timeDimension") String s ) { this.url = url; this.dimensions = dimensions; + this.renamedDimensions = renamedDimensions; this.timeDimension = s; - this.newDimensionNames = newDimensionNames; } @Override @@ -44,35 +49,27 @@ public class WebFirehoseFactory implements FirehoseFactory final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions, url), queue); - Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() - { - public void uncaughtException(Thread th, Throwable ex) - { - System.out.println("Uncaught exception: " + ex); - } - }; - final Thread t = new Thread(updateStream); - t.setUncaughtExceptionHandler(h); - t.start(); +// Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() +// { +// public void uncaughtException(Thread th, Throwable ex) +// { +// log.info("Uncaught exception: " + ex); +// } +// }; +// final Thread streamReader = new Thread(updateStream); +// streamReader.setUncaughtExceptionHandler(h); +// streamReader.start(); + final ExecutorService service = Executors.newSingleThreadExecutor(); + service.submit(updateStream); return new Firehose() { - private final Runnable doNothingRunnable = new Runnable() - { - public void run() - { - - } - }; + private final Runnable doNothingRunnable = new NoopRunnable(); @Override public boolean hasMore() { - if (t.isAlive()) { - return true; - } else { - return false; - } + return !(service.isTerminated()); } @@ -92,7 +89,7 @@ public class WebFirehoseFactory implements FirehoseFactory Map processedMap = processMap(update); return new MapBasedInputRow( ((Integer) processedMap.get(timeDimension)).longValue() * 1000, - newDimensionNames, + renamedDimensions, processedMap ); } @@ -103,7 +100,7 @@ public class WebFirehoseFactory implements FirehoseFactory int iter = 0; while (iter < dimensions.size()) { Object obj = update.get(dimensions.get(iter)); - renamedMap.put(newDimensionNames.get(iter), obj); + renamedMap.put(renamedDimensions.get(iter), obj); iter++; } return renamedMap; @@ -111,7 +108,7 @@ public class WebFirehoseFactory implements FirehoseFactory private void processNullDimensions(Map map) { - for (String key : newDimensionNames) { + for (String key : renamedDimensions) { if (map.get(key) == null) { if (key.equals(timeDimension)) { map.put(key, new Integer((int) System.currentTimeMillis() / 1000)); @@ -140,7 +137,8 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public void close() throws IOException { - System.out.println("CLOSING!!!"); + log.info("CLOSING!!!"); + service.shutdown(); } }; diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java index cfcfa52acdf..8b5a3b24d5e 100644 --- a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -1,33 +1,44 @@ package druid.examples.webStream; import com.google.common.io.InputSupplier; +import com.metamx.emitter.EmittingLogger; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.util.List; public class WebJsonSupplier implements InputSupplier { private List dimensions; + private static final EmittingLogger log = new EmittingLogger(WebJsonSupplier.class); + private String urlString; + private URL url; public WebJsonSupplier(List dimensions, String urlString) { this.dimensions = dimensions; this.urlString = urlString; + try{ + this.url = new URL(urlString); + } + catch (Exception e){ + e.printStackTrace(); + log.info("Malformed url"); + } } @Override public BufferedReader getInput() throws IOException { URL url = new URL(urlString); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + URLConnection connection = url.openConnection(); connection.setDoInput(true); - connection.setDoOutput(true); - connection.setRequestMethod("GET"); + //connection.setDoOutput(true); BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); return reader; From 235ecce522346e14042fa4fbe56791c1070a2b66 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Wed, 26 Jun 2013 15:12:55 -0700 Subject: [PATCH 07/25] fixed constants and made them static finals --- .../src/main/java/druid/examples/webStream/UpdateStream.java | 3 ++- .../main/java/druid/examples/webStream/WebFirehoseFactory.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index a266fd37f01..c22e44ec83f 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -24,6 +24,7 @@ public class UpdateStream implements Runnable private BlockingQueue> queue; private ObjectMapper mapper; private final TypeReference> typeRef; + private static final long queueWaitTime = 15L; public UpdateStream(InputSupplier supplier, BlockingQueue> queue) { @@ -49,7 +50,7 @@ public class UpdateStream implements Runnable if (isValid(line)) { try { HashMap map = mapper.readValue(line, typeRef); - queue.offer(map, 15L, TimeUnit.SECONDS); + queue.offer(map, queueWaitTime, TimeUnit.SECONDS); log.info("Successfully added to queue"); } catch (JsonParseException e) { diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 843679956e4..c18a13e4a2c 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -27,6 +27,8 @@ public class WebFirehoseFactory implements FirehoseFactory private final List dimensions; private final String timeDimension; private final List renamedDimensions; + private static final int QUEUE_SIZE = 2000; + @JsonCreator public WebFirehoseFactory( @@ -45,7 +47,6 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public Firehose connect() throws IOException { - final int QUEUE_SIZE = 2000; final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions, url), queue); From 8bc4d7c436b79718706cd42168ca0a9f8302fdf9 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Wed, 26 Jun 2013 15:16:21 -0700 Subject: [PATCH 08/25] added druid header --- .../examples/webStream/NoopRunnable.java | 26 +++++++++++----- .../examples/webStream/TestCaseSupplier.java | 24 +++++++++++++-- .../examples/webStream/UpdateStream.java | 30 ++++++++++++++----- .../webStream/WebFirehoseFactory.java | 20 ++++++++++++- .../webStream/WebFirehoseFactoryTest.java | 19 ++++++++++++ .../examples/webStream/WebJsonSupplier.java | 26 +++++++++++++--- 6 files changed, 123 insertions(+), 22 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/NoopRunnable.java b/examples/src/main/java/druid/examples/webStream/NoopRunnable.java index 7001b28a436..6ccd4ba1584 100644 --- a/examples/src/main/java/druid/examples/webStream/NoopRunnable.java +++ b/examples/src/main/java/druid/examples/webStream/NoopRunnable.java @@ -1,12 +1,24 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; -/** - * Created with IntelliJ IDEA. - * User: dhruvparthasarathy - * Date: 6/26/13 - * Time: 2:40 PM - * To change this template use File | Settings | File Templates. - */ public class NoopRunnable implements Runnable { @Override diff --git a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java index 4198ade1fa8..9ded259825d 100644 --- a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java @@ -1,19 +1,37 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; -import com.google.common.collect.Lists; import com.google.common.io.InputSupplier; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; -import java.util.ArrayList; public class TestCaseSupplier implements InputSupplier { private final String s; + public TestCaseSupplier(String s) { - this.s=s; + this.s = s; } @Override diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index c22e44ec83f..440bbc44549 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; import com.google.common.base.Throwables; @@ -8,10 +27,6 @@ import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; import java.io.BufferedReader; -import java.io.IOException; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.ProtocolException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.BlockingQueue; @@ -23,7 +38,7 @@ public class UpdateStream implements Runnable private InputSupplier supplier; private BlockingQueue> queue; private ObjectMapper mapper; - private final TypeReference> typeRef; + private final TypeReference> typeRef; private static final long queueWaitTime = 15L; public UpdateStream(InputSupplier supplier, BlockingQueue> queue) @@ -31,12 +46,13 @@ public class UpdateStream implements Runnable this.supplier = supplier; this.queue = queue; this.mapper = new ObjectMapper(); - this.typeRef= new TypeReference>() + this.typeRef = new TypeReference>() { }; } - private boolean isValid(String s){ + private boolean isValid(String s) + { return !(s.isEmpty()); } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index c18a13e4a2c..0718406c909 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; import com.fasterxml.jackson.annotation.JsonCreator; @@ -15,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index d62f8ddcdd2..d960cfe92ee 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; import com.google.common.io.InputSupplier; diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java index 8b5a3b24d5e..e103fd879b8 100644 --- a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; import com.google.common.io.InputSupplier; @@ -6,7 +25,6 @@ import com.metamx.emitter.EmittingLogger; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.util.List; @@ -23,10 +41,10 @@ public class WebJsonSupplier implements InputSupplier { this.dimensions = dimensions; this.urlString = urlString; - try{ + try { this.url = new URL(urlString); } - catch (Exception e){ + catch (Exception e) { e.printStackTrace(); log.info("Malformed url"); } @@ -36,7 +54,7 @@ public class WebJsonSupplier implements InputSupplier public BufferedReader getInput() throws IOException { URL url = new URL(urlString); - URLConnection connection = url.openConnection(); + URLConnection connection = url.openConnection(); connection.setDoInput(true); //connection.setDoOutput(true); From 80f4ae25b598093eee10d89089407b34d088a0c6 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Thu, 27 Jun 2013 19:00:34 -0700 Subject: [PATCH 09/25] updated with further cleanliness --- examples/pom.xml | 3 +- .../examples/RealtimeStandaloneMain.java | 4 +- .../examples/flights/FlightsConverter.java | 3 +- .../flights/FlightsFirehoseFactory.java | 9 +- .../examples/rand/RandomFirehoseFactory.java | 140 ++++++++++-------- .../TwitterSpritzerFirehoseFactory.java | 42 ++++-- .../examples/webStream/TestCaseSupplier.java | 11 +- .../examples/webStream/UpdateStream.java | 41 +++-- .../webStream/WebFirehoseFactory.java | 79 ++++------ .../webStream/WebFirehoseFactoryTest.java | 5 +- 10 files changed, 165 insertions(+), 172 deletions(-) diff --git a/examples/pom.xml b/examples/pom.xml index 4a1cbc929ac..375d7963e7d 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.metamx.druid druid-examples diff --git a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java b/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java index 54d4a492584..46366eb32df 100644 --- a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java +++ b/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java @@ -13,10 +13,10 @@ import com.metamx.druid.loading.DataSegmentPusher; import com.metamx.druid.log.LogLevelAdjuster; import com.metamx.druid.realtime.RealtimeNode; import com.metamx.druid.realtime.SegmentPublisher; -import druid.examples.webStream.WebFirehoseFactory; import druid.examples.flights.FlightsFirehoseFactory; import druid.examples.rand.RandomFirehoseFactory; import druid.examples.twitter.TwitterSpritzerFirehoseFactory; +import druid.examples.webStream.WebFirehoseFactory; import java.io.File; import java.io.IOException; @@ -53,7 +53,7 @@ public class RealtimeStandaloneMain rn.setDataSegmentPusher(new NoopDataSegmentPusher()); rn.setServerView(new NoopServerView()); rn.setInventoryView(new NoopInventoryView()); - + Runtime.getRuntime().addShutdownHook( new Thread( new Runnable() diff --git a/examples/src/main/java/druid/examples/flights/FlightsConverter.java b/examples/src/main/java/druid/examples/flights/FlightsConverter.java index 53ea6638e60..730eea0bd92 100644 --- a/examples/src/main/java/druid/examples/flights/FlightsConverter.java +++ b/examples/src/main/java/druid/examples/flights/FlightsConverter.java @@ -114,8 +114,7 @@ public class FlightsConverter if (value.equals("NA")) { event.put(metricDimension, 0); - } - else { + } else { event.put(metricDimension, Integer.parseInt(value)); } } diff --git a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java index 66b8de4450c..07eb856425e 100644 --- a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java @@ -80,8 +80,7 @@ public class FlightsFirehoseFactory implements FirehoseFactory try { if (line != null) { return true; - } - else if (in != null) { + } else if (in != null) { line = in.readLine(); if (line == null) { @@ -90,16 +89,14 @@ public class FlightsFirehoseFactory implements FirehoseFactory } return true; - } - else if (files.hasNext()) { + } else if (files.hasNext()) { final File nextFile = files.next(); if (nextFile.getName().endsWith(".gz")) { in = new BufferedReader( new InputStreamReader(new GZIPInputStream(new FileInputStream(nextFile)), Charsets.UTF_8) ); - } - else { + } else { in = new BufferedReader(new FileReader(nextFile)); } return hasMore(); diff --git a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java index 71a0c1fbb92..54010e822ed 100644 --- a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java @@ -35,81 +35,89 @@ import static java.lang.Thread.sleep; * the moment an event is delivered.) * Values are offset by adding the modulus of the token number to the random number * so that token values have distinct, non-overlapping ranges. - * + *

*

* Example spec file: * 
- [{
-   "schema" : { "dataSource":"randseq",
-                "aggregators":[ {"type":"count", "name":"events"},
- 	       		                    {"type":"doubleSum","name":"outColumn","fieldName":"inColumn"} ],
-                "indexGranularity":"minute",
- 	              "shardSpec" : { "type": "none" } },
-   "config" : { "maxRowsInMemory" : 50000,
-                "intermediatePersistPeriod" : "PT2m" },
-
-   "firehose" : { "type" : "rand",
-                  "sleepUsec": 100000,
-                  "maxGeneratedRows" : 5000000,
-                  "seed" : 0,
-                  "nTokens" : 19,
-                  "nPerSleep" : 3
-                 },
-
-   "plumber" : { "type" : "realtime",
-                 "windowPeriod" : "PT5m",
-                 "segmentGranularity":"hour",
-                 "basePersistDirectory" : "/tmp/realtime/basePersist" }
- }]
- *
+ * [{ + * "schema" : { "dataSource":"randseq", + * "aggregators":[ {"type":"count", "name":"events"}, + * {"type":"doubleSum","name":"outColumn","fieldName":"inColumn"} ], + * "indexGranularity":"minute", + * "shardSpec" : { "type": "none" } }, + * "config" : { "maxRowsInMemory" : 50000, + * "intermediatePersistPeriod" : "PT2m" }, * + * "firehose" : { "type" : "rand", + * "sleepUsec": 100000, + * "maxGeneratedRows" : 5000000, + * "seed" : 0, + * "nTokens" : 19, + * "nPerSleep" : 3 + * }, + * + * "plumber" : { "type" : "realtime", + * "windowPeriod" : "PT5m", + * "segmentGranularity":"hour", + * "basePersistDirectory" : "/tmp/realtime/basePersist" } + * }] + * + *

* Example query using POST to /druid/v2/ (where UTC date and time MUST include the current hour): * 

- {
-     "queryType": "groupBy",
-     "dataSource": "randSeq",
-     "granularity": "all",
-     "dimensions": [],
-     "aggregations":[
-     { "type": "count", "name": "rows"},
-     { "type": "doubleSum", "fieldName": "events", "name": "e"},
-     { "type": "doubleSum", "fieldName": "outColumn", "name": "randomNumberSum"}
-     ],
-     "postAggregations":[
-     {  "type":"arithmetic",
-        "name":"avg_random",
-        "fn":"/",
-        "fields":[ {"type":"fieldAccess","name":"randomNumberSum","fieldName":"randomNumberSum"},
-                   {"type":"fieldAccess","name":"rows","fieldName":"rows"} ]}
-     ],
-     "intervals":["2012-10-01T00:00/2020-01-01T00"]
- }
+ * {
+ * "queryType": "groupBy",
+ * "dataSource": "randSeq",
+ * "granularity": "all",
+ * "dimensions": [],
+ * "aggregations":[
+ * { "type": "count", "name": "rows"},
+ * { "type": "doubleSum", "fieldName": "events", "name": "e"},
+ * { "type": "doubleSum", "fieldName": "outColumn", "name": "randomNumberSum"}
+ * ],
+ * "postAggregations":[
+ * {  "type":"arithmetic",
+ * "name":"avg_random",
+ * "fn":"/",
+ * "fields":[ {"type":"fieldAccess","name":"randomNumberSum","fieldName":"randomNumberSum"},
+ * {"type":"fieldAccess","name":"rows","fieldName":"rows"} ]}
+ * ],
+ * "intervals":["2012-10-01T00:00/2020-01-01T00"]
+ * }
  *
*/ @JsonTypeName("rand") public class RandomFirehoseFactory implements FirehoseFactory { private static final Logger log = new Logger(RandomFirehoseFactory.class); - /** msec to sleep before generating a new row; if this and delayNsec are 0, then go as fast as possible. + /** + * msec to sleep before generating a new row; if this and delayNsec are 0, then go as fast as possible. * json param sleepUsec (microseconds) is used to initialize this. */ private final long delayMsec; - /** nsec to sleep before generating a new row; if this and delayMsec are 0, then go as fast as possible. + /** + * nsec to sleep before generating a new row; if this and delayMsec are 0, then go as fast as possible. * json param sleepUsec (microseconds) is used to initialize this. - */ + */ private final int delayNsec; - /** max rows to generate, -1 is infinite, 0 means nothing is generated; use this to prevent - * infinite space consumption or to see what happens when a Firehose stops delivering - * values, or to have hasMore() return false. + /** + * max rows to generate, -1 is infinite, 0 means nothing is generated; use this to prevent + * infinite space consumption or to see what happens when a Firehose stops delivering + * values, or to have hasMore() return false. */ private final long maxGeneratedRows; - /** seed for random number generator; if 0, then no seed is used. */ + /** + * seed for random number generator; if 0, then no seed is used. + */ private final long seed; - /** number of tokens to randomly associate with values (no heap limits). This can be used to + /** + * number of tokens to randomly associate with values (no heap limits). This can be used to * stress test the number of tokens. */ private final int nTokens; - /** Number of token events per sleep interval. */ + /** + * Number of token events per sleep interval. + */ private final int nPerSleep; @JsonCreator @@ -124,29 +132,30 @@ public class RandomFirehoseFactory implements FirehoseFactory long nsec = (sleepUsec > 0) ? sleepUsec * 1000L : 0; long msec = nsec / 1000000L; this.delayMsec = msec; - this.delayNsec = (int)(nsec - (msec * 1000000L)); + this.delayNsec = (int) (nsec - (msec * 1000000L)); this.maxGeneratedRows = maxGeneratedRows; this.seed = seed; this.nTokens = nTokens; this.nPerSleep = nPerSleep; if (nTokens <= 0) { - log.warn("nTokens parameter " + nTokens +" ignored; must be greater than or equal to 1"); + log.warn("nTokens parameter " + nTokens + " ignored; must be greater than or equal to 1"); nTokens = 1; } if (nPerSleep <= 0) { - log.warn("nPerSleep parameter " + nPerSleep +" ignored; must be greater than or equal to 1"); + log.warn("nPerSleep parameter " + nPerSleep + " ignored; must be greater than or equal to 1"); nPerSleep = 1; } log.info("maxGeneratedRows=" + maxGeneratedRows); - log.info("seed=" + ( (seed == 0L) ? "random value" : seed )); + log.info("seed=" + ((seed == 0L) ? "random value" : seed)); log.info("nTokens=" + nTokens); log.info("nPerSleep=" + nPerSleep); - double dmsec = (double)delayMsec + ((double)this.delayNsec)/1000000.; + double dmsec = (double) delayMsec + ((double) this.delayNsec) / 1000000.; if (dmsec > 0.0) { log.info("sleep period=" + dmsec + "msec"); - log.info("approximate max rate of record generation=" + (nPerSleep * 1000./dmsec) + "/sec" + - " or " + (60. * nPerSleep * 1000./dmsec) + "/minute" - ); + log.info( + "approximate max rate of record generation=" + (nPerSleep * 1000. / dmsec) + "/sec" + + " or " + (60. * nPerSleep * 1000. / dmsec) + "/minute" + ); } else { log.info("sleep period= NONE"); log.info("approximate max rate of record generation= as fast as possible"); @@ -170,7 +179,7 @@ public class RandomFirehoseFactory implements FirehoseFactory @Override public boolean hasMore() { - if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows) { + if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows) { return waitIfmaxGeneratedRows; } else { return true; // there are always more random numbers @@ -184,7 +193,7 @@ public class RandomFirehoseFactory implements FirehoseFactory final long nth = (rowCount % nTokens) + 1; long sleepMsec = delayMsec; // all done? - if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows && waitIfmaxGeneratedRows) { + if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows && waitIfmaxGeneratedRows) { // sleep a long time instead of terminating sleepMsec = 2000000000L; } @@ -193,7 +202,8 @@ public class RandomFirehoseFactory implements FirehoseFactory if (modulus == 0) { sleep(sleepMsec, delayNsec); } - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException("InterruptedException"); } } @@ -202,7 +212,7 @@ public class RandomFirehoseFactory implements FirehoseFactory } final Map theMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - theMap.put("inColumn", anotherRand((int)nth)); + theMap.put("inColumn", anotherRand((int) nth)); theMap.put("target", ("a" + nth)); return new MapBasedInputRow(System.currentTimeMillis(), dimensions, theMap); } @@ -210,7 +220,7 @@ public class RandomFirehoseFactory implements FirehoseFactory private Float anotherRand(int scale) { double f = rand.nextDouble(); // [0.0,1.0] - return new Float(f + (double)scale); + return new Float(f + (double) scale); } @Override diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java index e87b5e44de2..4b588ec826c 100644 --- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java @@ -11,13 +11,13 @@ import com.metamx.druid.realtime.firehose.Firehose; import com.metamx.druid.realtime.firehose.FirehoseFactory; import twitter4j.ConnectionLifeCycleListener; import twitter4j.HashtagEntity; +import twitter4j.StallWarning; import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; import twitter4j.TwitterStream; import twitter4j.TwitterStreamFactory; import twitter4j.User; -import twitter4j.StallWarning; import java.io.IOException; import java.util.Arrays; @@ -49,21 +49,23 @@ import static java.lang.Thread.sleep; * is UTC): * 
  *
- * - * + *

+ *

* Notes on twitter.com HTTP (REST) API: v1.0 will be disabled around 2013-03 so v1.1 should be used; * twitter4j 3.0 (not yet released) will support the v1.1 api. * Specifically, we should be using https://stream.twitter.com/1.1/statuses/sample.json * See: http://jira.twitter4j.org/browse/TFJ-186 + *

+ * Notes on JSON parsing: as of twitter4j 2.2.x, the json parser has some bugs (ex: Status.toString() + * can have number format exceptions), so it might be necessary to extract raw json and process it + * separately. If so, set twitter4.jsonStoreEnabled=true and look at DataObjectFactory#getRawJSON(); + * com.fasterxml.jackson.databind.ObjectMapper should be used to parse. * - * Notes on JSON parsing: as of twitter4j 2.2.x, the json parser has some bugs (ex: Status.toString() - * can have number format exceptions), so it might be necessary to extract raw json and process it - * separately. If so, set twitter4.jsonStoreEnabled=true and look at DataObjectFactory#getRawJSON(); - * com.fasterxml.jackson.databind.ObjectMapper should be used to parse. * @author pbaclace */ @JsonTypeName("twitzer") -public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { +public class TwitterSpritzerFirehoseFactory implements FirehoseFactory +{ private static final Logger log = new Logger(TwitterSpritzerFirehoseFactory.class); /** * max events to receive, -1 is infinite, 0 means nothing is delivered; use this to prevent @@ -94,7 +96,8 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { @Override public Firehose connect() throws IOException { - final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { + final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() + { @Override public void onConnect() { @@ -134,7 +137,8 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); - statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j + statusListener = new StatusListener() + { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { @@ -147,7 +151,8 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { if (!success) { log.warn("queue too slow!"); } - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @@ -179,7 +184,8 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { } @Override - public void onStallWarning(StallWarning warning) { + public void onStallWarning(StallWarning warning) + { System.out.println("Got stall warning:" + warning); } }; @@ -188,9 +194,11 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); - return new Firehose() { + return new Firehose() + { - private final Runnable doNothingRunnable = new Runnable() { + private final Runnable doNothingRunnable = new Runnable() + { public void run() { } @@ -240,7 +248,8 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { @@ -254,7 +263,8 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { Status status; try { status = queue.take(); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } diff --git a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java index 9ded259825d..abce656c97f 100644 --- a/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/TestCaseSupplier.java @@ -27,19 +27,16 @@ import java.io.StringReader; public class TestCaseSupplier implements InputSupplier { - private final String s; + private final String testString; - public TestCaseSupplier(String s) + public TestCaseSupplier(String testString) { - this.s = s; + this.testString = testString; } @Override public BufferedReader getInput() throws IOException { - StringBuilder buffer = new StringBuilder(); - buffer.append(s); - BufferedReader br = new BufferedReader(new StringReader(buffer.toString())); - return br; + return new BufferedReader(new StringReader(testString)); } } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 440bbc44549..626bd2f3b19 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -19,12 +19,11 @@ package druid.examples.webStream; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.io.InputSupplier; import com.metamx.emitter.EmittingLogger; -import org.codehaus.jackson.JsonParseException; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; import java.io.BufferedReader; import java.util.HashMap; @@ -35,17 +34,21 @@ import java.util.concurrent.TimeUnit; public class UpdateStream implements Runnable { private static final EmittingLogger log = new EmittingLogger(UpdateStream.class); - private InputSupplier supplier; - private BlockingQueue> queue; - private ObjectMapper mapper; - private final TypeReference> typeRef; private static final long queueWaitTime = 15L; + private final TypeReference> typeRef; + private final InputSupplier supplier; + private final BlockingQueue> queue; + private final ObjectMapper mapper; - public UpdateStream(InputSupplier supplier, BlockingQueue> queue) + public UpdateStream( + InputSupplier supplier, + BlockingQueue> queue, + ObjectMapper mapper + ) { this.supplier = supplier; this.queue = queue; - this.mapper = new ObjectMapper(); + this.mapper = mapper; this.typeRef = new TypeReference>() { }; @@ -60,28 +63,18 @@ public class UpdateStream implements Runnable public void run() throws RuntimeException { try { - BufferedReader reader = (BufferedReader) supplier.getInput(); + BufferedReader reader = supplier.getInput(); String line; while ((line = reader.readLine()) != null) { if (isValid(line)) { - try { - HashMap map = mapper.readValue(line, typeRef); - queue.offer(map, queueWaitTime, TimeUnit.SECONDS); - log.info("Successfully added to queue"); - } - catch (JsonParseException e) { - log.info("Invalid JSON Stream. Please check if the url returns a proper JSON stream."); - Throwables.propagate(e); - } - catch (Exception e) { - Throwables.propagate(e); - } + HashMap map = mapper.readValue(line, typeRef); + queue.offer(map, queueWaitTime, TimeUnit.SECONDS); + log.info("Successfully added to queue"); } } } catch (Exception e) { - e.printStackTrace(); - Throwables.propagate(e); + throw Throwables.propagate(e); } } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 0718406c909..949e948176d 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -22,14 +22,18 @@ package druid.examples.webStream; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Throwables; +import com.google.common.collect.Maps; +import com.metamx.common.parsers.TimestampParser; import com.metamx.druid.input.InputRow; import com.metamx.druid.input.MapBasedInputRow; +import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.druid.realtime.firehose.Firehose; import com.metamx.druid.realtime.firehose.FirehoseFactory; import com.metamx.emitter.EmittingLogger; +import org.joda.time.DateTime; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; @@ -41,11 +45,11 @@ import java.util.concurrent.Executors; public class WebFirehoseFactory implements FirehoseFactory { private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); + private static final int QUEUE_SIZE = 2000; private final String url; private final List dimensions; private final String timeDimension; private final List renamedDimensions; - private static final int QUEUE_SIZE = 2000; @JsonCreator @@ -53,13 +57,13 @@ public class WebFirehoseFactory implements FirehoseFactory @JsonProperty("url") String url, @JsonProperty("dimensions") List dimensions, @JsonProperty("renamedDimensions") List renamedDimensions, - @JsonProperty("timeDimension") String s + @JsonProperty("timeDimension") String timeDimension ) { this.url = url; this.dimensions = dimensions; this.renamedDimensions = renamedDimensions; - this.timeDimension = s; + this.timeDimension = timeDimension; } @Override @@ -67,17 +71,7 @@ public class WebFirehoseFactory implements FirehoseFactory { final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions, url), queue); -// Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() -// { -// public void uncaughtException(Thread th, Throwable ex) -// { -// log.info("Uncaught exception: " + ex); -// } -// }; -// final Thread streamReader = new Thread(updateStream); -// streamReader.setUncaughtExceptionHandler(h); -// streamReader.start(); + Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions, url), queue, new DefaultObjectMapper()); final ExecutorService service = Executors.newSingleThreadExecutor(); service.submit(updateStream); @@ -95,53 +89,45 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public InputRow nextRow() { - if (Thread.currentThread().isInterrupted()) { - throw new RuntimeException("Interrupted, time to stop"); - } - Map update; try { - update = queue.take(); + Map processedMap = processMap(queue.take()); + DateTime date = TimestampParser.createTimestampParser("auto") + .apply(processedMap.get(timeDimension).toString()); + long seconds = (long) date.getMillis(); + //the parser doesn't check for posix. Only expects iso or millis. This checks for posix + if (new DateTime(seconds * 1000).getYear() == new DateTime().getYear()) { + seconds = (long) date.getMillis() * 1000; + } + return new MapBasedInputRow( + seconds, + renamedDimensions, + processedMap + ); } - catch (InterruptedException e) { - throw new RuntimeException("InterrutpedException", e); + catch (Exception e) { + throw Throwables.propagate(e); } - Map processedMap = processMap(update); - return new MapBasedInputRow( - ((Integer) processedMap.get(timeDimension)).longValue() * 1000, - renamedDimensions, - processedMap - ); } private Map renameKeys(Map update) { - Map renamedMap = new HashMap(); - int iter = 0; - while (iter < dimensions.size()) { + Map renamedMap = Maps.newHashMap(); + for (int iter = 0; iter < dimensions.size(); iter++) { + if (update.get(dimensions.get(iter)) == null) { + update.put(dimensions.get(iter), null); + } Object obj = update.get(dimensions.get(iter)); renamedMap.put(renamedDimensions.get(iter), obj); - iter++; + } + if (renamedMap.get(timeDimension) == null) { + renamedMap.put(timeDimension, System.currentTimeMillis()); } return renamedMap; } - private void processNullDimensions(Map map) - { - for (String key : renamedDimensions) { - if (map.get(key) == null) { - if (key.equals(timeDimension)) { - map.put(key, new Integer((int) System.currentTimeMillis() / 1000)); - } else { - map.put(key, null); - } - } - } - } - private Map processMap(Map map) { Map renamedMap = renameKeys(map); - processNullDimensions(renamedMap); return renamedMap; } @@ -156,7 +142,6 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public void close() throws IOException { - log.info("CLOSING!!!"); service.shutdown(); } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index d960cfe92ee..d14b7dd94e8 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -21,6 +21,7 @@ package druid.examples.webStream; import com.google.common.io.InputSupplier; import com.metamx.druid.input.InputRow; +import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.druid.realtime.firehose.Firehose; import com.metamx.druid.realtime.firehose.FirehoseFactory; import org.testng.annotations.Test; @@ -128,7 +129,7 @@ public class WebFirehoseFactoryTest InputSupplier testCaseSupplier = new TestCaseSupplier( "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" ); - UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue); + UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue, new DefaultObjectMapper()); Thread t = new Thread(updateStream); t.start(); Map expectedAnswer = new HashMap(); @@ -188,7 +189,7 @@ public class WebFirehoseFactoryTest InputSupplier testCaseSupplier = new TestCaseSupplier( "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"kw\": \"spcnws\", \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" ); - UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue); + UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue, new DefaultObjectMapper()); Thread t = new Thread(updateStream); t.start(); InputRow row = webbieHose.nextRow(); From c15f18c34358154e58e1a5931958a3d94ed2e379 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Thu, 27 Jun 2013 19:03:25 -0700 Subject: [PATCH 10/25] removed NoopRunnable --- .../com/metamx/druid/guava/Runnables.java | 6 ++++ .../examples/webStream/NoopRunnable.java | 28 ------------------- .../webStream/WebFirehoseFactory.java | 3 +- 3 files changed, 8 insertions(+), 29 deletions(-) delete mode 100644 examples/src/main/java/druid/examples/webStream/NoopRunnable.java diff --git a/common/src/main/java/com/metamx/druid/guava/Runnables.java b/common/src/main/java/com/metamx/druid/guava/Runnables.java index 8c41886d349..118d4938a9d 100644 --- a/common/src/main/java/com/metamx/druid/guava/Runnables.java +++ b/common/src/main/java/com/metamx/druid/guava/Runnables.java @@ -34,4 +34,10 @@ public class Runnables } }; } + + public static Runnable getNoopRunnable(){ + return new Runnable(){ + public void run(){} + }; + } } diff --git a/examples/src/main/java/druid/examples/webStream/NoopRunnable.java b/examples/src/main/java/druid/examples/webStream/NoopRunnable.java deleted file mode 100644 index 6ccd4ba1584..00000000000 --- a/examples/src/main/java/druid/examples/webStream/NoopRunnable.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package druid.examples.webStream; - -public class NoopRunnable implements Runnable -{ - @Override - public void run() - { - } -} diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 949e948176d..8cad52403b5 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Throwables; import com.google.common.collect.Maps; import com.metamx.common.parsers.TimestampParser; +import com.metamx.druid.guava.Runnables; import com.metamx.druid.input.InputRow; import com.metamx.druid.input.MapBasedInputRow; import com.metamx.druid.jackson.DefaultObjectMapper; @@ -77,7 +78,7 @@ public class WebFirehoseFactory implements FirehoseFactory return new Firehose() { - private final Runnable doNothingRunnable = new NoopRunnable(); + private final Runnable doNothingRunnable = Runnables.getNoopRunnable(); @Override public boolean hasMore() From 6f6bf1996c38c73d195509fd7a06c8188fcbff6c Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Thu, 27 Jun 2013 19:09:35 -0700 Subject: [PATCH 11/25] log.info changed to log.debug --- .../main/java/druid/examples/webStream/UpdateStream.java | 2 +- .../java/druid/examples/webStream/WebFirehoseFactory.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 626bd2f3b19..3cb7e5ef27a 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -69,7 +69,7 @@ public class UpdateStream implements Runnable if (isValid(line)) { HashMap map = mapper.readValue(line, typeRef); queue.offer(map, queueWaitTime, TimeUnit.SECONDS); - log.info("Successfully added to queue"); + log.debug("Successfully added to queue"); } } } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 8cad52403b5..cebf2b7913c 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -114,11 +114,10 @@ public class WebFirehoseFactory implements FirehoseFactory { Map renamedMap = Maps.newHashMap(); for (int iter = 0; iter < dimensions.size(); iter++) { - if (update.get(dimensions.get(iter)) == null) { - update.put(dimensions.get(iter), null); + if (update.get(dimensions.get(iter)) != null) { + Object obj = update.get(dimensions.get(iter)); + renamedMap.put(renamedDimensions.get(iter), obj); } - Object obj = update.get(dimensions.get(iter)); - renamedMap.put(renamedDimensions.get(iter), obj); } if (renamedMap.get(timeDimension) == null) { renamedMap.put(timeDimension, System.currentTimeMillis()); From adc00e6576c8b7e1bcf89a3054b452ab9312c153 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 12:13:25 -0700 Subject: [PATCH 12/25] fixed with Erics suggestions --- .../examples/flights/FlightsConverter.java | 3 +- .../flights/FlightsFirehoseFactory.java | 9 +- .../examples/rand/RandomFirehoseFactory.java | 140 ++++---- .../TwitterSpritzerFirehoseFactory.java | 42 +-- .../examples/webStream/UpdateStream.java | 37 +- .../webStream/WebFirehoseFactory.java | 61 ++-- .../webStream/WebFirehoseFactoryTest.java | 315 +++++++++--------- .../examples/webStream/WebJsonSupplier.java | 5 +- 8 files changed, 311 insertions(+), 301 deletions(-) diff --git a/examples/src/main/java/druid/examples/flights/FlightsConverter.java b/examples/src/main/java/druid/examples/flights/FlightsConverter.java index 730eea0bd92..53ea6638e60 100644 --- a/examples/src/main/java/druid/examples/flights/FlightsConverter.java +++ b/examples/src/main/java/druid/examples/flights/FlightsConverter.java @@ -114,7 +114,8 @@ public class FlightsConverter if (value.equals("NA")) { event.put(metricDimension, 0); - } else { + } + else { event.put(metricDimension, Integer.parseInt(value)); } } diff --git a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java index 07eb856425e..66b8de4450c 100644 --- a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java @@ -80,7 +80,8 @@ public class FlightsFirehoseFactory implements FirehoseFactory try { if (line != null) { return true; - } else if (in != null) { + } + else if (in != null) { line = in.readLine(); if (line == null) { @@ -89,14 +90,16 @@ public class FlightsFirehoseFactory implements FirehoseFactory } return true; - } else if (files.hasNext()) { + } + else if (files.hasNext()) { final File nextFile = files.next(); if (nextFile.getName().endsWith(".gz")) { in = new BufferedReader( new InputStreamReader(new GZIPInputStream(new FileInputStream(nextFile)), Charsets.UTF_8) ); - } else { + } + else { in = new BufferedReader(new FileReader(nextFile)); } return hasMore(); diff --git a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java index 54010e822ed..71a0c1fbb92 100644 --- a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java @@ -35,89 +35,81 @@ import static java.lang.Thread.sleep; * the moment an event is delivered.) * Values are offset by adding the modulus of the token number to the random number * so that token values have distinct, non-overlapping ranges. - *

+ * *

* Example spec file: * 
- * [{
- * "schema" : { "dataSource":"randseq",
- * "aggregators":[ {"type":"count", "name":"events"},
- * {"type":"doubleSum","name":"outColumn","fieldName":"inColumn"} ],
- * "indexGranularity":"minute",
- * "shardSpec" : { "type": "none" } },
- * "config" : { "maxRowsInMemory" : 50000,
- * "intermediatePersistPeriod" : "PT2m" },
- *
- * "firehose" : { "type" : "rand",
- * "sleepUsec": 100000,
- * "maxGeneratedRows" : 5000000,
- * "seed" : 0,
- * "nTokens" : 19,
- * "nPerSleep" : 3
- * },
- *
- * "plumber" : { "type" : "realtime",
- * "windowPeriod" : "PT5m",
- * "segmentGranularity":"hour",
- * "basePersistDirectory" : "/tmp/realtime/basePersist" }
- * }]
+ [{
+   "schema" : { "dataSource":"randseq",
+                "aggregators":[ {"type":"count", "name":"events"},
+ 	       		                    {"type":"doubleSum","name":"outColumn","fieldName":"inColumn"} ],
+                "indexGranularity":"minute",
+ 	              "shardSpec" : { "type": "none" } },
+   "config" : { "maxRowsInMemory" : 50000,
+                "intermediatePersistPeriod" : "PT2m" },
+
+   "firehose" : { "type" : "rand",
+                  "sleepUsec": 100000,
+                  "maxGeneratedRows" : 5000000,
+                  "seed" : 0,
+                  "nTokens" : 19,
+                  "nPerSleep" : 3
+                 },
+
+   "plumber" : { "type" : "realtime",
+                 "windowPeriod" : "PT5m",
+                 "segmentGranularity":"hour",
+                 "basePersistDirectory" : "/tmp/realtime/basePersist" }
+ }]
  *
- *

+ * * Example query using POST to /druid/v2/ (where UTC date and time MUST include the current hour): * 

- * {
- * "queryType": "groupBy",
- * "dataSource": "randSeq",
- * "granularity": "all",
- * "dimensions": [],
- * "aggregations":[
- * { "type": "count", "name": "rows"},
- * { "type": "doubleSum", "fieldName": "events", "name": "e"},
- * { "type": "doubleSum", "fieldName": "outColumn", "name": "randomNumberSum"}
- * ],
- * "postAggregations":[
- * {  "type":"arithmetic",
- * "name":"avg_random",
- * "fn":"/",
- * "fields":[ {"type":"fieldAccess","name":"randomNumberSum","fieldName":"randomNumberSum"},
- * {"type":"fieldAccess","name":"rows","fieldName":"rows"} ]}
- * ],
- * "intervals":["2012-10-01T00:00/2020-01-01T00"]
- * }
+ {
+     "queryType": "groupBy",
+     "dataSource": "randSeq",
+     "granularity": "all",
+     "dimensions": [],
+     "aggregations":[
+     { "type": "count", "name": "rows"},
+     { "type": "doubleSum", "fieldName": "events", "name": "e"},
+     { "type": "doubleSum", "fieldName": "outColumn", "name": "randomNumberSum"}
+     ],
+     "postAggregations":[
+     {  "type":"arithmetic",
+        "name":"avg_random",
+        "fn":"/",
+        "fields":[ {"type":"fieldAccess","name":"randomNumberSum","fieldName":"randomNumberSum"},
+                   {"type":"fieldAccess","name":"rows","fieldName":"rows"} ]}
+     ],
+     "intervals":["2012-10-01T00:00/2020-01-01T00"]
+ }
  *
*/ @JsonTypeName("rand") public class RandomFirehoseFactory implements FirehoseFactory { private static final Logger log = new Logger(RandomFirehoseFactory.class); - /** - * msec to sleep before generating a new row; if this and delayNsec are 0, then go as fast as possible. + /** msec to sleep before generating a new row; if this and delayNsec are 0, then go as fast as possible. * json param sleepUsec (microseconds) is used to initialize this. */ private final long delayMsec; - /** - * nsec to sleep before generating a new row; if this and delayMsec are 0, then go as fast as possible. + /** nsec to sleep before generating a new row; if this and delayMsec are 0, then go as fast as possible. * json param sleepUsec (microseconds) is used to initialize this. - */ + */ private final int delayNsec; - /** - * max rows to generate, -1 is infinite, 0 means nothing is generated; use this to prevent - * infinite space consumption or to see what happens when a Firehose stops delivering - * values, or to have hasMore() return false. + /** max rows to generate, -1 is infinite, 0 means nothing is generated; use this to prevent + * infinite space consumption or to see what happens when a Firehose stops delivering + * values, or to have hasMore() return false. */ private final long maxGeneratedRows; - /** - * seed for random number generator; if 0, then no seed is used. - */ + /** seed for random number generator; if 0, then no seed is used. */ private final long seed; - /** - * number of tokens to randomly associate with values (no heap limits). This can be used to + /** number of tokens to randomly associate with values (no heap limits). This can be used to * stress test the number of tokens. */ private final int nTokens; - /** - * Number of token events per sleep interval. - */ + /** Number of token events per sleep interval. */ private final int nPerSleep; @JsonCreator @@ -132,30 +124,29 @@ public class RandomFirehoseFactory implements FirehoseFactory long nsec = (sleepUsec > 0) ? sleepUsec * 1000L : 0; long msec = nsec / 1000000L; this.delayMsec = msec; - this.delayNsec = (int) (nsec - (msec * 1000000L)); + this.delayNsec = (int)(nsec - (msec * 1000000L)); this.maxGeneratedRows = maxGeneratedRows; this.seed = seed; this.nTokens = nTokens; this.nPerSleep = nPerSleep; if (nTokens <= 0) { - log.warn("nTokens parameter " + nTokens + " ignored; must be greater than or equal to 1"); + log.warn("nTokens parameter " + nTokens +" ignored; must be greater than or equal to 1"); nTokens = 1; } if (nPerSleep <= 0) { - log.warn("nPerSleep parameter " + nPerSleep + " ignored; must be greater than or equal to 1"); + log.warn("nPerSleep parameter " + nPerSleep +" ignored; must be greater than or equal to 1"); nPerSleep = 1; } log.info("maxGeneratedRows=" + maxGeneratedRows); - log.info("seed=" + ((seed == 0L) ? "random value" : seed)); + log.info("seed=" + ( (seed == 0L) ? "random value" : seed )); log.info("nTokens=" + nTokens); log.info("nPerSleep=" + nPerSleep); - double dmsec = (double) delayMsec + ((double) this.delayNsec) / 1000000.; + double dmsec = (double)delayMsec + ((double)this.delayNsec)/1000000.; if (dmsec > 0.0) { log.info("sleep period=" + dmsec + "msec"); - log.info( - "approximate max rate of record generation=" + (nPerSleep * 1000. / dmsec) + "/sec" + - " or " + (60. * nPerSleep * 1000. / dmsec) + "/minute" - ); + log.info("approximate max rate of record generation=" + (nPerSleep * 1000./dmsec) + "/sec" + + " or " + (60. * nPerSleep * 1000./dmsec) + "/minute" + ); } else { log.info("sleep period= NONE"); log.info("approximate max rate of record generation= as fast as possible"); @@ -179,7 +170,7 @@ public class RandomFirehoseFactory implements FirehoseFactory @Override public boolean hasMore() { - if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows) { + if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows) { return waitIfmaxGeneratedRows; } else { return true; // there are always more random numbers @@ -193,7 +184,7 @@ public class RandomFirehoseFactory implements FirehoseFactory final long nth = (rowCount % nTokens) + 1; long sleepMsec = delayMsec; // all done? - if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows && waitIfmaxGeneratedRows) { + if (maxGeneratedRows >= 0 && rowCount >= maxGeneratedRows && waitIfmaxGeneratedRows) { // sleep a long time instead of terminating sleepMsec = 2000000000L; } @@ -202,8 +193,7 @@ public class RandomFirehoseFactory implements FirehoseFactory if (modulus == 0) { sleep(sleepMsec, delayNsec); } - } - catch (InterruptedException e) { + } catch (InterruptedException e) { throw new RuntimeException("InterruptedException"); } } @@ -212,7 +202,7 @@ public class RandomFirehoseFactory implements FirehoseFactory } final Map theMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - theMap.put("inColumn", anotherRand((int) nth)); + theMap.put("inColumn", anotherRand((int)nth)); theMap.put("target", ("a" + nth)); return new MapBasedInputRow(System.currentTimeMillis(), dimensions, theMap); } @@ -220,7 +210,7 @@ public class RandomFirehoseFactory implements FirehoseFactory private Float anotherRand(int scale) { double f = rand.nextDouble(); // [0.0,1.0] - return new Float(f + (double) scale); + return new Float(f + (double)scale); } @Override diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java index 4b588ec826c..e87b5e44de2 100644 --- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java @@ -11,13 +11,13 @@ import com.metamx.druid.realtime.firehose.Firehose; import com.metamx.druid.realtime.firehose.FirehoseFactory; import twitter4j.ConnectionLifeCycleListener; import twitter4j.HashtagEntity; -import twitter4j.StallWarning; import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; import twitter4j.TwitterStream; import twitter4j.TwitterStreamFactory; import twitter4j.User; +import twitter4j.StallWarning; import java.io.IOException; import java.util.Arrays; @@ -49,23 +49,21 @@ import static java.lang.Thread.sleep; * is UTC): * 
  *
- *

- *

+ * + * * Notes on twitter.com HTTP (REST) API: v1.0 will be disabled around 2013-03 so v1.1 should be used; * twitter4j 3.0 (not yet released) will support the v1.1 api. * Specifically, we should be using https://stream.twitter.com/1.1/statuses/sample.json * See: http://jira.twitter4j.org/browse/TFJ-186 - *

- * Notes on JSON parsing: as of twitter4j 2.2.x, the json parser has some bugs (ex: Status.toString() - * can have number format exceptions), so it might be necessary to extract raw json and process it - * separately. If so, set twitter4.jsonStoreEnabled=true and look at DataObjectFactory#getRawJSON(); - * com.fasterxml.jackson.databind.ObjectMapper should be used to parse. * + * Notes on JSON parsing: as of twitter4j 2.2.x, the json parser has some bugs (ex: Status.toString() + * can have number format exceptions), so it might be necessary to extract raw json and process it + * separately. If so, set twitter4.jsonStoreEnabled=true and look at DataObjectFactory#getRawJSON(); + * com.fasterxml.jackson.databind.ObjectMapper should be used to parse. * @author pbaclace */ @JsonTypeName("twitzer") -public class TwitterSpritzerFirehoseFactory implements FirehoseFactory -{ +public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { private static final Logger log = new Logger(TwitterSpritzerFirehoseFactory.class); /** * max events to receive, -1 is infinite, 0 means nothing is delivered; use this to prevent @@ -96,8 +94,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory @Override public Firehose connect() throws IOException { - final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() - { + final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override public void onConnect() { @@ -137,8 +134,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); - statusListener = new StatusListener() - { // This is what really gets called to deliver stuff from twitter4j + statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { @@ -151,8 +147,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory if (!success) { log.warn("queue too slow!"); } - } - catch (InterruptedException e) { + } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @@ -184,8 +179,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory } @Override - public void onStallWarning(StallWarning warning) - { + public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; @@ -194,11 +188,9 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); - return new Firehose() - { + return new Firehose() { - private final Runnable doNothingRunnable = new Runnable() - { + private final Runnable doNothingRunnable = new Runnable() { public void run() { } @@ -248,8 +240,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { @@ -263,8 +254,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory Status status; try { status = queue.take(); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 3cb7e5ef27a..57d8fc67311 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -22,10 +22,12 @@ package druid.examples.webStream; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; +import com.google.common.collect.Maps; import com.google.common.io.InputSupplier; import com.metamx.emitter.EmittingLogger; import java.io.BufferedReader; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.concurrent.BlockingQueue; @@ -39,11 +41,17 @@ public class UpdateStream implements Runnable private final InputSupplier supplier; private final BlockingQueue> queue; private final ObjectMapper mapper; + private final ArrayList dimensions; + private final ArrayList renamedDimensions; + private final String timeDimension; public UpdateStream( InputSupplier supplier, BlockingQueue> queue, - ObjectMapper mapper + ObjectMapper mapper, + ArrayList dimensions, + ArrayList renamedDimensions, + String timeDimension ) { this.supplier = supplier; @@ -52,6 +60,9 @@ public class UpdateStream implements Runnable this.typeRef = new TypeReference>() { }; + this.timeDimension = timeDimension; + this.dimensions = dimensions; + this.renamedDimensions = renamedDimensions; } private boolean isValid(String s) @@ -60,7 +71,7 @@ public class UpdateStream implements Runnable } @Override - public void run() throws RuntimeException + public void run() { try { BufferedReader reader = supplier.getInput(); @@ -68,8 +79,13 @@ public class UpdateStream implements Runnable while ((line = reader.readLine()) != null) { if (isValid(line)) { HashMap map = mapper.readValue(line, typeRef); - queue.offer(map, queueWaitTime, TimeUnit.SECONDS); - log.debug("Successfully added to queue"); + if (map.get(timeDimension) != null) { + Map renamedMap = renameKeys(map); + queue.offer(renamedMap, queueWaitTime, TimeUnit.SECONDS); + log.debug("Successfully added to queue"); + } else { + log.debug("missing timestamp"); + } } } } @@ -78,4 +94,17 @@ public class UpdateStream implements Runnable } } + + private Map renameKeys(Map update) + { + Map renamedMap = Maps.newHashMap(); + for (int iter = 0; iter < dimensions.size(); iter++) { + if (update.get(dimensions.get(iter)) != null) { + Object obj = update.get(dimensions.get(iter)); + renamedMap.put(renamedDimensions.get(iter), obj); + } + } + return renamedMap; + } + } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index cebf2b7913c..3896fe30b99 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Throwables; -import com.google.common.collect.Maps; import com.metamx.common.parsers.TimestampParser; import com.metamx.druid.guava.Runnables; import com.metamx.druid.input.InputRow; @@ -35,7 +34,7 @@ import com.metamx.emitter.EmittingLogger; import org.joda.time.DateTime; import java.io.IOException; -import java.util.List; +import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -48,23 +47,26 @@ public class WebFirehoseFactory implements FirehoseFactory private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); private static final int QUEUE_SIZE = 2000; private final String url; - private final List dimensions; + private final ArrayList dimensions; private final String timeDimension; - private final List renamedDimensions; + private final ArrayList renamedDimensions; + private final String timeFormat; @JsonCreator public WebFirehoseFactory( @JsonProperty("url") String url, - @JsonProperty("dimensions") List dimensions, - @JsonProperty("renamedDimensions") List renamedDimensions, - @JsonProperty("timeDimension") String timeDimension + @JsonProperty("dimensions") ArrayList dimensions, + @JsonProperty("renamedDimensions") ArrayList renamedDimensions, + @JsonProperty("timeDimension") String timeDimension, + @JsonProperty("timeFormat") String timeFormat ) { this.url = url; this.dimensions = dimensions; this.renamedDimensions = renamedDimensions; this.timeDimension = timeDimension; + this.timeFormat = timeFormat; } @Override @@ -72,7 +74,14 @@ public class WebFirehoseFactory implements FirehoseFactory { final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - Runnable updateStream = new UpdateStream(new WebJsonSupplier(dimensions, url), queue, new DefaultObjectMapper()); + Runnable updateStream = new UpdateStream( + new WebJsonSupplier(url), + queue, + new DefaultObjectMapper(), + dimensions, + renamedDimensions, + timeDimension + ); final ExecutorService service = Executors.newSingleThreadExecutor(); service.submit(updateStream); @@ -83,7 +92,7 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public boolean hasMore() { - return !(service.isTerminated()); + return !(service.isTerminated()) && queue.size() > 0; } @@ -91,18 +100,14 @@ public class WebFirehoseFactory implements FirehoseFactory public InputRow nextRow() { try { - Map processedMap = processMap(queue.take()); - DateTime date = TimestampParser.createTimestampParser("auto") - .apply(processedMap.get(timeDimension).toString()); + Map map = queue.take(); + DateTime date = TimestampParser.createTimestampParser(timeFormat) + .apply(map.get(timeDimension).toString()); long seconds = (long) date.getMillis(); - //the parser doesn't check for posix. Only expects iso or millis. This checks for posix - if (new DateTime(seconds * 1000).getYear() == new DateTime().getYear()) { - seconds = (long) date.getMillis() * 1000; - } return new MapBasedInputRow( seconds, renamedDimensions, - processedMap + map ); } catch (Exception e) { @@ -110,28 +115,6 @@ public class WebFirehoseFactory implements FirehoseFactory } } - private Map renameKeys(Map update) - { - Map renamedMap = Maps.newHashMap(); - for (int iter = 0; iter < dimensions.size(); iter++) { - if (update.get(dimensions.get(iter)) != null) { - Object obj = update.get(dimensions.get(iter)); - renamedMap.put(renamedDimensions.get(iter), obj); - } - } - if (renamedMap.get(timeDimension) == null) { - renamedMap.put(timeDimension, System.currentTimeMillis()); - } - return renamedMap; - } - - private Map processMap(Map map) - { - Map renamedMap = renameKeys(map); - return renamedMap; - } - - @Override public Runnable commit() { diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index d14b7dd94e8..8011d52f5fd 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -1,36 +1,15 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - package druid.examples.webStream; import com.google.common.io.InputSupplier; -import com.metamx.druid.input.InputRow; import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.realtime.firehose.Firehose; -import com.metamx.druid.realtime.firehose.FirehoseFactory; +import junit.framework.Assert; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -38,101 +17,38 @@ import java.util.concurrent.TimeUnit; public class WebFirehoseFactoryTest { - @Test - void testMalformedUrlConnect() throws Exception + private final ArrayList dimensions = new ArrayList(); + private InputSupplier testCaseSupplier; + final int QUEUE_SIZE = 2000; + BlockingQueue> queue; + String timeDimension = "t"; + DefaultObjectMapper mapper = new DefaultObjectMapper(); + Map expectedAnswer = new HashMap(); + + @BeforeClass + public void setUp() { - List dimensions = new LinkedList(); - dimensions.add("g"); - dimensions.add("c"); - dimensions.add("a"); - dimensions.add("cy"); - dimensions.add("l"); - dimensions.add("hh"); - dimensions.add("hc"); - dimensions.add("h"); - dimensions.add("u"); - dimensions.add("tz"); - dimensions.add("t"); - dimensions.add("r"); - dimensions.add("gr"); - dimensions.add("nk"); - dimensions.add("al"); - - String invalidURL = "http://invalid.url"; - FirehoseFactory test = new WebFirehoseFactory(invalidURL, dimensions, dimensions, "t"); - Firehose returnedFirehose = test.connect(); - Thread.sleep(3000); - assert returnedFirehose.hasMore() == false; - } - - - @Test - public void testUrlWithNoJsonStreamConnect() throws Exception - { - List dimensions = new LinkedList(); - dimensions.add("g"); - dimensions.add("c"); - dimensions.add("a"); - dimensions.add("cy"); - dimensions.add("l"); - dimensions.add("hh"); - dimensions.add("hc"); - dimensions.add("h"); - dimensions.add("u"); - dimensions.add("tz"); - dimensions.add("t"); - dimensions.add("r"); - dimensions.add("gr"); - dimensions.add("nk"); - dimensions.add("al"); - - String nonJsonUrl = "http://google.com"; - FirehoseFactory test = new WebFirehoseFactory(nonJsonUrl, dimensions, dimensions, "t"); - Firehose returnedFirehose = test.connect(); - Thread.sleep(3000); - assert returnedFirehose.hasMore() == false; - } - - @Test - public void correctUrlCheck() throws Exception - { - List dimensions = new LinkedList(); - dimensions.add("g"); - dimensions.add("c"); - dimensions.add("a"); - dimensions.add("cy"); - dimensions.add("l"); - dimensions.add("hh"); - dimensions.add("hc"); - dimensions.add("h"); - dimensions.add("u"); - dimensions.add("tz"); - dimensions.add("t"); - dimensions.add("r"); - dimensions.add("gr"); - dimensions.add("nk"); - dimensions.add("al"); - - String url = "http://developer.usa.gov/1usagov"; - FirehoseFactory test = new WebFirehoseFactory(url, dimensions, dimensions, "t"); - Firehose returnedFirehose = test.connect(); - Thread.sleep(3000); - assert returnedFirehose.hasMore() == true; - } - - - @Test - public void basicIngestionCheck() throws Exception - { - final int QUEUE_SIZE = 2000; - BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - InputSupplier testCaseSupplier = new TestCaseSupplier( + testCaseSupplier = new TestCaseSupplier( "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" ); - UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue, new DefaultObjectMapper()); - Thread t = new Thread(updateStream); - t.start(); - Map expectedAnswer = new HashMap(); + + dimensions.add("g"); + dimensions.add("c"); + dimensions.add("a"); + dimensions.add("cy"); + dimensions.add("l"); + dimensions.add("hh"); + dimensions.add("hc"); + dimensions.add("h"); + dimensions.add("u"); + dimensions.add("tz"); + dimensions.add("t"); + dimensions.add("r"); + dimensions.add("gr"); + dimensions.add("nk"); + dimensions.add("al"); + dimensions.add("ll"); + expectedAnswer.put("a", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); expectedAnswer.put("c", "US"); expectedAnswer.put("nk", 1); @@ -152,50 +68,151 @@ public class WebFirehoseFactoryTest expectedAnswer.put("hc", 1368193091); expectedAnswer.put("cy", "New York"); expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); - Map insertedRow = queue.poll(10, TimeUnit.SECONDS); - assert expectedAnswer.equals(insertedRow); + } + @Test(expectedExceptions = UnknownHostException.class) + public void checkInvalidUrl() throws Exception + { + + String invalidURL = "http://invalid.url"; + WebJsonSupplier supplier = new WebJsonSupplier(invalidURL); + supplier.getInput(); + } @Test - public void renameDimensionsCheck() throws Exception + public void basicIngestionCheck() throws Exception { - List dimensions = new ArrayList(); - dimensions.add("bitly_hash"); - dimensions.add("country"); - dimensions.add("user"); - dimensions.add("city"); - dimensions.add("encoding_user_login"); - dimensions.add("short_url"); - dimensions.add("timestamp_hash"); - dimensions.add("user_bitly_hash"); - dimensions.add("url"); - dimensions.add("timezone"); - dimensions.add("time"); - dimensions.add("referring_url"); - dimensions.add("geo_region"); - dimensions.add("known_users"); - dimensions.add("accept_language"); - - WebFirehoseFactory webbie = new WebFirehoseFactory( - "http://developer.usa.gov/1usagov", + queue = new ArrayBlockingQueue>(QUEUE_SIZE); + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + queue, + mapper, dimensions, dimensions, - "time" + timeDimension ); - Firehose webbieHose = webbie.connect(); - final int QUEUE_SIZE = 2000; - BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - InputSupplier testCaseSupplier = new TestCaseSupplier( - "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"kw\": \"spcnws\", \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" - ); - UpdateStream updateStream = new UpdateStream(testCaseSupplier, queue, new DefaultObjectMapper()); Thread t = new Thread(updateStream); t.start(); - InputRow row = webbieHose.nextRow(); - assert row.getDimensions().equals(dimensions); - + Map insertedRow = queue.poll(10, TimeUnit.SECONDS); + Assert.assertEquals(expectedAnswer, insertedRow); } + //If a timestamp is missing, we should throw away the event + @Test + public void missingTimeStampCheck() + { + queue = new ArrayBlockingQueue>(QUEUE_SIZE); + InputSupplier testCaseSupplier = new TestCaseSupplier( + "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" + ); + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + queue, + mapper, + dimensions, + dimensions, + timeDimension + ); + updateStream.run(); + Assert.assertEquals(queue.size(), 0); + } + + //If any other value is missing, we should still add the event and process it properly + @Test + public void otherNullValueCheck() throws Exception + { + Map expectedAnswer = new HashMap(); + expectedAnswer.put("a", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); + expectedAnswer.put("nk", 1); + expectedAnswer.put("tz", "America/New_York"); + expectedAnswer.put("gr", "NY"); + expectedAnswer.put("g", "1Chgyj"); + expectedAnswer.put("h", "15vMQjX"); + expectedAnswer.put("l", "o_d63rn9enb"); + expectedAnswer.put("al", "en-US,en;q=0.5"); + expectedAnswer.put("hh", "1.usa.gov"); + expectedAnswer.put( + "r", + "http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX" + ); + expectedAnswer.put("u", "http://www.spc.ncep.noaa.gov/"); + expectedAnswer.put("t", 1372121562); + expectedAnswer.put("hc", 1368193091); + expectedAnswer.put("cy", "New York"); + expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); + queue = new ArrayBlockingQueue>(QUEUE_SIZE); + testCaseSupplier = new TestCaseSupplier( + "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" + ); + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + queue, + mapper, + dimensions, + dimensions, + timeDimension + ); + Thread t = new Thread(updateStream); + t.start(); + Map insertedRow = queue.poll(10, TimeUnit.SECONDS); + Assert.assertEquals(expectedAnswer, insertedRow); + } + + @Test + public void checkRenameKeys() throws Exception + { + Map expectedAnswer = new HashMap(); + queue = new ArrayBlockingQueue>(QUEUE_SIZE); + ArrayList renamedDimensions = new ArrayList(); + renamedDimensions.add("bitly_hash"); + renamedDimensions.add("country"); + renamedDimensions.add("user"); + renamedDimensions.add("city"); + renamedDimensions.add("encoding_user_login"); + renamedDimensions.add("short_url"); + renamedDimensions.add("timestamp_hash"); + renamedDimensions.add("user_bitly_hash"); + renamedDimensions.add("url"); + renamedDimensions.add("timezone"); + renamedDimensions.add("time"); + renamedDimensions.add("referring_url"); + renamedDimensions.add("geo_region"); + renamedDimensions.add("known_users"); + renamedDimensions.add("accept_language"); + renamedDimensions.add("latitude_longitude"); + + expectedAnswer.put("user", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); + expectedAnswer.put("country", "US"); + expectedAnswer.put("known_users", 1); + expectedAnswer.put("timezone", "America/New_York"); + expectedAnswer.put("geo_region", "NY"); + expectedAnswer.put("bitly_hash", "1Chgyj"); + expectedAnswer.put("user_bitly_hash", "15vMQjX"); + expectedAnswer.put("encoding_user_login", "o_d63rn9enb"); + expectedAnswer.put("accept_language", "en-US,en;q=0.5"); + expectedAnswer.put("short_url", "1.usa.gov"); + expectedAnswer.put( + "referring_url", + "http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX" + ); + expectedAnswer.put("url", "http://www.spc.ncep.noaa.gov/"); + expectedAnswer.put("time", 1372121562); + expectedAnswer.put("timestamp_hash", 1368193091); + expectedAnswer.put("city", "New York"); + expectedAnswer.put("latitude_longitude", Arrays.asList(40.862598, -73.921799)); + + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + queue, + mapper, + dimensions, + renamedDimensions, + timeDimension + ); + updateStream.run(); + Map inputRow = queue.poll(10, TimeUnit.SECONDS); + Assert.assertEquals(expectedAnswer, inputRow); + } } diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java index e103fd879b8..9e69a1649f2 100644 --- a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -37,9 +37,8 @@ public class WebJsonSupplier implements InputSupplier private String urlString; private URL url; - public WebJsonSupplier(List dimensions, String urlString) + public WebJsonSupplier(String urlString) { - this.dimensions = dimensions; this.urlString = urlString; try { this.url = new URL(urlString); @@ -56,8 +55,6 @@ public class WebJsonSupplier implements InputSupplier URL url = new URL(urlString); URLConnection connection = url.openConnection(); connection.setDoInput(true); - //connection.setDoOutput(true); - BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); return reader; } From c8c686c738d04a7c64bda97320fa7a1217128d4a Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 12:19:38 -0700 Subject: [PATCH 13/25] added druid license comments --- .../webStream/WebFirehoseFactoryTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index 8011d52f5fd..c4669833b39 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.webStream; import com.google.common.io.InputSupplier; From 439e8ca4adfa09359f37aadd388196c513a5a1b1 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 13:35:22 -0700 Subject: [PATCH 14/25] now take a map for renaming --- .../examples/webStream/UpdateStream.java | 26 +++++----- .../webStream/WebFirehoseFactory.java | 34 ++++++++----- .../webStream/WebFirehoseFactoryTest.java | 50 ++++++++----------- .../examples/webStream/WebJsonSupplier.java | 6 +-- 4 files changed, 60 insertions(+), 56 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 57d8fc67311..3f980762746 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -41,16 +41,14 @@ public class UpdateStream implements Runnable private final InputSupplier supplier; private final BlockingQueue> queue; private final ObjectMapper mapper; - private final ArrayList dimensions; - private final ArrayList renamedDimensions; + private final Map renamedDimensions; private final String timeDimension; public UpdateStream( InputSupplier supplier, BlockingQueue> queue, ObjectMapper mapper, - ArrayList dimensions, - ArrayList renamedDimensions, + Map renamedDimensions, String timeDimension ) { @@ -61,7 +59,6 @@ public class UpdateStream implements Runnable { }; this.timeDimension = timeDimension; - this.dimensions = dimensions; this.renamedDimensions = renamedDimensions; } @@ -84,7 +81,7 @@ public class UpdateStream implements Runnable queue.offer(renamedMap, queueWaitTime, TimeUnit.SECONDS); log.debug("Successfully added to queue"); } else { - log.debug("missing timestamp"); + log.error("missing timestamp"); } } } @@ -97,14 +94,19 @@ public class UpdateStream implements Runnable private Map renameKeys(Map update) { - Map renamedMap = Maps.newHashMap(); - for (int iter = 0; iter < dimensions.size(); iter++) { - if (update.get(dimensions.get(iter)) != null) { - Object obj = update.get(dimensions.get(iter)); - renamedMap.put(renamedDimensions.get(iter), obj); + if (renamedDimensions!=null){ + Map renamedMap = Maps.newHashMap(); + for (String key : renamedDimensions.keySet()) { + if(update.get(key)!=null){ + Object obj= update.get(key); + renamedMap.put(renamedDimensions.get(key),obj); + } } + return renamedMap; + } + else{ + return update; } - return renamedMap; } } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 3896fe30b99..239e23b4acd 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -35,11 +35,13 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; @JsonTypeName("webstream") public class WebFirehoseFactory implements FirehoseFactory @@ -47,26 +49,36 @@ public class WebFirehoseFactory implements FirehoseFactory private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); private static final int QUEUE_SIZE = 2000; private final String url; - private final ArrayList dimensions; private final String timeDimension; - private final ArrayList renamedDimensions; + private final String newTimeDimension; + private final Map renamedDimensions; private final String timeFormat; + private final long waitTime = 15L; @JsonCreator public WebFirehoseFactory( @JsonProperty("url") String url, - @JsonProperty("dimensions") ArrayList dimensions, - @JsonProperty("renamedDimensions") ArrayList renamedDimensions, + @JsonProperty("renamedDimensions") Map renamedDimensions, @JsonProperty("timeDimension") String timeDimension, @JsonProperty("timeFormat") String timeFormat ) { this.url = url; - this.dimensions = dimensions; this.renamedDimensions = renamedDimensions; this.timeDimension = timeDimension; - this.timeFormat = timeFormat; + if (renamedDimensions!=null){ + newTimeDimension=renamedDimensions.get(timeDimension); + } + else{ + newTimeDimension=timeDimension; + } + if (timeFormat==null){ + this.timeFormat="auto"; + } + else{ + this.timeFormat = timeFormat; + } } @Override @@ -78,7 +90,6 @@ public class WebFirehoseFactory implements FirehoseFactory new WebJsonSupplier(url), queue, new DefaultObjectMapper(), - dimensions, renamedDimensions, timeDimension ); @@ -92,7 +103,7 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public boolean hasMore() { - return !(service.isTerminated()) && queue.size() > 0; + return !service.isTerminated(); } @@ -102,11 +113,10 @@ public class WebFirehoseFactory implements FirehoseFactory try { Map map = queue.take(); DateTime date = TimestampParser.createTimestampParser(timeFormat) - .apply(map.get(timeDimension).toString()); - long seconds = (long) date.getMillis(); + .apply(map.get(newTimeDimension).toString()); return new MapBasedInputRow( - seconds, - renamedDimensions, + date.getMillis(), + new ArrayList(map.keySet()), map ); } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index c4669833b39..fb68933f606 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -107,12 +107,10 @@ public class WebFirehoseFactoryTest testCaseSupplier, queue, mapper, - dimensions, - dimensions, + null, timeDimension ); - Thread t = new Thread(updateStream); - t.start(); + updateStream.run(); Map insertedRow = queue.poll(10, TimeUnit.SECONDS); Assert.assertEquals(expectedAnswer, insertedRow); } @@ -129,8 +127,7 @@ public class WebFirehoseFactoryTest testCaseSupplier, queue, mapper, - dimensions, - dimensions, + null, timeDimension ); updateStream.run(); @@ -168,12 +165,10 @@ public class WebFirehoseFactoryTest testCaseSupplier, queue, mapper, - dimensions, - dimensions, + null, timeDimension ); - Thread t = new Thread(updateStream); - t.start(); + updateStream.run(); Map insertedRow = queue.poll(10, TimeUnit.SECONDS); Assert.assertEquals(expectedAnswer, insertedRow); } @@ -183,23 +178,23 @@ public class WebFirehoseFactoryTest { Map expectedAnswer = new HashMap(); queue = new ArrayBlockingQueue>(QUEUE_SIZE); - ArrayList renamedDimensions = new ArrayList(); - renamedDimensions.add("bitly_hash"); - renamedDimensions.add("country"); - renamedDimensions.add("user"); - renamedDimensions.add("city"); - renamedDimensions.add("encoding_user_login"); - renamedDimensions.add("short_url"); - renamedDimensions.add("timestamp_hash"); - renamedDimensions.add("user_bitly_hash"); - renamedDimensions.add("url"); - renamedDimensions.add("timezone"); - renamedDimensions.add("time"); - renamedDimensions.add("referring_url"); - renamedDimensions.add("geo_region"); - renamedDimensions.add("known_users"); - renamedDimensions.add("accept_language"); - renamedDimensions.add("latitude_longitude"); + Map renamedDimensions = new HashMap(); + renamedDimensions.put("g","bitly_hash"); + renamedDimensions.put("c","country"); + renamedDimensions.put("a","user"); + renamedDimensions.put("cy","city"); + renamedDimensions.put("l","encoding_user_login"); + renamedDimensions.put("hh","short_url"); + renamedDimensions.put("hc","timestamp_hash"); + renamedDimensions.put("h","user_bitly_hash"); + renamedDimensions.put("u","url"); + renamedDimensions.put("tz","timezone"); + renamedDimensions.put("t","time"); + renamedDimensions.put("r","referring_url"); + renamedDimensions.put("gr","geo_region"); + renamedDimensions.put("nk","known_users"); + renamedDimensions.put("al","accept_language"); + renamedDimensions.put("ll","latitude_longitude"); expectedAnswer.put("user", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); expectedAnswer.put("country", "US"); @@ -225,7 +220,6 @@ public class WebFirehoseFactoryTest testCaseSupplier, queue, mapper, - dimensions, renamedDimensions, timeDimension ); diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java index 9e69a1649f2..b8a1cd5f7a0 100644 --- a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -44,8 +44,7 @@ public class WebJsonSupplier implements InputSupplier this.url = new URL(urlString); } catch (Exception e) { - e.printStackTrace(); - log.info("Malformed url"); + log.error(e,"Malformed url"); } } @@ -55,7 +54,6 @@ public class WebJsonSupplier implements InputSupplier URL url = new URL(urlString); URLConnection connection = url.openConnection(); connection.setDoInput(true); - BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); - return reader; + return new BufferedReader(new InputStreamReader(url.openStream())); } } From e7da31e42d8d7b96cbce5ecaeac0d43b723ffe98 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 13:41:19 -0700 Subject: [PATCH 15/25] seems to be working --- .../webStream/WebFirehoseFactory.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 239e23b4acd..b2f65bb467e 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -35,13 +35,11 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; @JsonTypeName("webstream") public class WebFirehoseFactory implements FirehoseFactory @@ -51,15 +49,14 @@ public class WebFirehoseFactory implements FirehoseFactory private final String url; private final String timeDimension; private final String newTimeDimension; - private final Map renamedDimensions; + private final Map renamedDimensions; private final String timeFormat; - private final long waitTime = 15L; @JsonCreator public WebFirehoseFactory( @JsonProperty("url") String url, - @JsonProperty("renamedDimensions") Map renamedDimensions, + @JsonProperty("renamedDimensions") Map renamedDimensions, @JsonProperty("timeDimension") String timeDimension, @JsonProperty("timeFormat") String timeFormat ) @@ -67,16 +64,16 @@ public class WebFirehoseFactory implements FirehoseFactory this.url = url; this.renamedDimensions = renamedDimensions; this.timeDimension = timeDimension; - if (renamedDimensions!=null){ - newTimeDimension=renamedDimensions.get(timeDimension); + if (renamedDimensions != null) { + newTimeDimension = renamedDimensions.get(timeDimension); } - else{ - newTimeDimension=timeDimension; + else { + newTimeDimension = timeDimension; } - if (timeFormat==null){ - this.timeFormat="auto"; + if (timeFormat == null) { + this.timeFormat = "auto"; } - else{ + else { this.timeFormat = timeFormat; } } From 01b4728c40c9d4073c2bd628fecae8f7bce15878 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 14:27:35 -0700 Subject: [PATCH 16/25] removed shared queue structure. Queue now encapsulated within updateStream --- .../examples/webStream/UpdateStream.java | 13 +++++++++--- .../webStream/WebFirehoseFactory.java | 7 ++----- .../webStream/WebFirehoseFactoryTest.java | 21 ++++--------------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 3f980762746..08269932520 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -30,6 +30,7 @@ import java.io.BufferedReader; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; @@ -39,21 +40,20 @@ public class UpdateStream implements Runnable private static final long queueWaitTime = 15L; private final TypeReference> typeRef; private final InputSupplier supplier; - private final BlockingQueue> queue; + private final int QUEUE_SIZE=10000; + private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); private final ObjectMapper mapper; private final Map renamedDimensions; private final String timeDimension; public UpdateStream( InputSupplier supplier, - BlockingQueue> queue, ObjectMapper mapper, Map renamedDimensions, String timeDimension ) { this.supplier = supplier; - this.queue = queue; this.mapper = mapper; this.typeRef = new TypeReference>() { @@ -109,4 +109,11 @@ public class UpdateStream implements Runnable } } + public Map takeFromQueue() throws InterruptedException{ + return queue.take(); + } + + public int getQueueSize(){ + return queue.size(); + } } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index b2f65bb467e..cc58d3cba97 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -45,7 +45,6 @@ import java.util.concurrent.Executors; public class WebFirehoseFactory implements FirehoseFactory { private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); - private static final int QUEUE_SIZE = 2000; private final String url; private final String timeDimension; private final String newTimeDimension; @@ -81,11 +80,9 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public Firehose connect() throws IOException { - final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - Runnable updateStream = new UpdateStream( + final UpdateStream updateStream = new UpdateStream( new WebJsonSupplier(url), - queue, new DefaultObjectMapper(), renamedDimensions, timeDimension @@ -108,7 +105,7 @@ public class WebFirehoseFactory implements FirehoseFactory public InputRow nextRow() { try { - Map map = queue.take(); + Map map = updateStream.takeFromQueue(); DateTime date = TimestampParser.createTimestampParser(timeFormat) .apply(map.get(newTimeDimension).toString()); return new MapBasedInputRow( diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index fb68933f606..cc117dc5034 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -30,16 +30,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; public class WebFirehoseFactoryTest { private final ArrayList dimensions = new ArrayList(); private InputSupplier testCaseSupplier; - final int QUEUE_SIZE = 2000; - BlockingQueue> queue; String timeDimension = "t"; DefaultObjectMapper mapper = new DefaultObjectMapper(); Map expectedAnswer = new HashMap(); @@ -102,16 +97,14 @@ public class WebFirehoseFactoryTest @Test public void basicIngestionCheck() throws Exception { - queue = new ArrayBlockingQueue>(QUEUE_SIZE); UpdateStream updateStream = new UpdateStream( testCaseSupplier, - queue, mapper, null, timeDimension ); updateStream.run(); - Map insertedRow = queue.poll(10, TimeUnit.SECONDS); + Map insertedRow = updateStream.takeFromQueue(); Assert.assertEquals(expectedAnswer, insertedRow); } @@ -119,19 +112,17 @@ public class WebFirehoseFactoryTest @Test public void missingTimeStampCheck() { - queue = new ArrayBlockingQueue>(QUEUE_SIZE); InputSupplier testCaseSupplier = new TestCaseSupplier( "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" ); UpdateStream updateStream = new UpdateStream( testCaseSupplier, - queue, mapper, null, timeDimension ); updateStream.run(); - Assert.assertEquals(queue.size(), 0); + Assert.assertEquals(updateStream.getQueueSize(), 0); } //If any other value is missing, we should still add the event and process it properly @@ -157,19 +148,17 @@ public class WebFirehoseFactoryTest expectedAnswer.put("hc", 1368193091); expectedAnswer.put("cy", "New York"); expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); - queue = new ArrayBlockingQueue>(QUEUE_SIZE); testCaseSupplier = new TestCaseSupplier( "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" ); UpdateStream updateStream = new UpdateStream( testCaseSupplier, - queue, mapper, null, timeDimension ); updateStream.run(); - Map insertedRow = queue.poll(10, TimeUnit.SECONDS); + Map insertedRow = updateStream.takeFromQueue(); Assert.assertEquals(expectedAnswer, insertedRow); } @@ -177,7 +166,6 @@ public class WebFirehoseFactoryTest public void checkRenameKeys() throws Exception { Map expectedAnswer = new HashMap(); - queue = new ArrayBlockingQueue>(QUEUE_SIZE); Map renamedDimensions = new HashMap(); renamedDimensions.put("g","bitly_hash"); renamedDimensions.put("c","country"); @@ -218,13 +206,12 @@ public class WebFirehoseFactoryTest UpdateStream updateStream = new UpdateStream( testCaseSupplier, - queue, mapper, renamedDimensions, timeDimension ); updateStream.run(); - Map inputRow = queue.poll(10, TimeUnit.SECONDS); + Map inputRow = updateStream.takeFromQueue(); Assert.assertEquals(expectedAnswer, inputRow); } From 142271aad2bfb1befc8a66a3b78894cc370a5050 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 16:46:47 -0700 Subject: [PATCH 17/25] better encapsulation --- .../examples/webStream/UpdateStream.java | 42 ++- .../webStream/UpdateStreamFactory.java | 44 +++ .../examples/webStream/UpdateStreamTest.java | 144 +++++++++ .../webStream/WebFirehoseFactory.java | 45 ++- .../webStream/WebFirehoseFactoryTest.java | 275 +++++++----------- 5 files changed, 339 insertions(+), 211 deletions(-) create mode 100644 examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java create mode 100644 examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 08269932520..87e1a2ffffb 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -24,10 +24,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Maps; import com.google.common.io.InputSupplier; +import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.emitter.EmittingLogger; import java.io.BufferedReader; -import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; @@ -40,21 +40,20 @@ public class UpdateStream implements Runnable private static final long queueWaitTime = 15L; private final TypeReference> typeRef; private final InputSupplier supplier; - private final int QUEUE_SIZE=10000; + private final int QUEUE_SIZE = 10000; private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - private final ObjectMapper mapper; - private final Map renamedDimensions; + private final ObjectMapper mapper = new DefaultObjectMapper(); + private final Map renamedDimensions; private final String timeDimension; + private final long waitTime = 15L; public UpdateStream( InputSupplier supplier, - ObjectMapper mapper, - Map renamedDimensions, + Map renamedDimensions, String timeDimension ) { this.supplier = supplier; - this.mapper = mapper; this.typeRef = new TypeReference>() { }; @@ -94,26 +93,37 @@ public class UpdateStream implements Runnable private Map renameKeys(Map update) { - if (renamedDimensions!=null){ + if (renamedDimensions != null) { Map renamedMap = Maps.newHashMap(); for (String key : renamedDimensions.keySet()) { - if(update.get(key)!=null){ - Object obj= update.get(key); - renamedMap.put(renamedDimensions.get(key),obj); + if (update.get(key) != null) { + Object obj = update.get(key); + renamedMap.put(renamedDimensions.get(key), obj); } } return renamedMap; - } - else{ + } else { return update; } } - public Map takeFromQueue() throws InterruptedException{ - return queue.take(); + public Map pollFromQueue() throws InterruptedException + { + return queue.poll(waitTime, TimeUnit.SECONDS); } - public int getQueueSize(){ + public int getQueueSize() + { return queue.size(); } + + public String getNewTimeDimension() + { + if (renamedDimensions != null) { + return renamedDimensions.get(timeDimension); + } else { + return timeDimension; + } + + } } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java new file mode 100644 index 00000000000..2bff55b3ce3 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java @@ -0,0 +1,44 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; + +import java.util.Map; + +public class UpdateStreamFactory +{ + private final InputSupplier inputSupplier; + private final Map renamedDimensions; + private final String timeDimension; + + public UpdateStreamFactory(InputSupplier inputSupplier, Map renamedDimensions, String timeDimension) + { + this.inputSupplier = inputSupplier; + this.renamedDimensions = renamedDimensions; + this.timeDimension = timeDimension; + } + + public UpdateStream build() + { + return new UpdateStream(inputSupplier, renamedDimensions, timeDimension); + } + +} diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java new file mode 100644 index 00000000000..1aed4566213 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java @@ -0,0 +1,144 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; +import com.metamx.druid.jackson.DefaultObjectMapper; +import junit.framework.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class UpdateStreamTest +{ + private final ArrayList dimensions = new ArrayList(); + private InputSupplier testCaseSupplier; + DefaultObjectMapper mapper = new DefaultObjectMapper(); + Map expectedAnswer = new HashMap(); + String timeDimension; + + @BeforeClass + public void setUp() + { + timeDimension = "time"; + testCaseSupplier = new TestCaseSupplier( + "{\"item1\": \"value1\"," + + "\"item2\":2," + + "\"time\":1372121562 }" + ); + + dimensions.add("item1"); + dimensions.add("item2"); + dimensions.add("time"); + + expectedAnswer.put("item1", "value1"); + expectedAnswer.put("item2", 2); + expectedAnswer.put("time", 1372121562); + } + + @Test(expectedExceptions = UnknownHostException.class) + public void checkInvalidUrl() throws Exception + { + + String invalidURL = "http://invalid.url"; + WebJsonSupplier supplier = new WebJsonSupplier(invalidURL); + supplier.getInput(); + } + + @Test + public void basicIngestionCheck() throws Exception + { + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + null, + timeDimension + ); + updateStream.run(); + Map insertedRow = updateStream.pollFromQueue(); + Assert.assertEquals(expectedAnswer, insertedRow); + } + + //If a timestamp is missing, we should throw away the event + @Test + public void missingTimeStampCheck() + { + testCaseSupplier = new TestCaseSupplier( + "{\"item1\": \"value1\"," + + "\"item2\":2}" + ); + + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + null, + timeDimension + ); + updateStream.run(); + Assert.assertEquals(updateStream.getQueueSize(), 0); + } + + //If any other value is missing, we should still add the event and process it properly + @Test + public void otherNullValueCheck() throws Exception + { + testCaseSupplier = new TestCaseSupplier( + "{\"item1\": \"value1\"," + + "\"time\":1372121562 }" + ); + Map expectedAnswer = new HashMap(); + expectedAnswer.put("item1", "value1"); + expectedAnswer.put("time", 1372121562); + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + null, + timeDimension + ); + updateStream.run(); + Map insertedRow = updateStream.pollFromQueue(); + Assert.assertEquals(expectedAnswer, insertedRow); + } + + @Test + public void checkRenameKeys() throws Exception + { + Map expectedAnswer = new HashMap(); + Map renamedDimensions = new HashMap(); + renamedDimensions.put("item1", "i1"); + renamedDimensions.put("item2", "i2"); + renamedDimensions.put("time", "t"); + + expectedAnswer.put("i1", "value1"); + expectedAnswer.put("i2", 2); + expectedAnswer.put("t", 1372121562); + + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + renamedDimensions, + timeDimension + ); + updateStream.run(); + Map inputRow = updateStream.pollFromQueue(); + Assert.assertEquals(expectedAnswer, inputRow); + } + +} diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index cc58d3cba97..a174ba84654 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -27,7 +27,6 @@ import com.metamx.common.parsers.TimestampParser; import com.metamx.druid.guava.Runnables; import com.metamx.druid.input.InputRow; import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.druid.realtime.firehose.Firehose; import com.metamx.druid.realtime.firehose.FirehoseFactory; import com.metamx.emitter.EmittingLogger; @@ -36,8 +35,6 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayList; import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,11 +42,8 @@ import java.util.concurrent.Executors; public class WebFirehoseFactory implements FirehoseFactory { private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); - private final String url; - private final String timeDimension; - private final String newTimeDimension; - private final Map renamedDimensions; private final String timeFormat; + private final UpdateStreamFactory updateStreamFactory; @JsonCreator @@ -60,19 +54,15 @@ public class WebFirehoseFactory implements FirehoseFactory @JsonProperty("timeFormat") String timeFormat ) { - this.url = url; - this.renamedDimensions = renamedDimensions; - this.timeDimension = timeDimension; - if (renamedDimensions != null) { - newTimeDimension = renamedDimensions.get(timeDimension); - } - else { - newTimeDimension = timeDimension; - } + this(new UpdateStreamFactory(new WebJsonSupplier(url), renamedDimensions, timeDimension), timeFormat); + } + + public WebFirehoseFactory(UpdateStreamFactory updateStreamFactory, String timeFormat) + { + this.updateStreamFactory = updateStreamFactory; if (timeFormat == null) { this.timeFormat = "auto"; - } - else { + } else { this.timeFormat = timeFormat; } } @@ -81,23 +71,25 @@ public class WebFirehoseFactory implements FirehoseFactory public Firehose connect() throws IOException { - final UpdateStream updateStream = new UpdateStream( - new WebJsonSupplier(url), - new DefaultObjectMapper(), - renamedDimensions, - timeDimension - ); + final UpdateStream updateStream = updateStreamFactory.build(); final ExecutorService service = Executors.newSingleThreadExecutor(); service.submit(updateStream); return new Firehose() { + Map map; private final Runnable doNothingRunnable = Runnables.getNoopRunnable(); @Override public boolean hasMore() { - return !service.isTerminated(); + try { + map = updateStream.pollFromQueue(); + return map != null; + } + catch (Exception e) { + throw Throwables.propagate(e); + } } @@ -105,9 +97,8 @@ public class WebFirehoseFactory implements FirehoseFactory public InputRow nextRow() { try { - Map map = updateStream.takeFromQueue(); DateTime date = TimestampParser.createTimestampParser(timeFormat) - .apply(map.get(newTimeDimension).toString()); + .apply(map.get(updateStream.getNewTimeDimension()).toString()); return new MapBasedInputRow( date.getMillis(), new ArrayList(map.keySet()), diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index cc117dc5034..5494a500826 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -19,200 +19,139 @@ package druid.examples.webStream; -import com.google.common.io.InputSupplier; -import com.metamx.druid.jackson.DefaultObjectMapper; +import com.beust.jcommander.internal.Lists; +import com.metamx.druid.input.InputRow; +import com.metamx.druid.realtime.firehose.Firehose; import junit.framework.Assert; +import org.joda.time.DateTime; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.Collections; +import java.util.List; public class WebFirehoseFactoryTest { - private final ArrayList dimensions = new ArrayList(); - private InputSupplier testCaseSupplier; - String timeDimension = "t"; - DefaultObjectMapper mapper = new DefaultObjectMapper(); - Map expectedAnswer = new HashMap(); + private List dimensions; + private WebFirehoseFactory webbie; + private Firehose firehose; + private InputRow inputRow; + private TestCaseSupplier testCaseSupplier; @BeforeClass - public void setUp() + public void setUp() throws Exception + { + dimensions = new ArrayList(); + dimensions.add("item1"); + dimensions.add("item2"); + dimensions.add("time"); + testCaseSupplier = new TestCaseSupplier( + "{\"item1\":\"value1\"," + + "\"item2\":2," + + "\"time\":1372121562 }" + ); + + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); + webbie = new WebFirehoseFactory(updateStreamFactory, "posix"); + firehose = webbie.connect(); + if (firehose.hasMore()) { + inputRow = firehose.nextRow(); + } else { + throw new RuntimeException("queue is empty"); + } + + } + + @Test + public void testDimensions() throws Exception + { + List actualAnswer = inputRow.getDimensions(); + Collections.sort(actualAnswer); + Assert.assertEquals(actualAnswer, dimensions); + } + + @Test + public void testPosixTimeStamp() throws Exception + { + long expectedTime = 1372121562L * 1000L; + Assert.assertEquals(expectedTime, inputRow.getTimestampFromEpoch()); + } + + @Test + public void testISOTimeStamp() throws Exception { testCaseSupplier = new TestCaseSupplier( - "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" + "{\"item1\": \"value1\"," + + "\"item2\":2," + + "\"time\":\"2013-07-08\"}" ); - dimensions.add("g"); - dimensions.add("c"); - dimensions.add("a"); - dimensions.add("cy"); - dimensions.add("l"); - dimensions.add("hh"); - dimensions.add("hc"); - dimensions.add("h"); - dimensions.add("u"); - dimensions.add("tz"); - dimensions.add("t"); - dimensions.add("r"); - dimensions.add("gr"); - dimensions.add("nk"); - dimensions.add("al"); - dimensions.add("ll"); - - expectedAnswer.put("a", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); - expectedAnswer.put("c", "US"); - expectedAnswer.put("nk", 1); - expectedAnswer.put("tz", "America/New_York"); - expectedAnswer.put("gr", "NY"); - expectedAnswer.put("g", "1Chgyj"); - expectedAnswer.put("h", "15vMQjX"); - expectedAnswer.put("l", "o_d63rn9enb"); - expectedAnswer.put("al", "en-US,en;q=0.5"); - expectedAnswer.put("hh", "1.usa.gov"); - expectedAnswer.put( - "r", - "http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX" - ); - expectedAnswer.put("u", "http://www.spc.ncep.noaa.gov/"); - expectedAnswer.put("t", 1372121562); - expectedAnswer.put("hc", 1368193091); - expectedAnswer.put("cy", "New York"); - expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); - - } - - @Test(expectedExceptions = UnknownHostException.class) - public void checkInvalidUrl() throws Exception - { - - String invalidURL = "http://invalid.url"; - WebJsonSupplier supplier = new WebJsonSupplier(invalidURL); - supplier.getInput(); + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); + webbie = new WebFirehoseFactory(updateStreamFactory, "iso"); + Firehose firehose1 = webbie.connect(); + if (firehose1.hasMore()) { + long milliSeconds = firehose1.nextRow().getTimestampFromEpoch(); + DateTime date = new DateTime("2013-07-08"); + Assert.assertEquals(date.getMillis(), milliSeconds); + } else { + Assert.assertFalse("hasMore returned false", true); + } } @Test - public void basicIngestionCheck() throws Exception + public void testAutoIsoTimeStamp() throws Exception { - UpdateStream updateStream = new UpdateStream( - testCaseSupplier, - mapper, - null, - timeDimension - ); - updateStream.run(); - Map insertedRow = updateStream.takeFromQueue(); - Assert.assertEquals(expectedAnswer, insertedRow); - } - - //If a timestamp is missing, we should throw away the event - @Test - public void missingTimeStampCheck() - { - InputSupplier testCaseSupplier = new TestCaseSupplier( - "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"c\": \"US\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" - ); - UpdateStream updateStream = new UpdateStream( - testCaseSupplier, - mapper, - null, - timeDimension - ); - updateStream.run(); - Assert.assertEquals(updateStream.getQueueSize(), 0); - } - - //If any other value is missing, we should still add the event and process it properly - @Test - public void otherNullValueCheck() throws Exception - { - Map expectedAnswer = new HashMap(); - expectedAnswer.put("a", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); - expectedAnswer.put("nk", 1); - expectedAnswer.put("tz", "America/New_York"); - expectedAnswer.put("gr", "NY"); - expectedAnswer.put("g", "1Chgyj"); - expectedAnswer.put("h", "15vMQjX"); - expectedAnswer.put("l", "o_d63rn9enb"); - expectedAnswer.put("al", "en-US,en;q=0.5"); - expectedAnswer.put("hh", "1.usa.gov"); - expectedAnswer.put( - "r", - "http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX" - ); - expectedAnswer.put("u", "http://www.spc.ncep.noaa.gov/"); - expectedAnswer.put("t", 1372121562); - expectedAnswer.put("hc", 1368193091); - expectedAnswer.put("cy", "New York"); - expectedAnswer.put("ll", Arrays.asList(40.862598, -73.921799)); testCaseSupplier = new TestCaseSupplier( - "{ \"a\": \"Mozilla\\/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko\\/20100101 Firefox\\/21.0\", \"nk\": 1, \"tz\": \"America\\/New_York\", \"gr\": \"NY\", \"g\": \"1Chgyj\", \"h\": \"15vMQjX\", \"l\": \"o_d63rn9enb\", \"al\": \"en-US,en;q=0.5\", \"hh\": \"1.usa.gov\", \"r\": \"http:\\/\\/forecast.weather.gov\\/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX\", \"u\": \"http:\\/\\/www.spc.ncep.noaa.gov\\/\", \"t\": 1372121562, \"hc\": 1368193091, \"cy\": \"New York\", \"ll\": [ 40.862598, -73.921799 ] }" + "{\"item1\": \"value1\"," + + "\"item2\":2," + + "\"time\":\"2013-07-08\"}" ); - UpdateStream updateStream = new UpdateStream( - testCaseSupplier, - mapper, - null, - timeDimension - ); - updateStream.run(); - Map insertedRow = updateStream.takeFromQueue(); - Assert.assertEquals(expectedAnswer, insertedRow); + + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); + webbie = new WebFirehoseFactory(updateStreamFactory, null); + Firehose firehose2 = webbie.connect(); + if (firehose2.hasMore()) { + long milliSeconds = firehose2.nextRow().getTimestampFromEpoch(); + DateTime date = new DateTime("2013-07-08"); + Assert.assertEquals(date.getMillis(), milliSeconds); + } else { + Assert.assertFalse("hasMore returned false", true); + } } @Test - public void checkRenameKeys() throws Exception + public void testAutoMilliSecondsTimeStamp() throws Exception { - Map expectedAnswer = new HashMap(); - Map renamedDimensions = new HashMap(); - renamedDimensions.put("g","bitly_hash"); - renamedDimensions.put("c","country"); - renamedDimensions.put("a","user"); - renamedDimensions.put("cy","city"); - renamedDimensions.put("l","encoding_user_login"); - renamedDimensions.put("hh","short_url"); - renamedDimensions.put("hc","timestamp_hash"); - renamedDimensions.put("h","user_bitly_hash"); - renamedDimensions.put("u","url"); - renamedDimensions.put("tz","timezone"); - renamedDimensions.put("t","time"); - renamedDimensions.put("r","referring_url"); - renamedDimensions.put("gr","geo_region"); - renamedDimensions.put("nk","known_users"); - renamedDimensions.put("al","accept_language"); - renamedDimensions.put("ll","latitude_longitude"); - - expectedAnswer.put("user", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"); - expectedAnswer.put("country", "US"); - expectedAnswer.put("known_users", 1); - expectedAnswer.put("timezone", "America/New_York"); - expectedAnswer.put("geo_region", "NY"); - expectedAnswer.put("bitly_hash", "1Chgyj"); - expectedAnswer.put("user_bitly_hash", "15vMQjX"); - expectedAnswer.put("encoding_user_login", "o_d63rn9enb"); - expectedAnswer.put("accept_language", "en-US,en;q=0.5"); - expectedAnswer.put("short_url", "1.usa.gov"); - expectedAnswer.put( - "referring_url", - "http://forecast.weather.gov/MapClick.php?site=okx&FcstType=text&zmx=1&zmy=1&map.x=98&map.y=200&site=OKX" + testCaseSupplier = new TestCaseSupplier( + "{\"item1\": \"value1\"," + + "\"item2\":2," + + "\"time\":1373241600000}" ); - expectedAnswer.put("url", "http://www.spc.ncep.noaa.gov/"); - expectedAnswer.put("time", 1372121562); - expectedAnswer.put("timestamp_hash", 1368193091); - expectedAnswer.put("city", "New York"); - expectedAnswer.put("latitude_longitude", Arrays.asList(40.862598, -73.921799)); - UpdateStream updateStream = new UpdateStream( - testCaseSupplier, - mapper, - renamedDimensions, - timeDimension - ); - updateStream.run(); - Map inputRow = updateStream.takeFromQueue(); - Assert.assertEquals(expectedAnswer, inputRow); + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); + webbie = new WebFirehoseFactory(updateStreamFactory, null); + Firehose firehose3 = webbie.connect(); + if (firehose3.hasMore()) { + long milliSeconds = firehose3.nextRow().getTimestampFromEpoch(); + DateTime date = new DateTime("2013-07-08"); + Assert.assertEquals(date.getMillis(), milliSeconds); + } else { + Assert.assertFalse("hasMore returned false", true); + } } + @Test + public void testGetDimension() + { + List column1 = Lists.newArrayList(); + column1.add("value1"); + Assert.assertEquals(column1, inputRow.getDimension("item1")); + } + + @Test + public void testGetFloatMetric() + { + Assert.assertEquals((float) 2.0, inputRow.getFloatMetric("item2")); + } } From 72fbc516bc4de666370f687afd0a389686f1e739 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 8 Jul 2013 18:53:08 -0700 Subject: [PATCH 18/25] added a RenameKeysUpdateStream wrapper --- .../webStream/RenamingKeysUpdateStream.java | 79 ++++++++++++++++ .../RenamingKeysUpdateStreamFactory.java | 37 ++++++++ .../RenamingKeysUpdateStreamTest.java | 92 +++++++++++++++++++ .../examples/webStream/UpdateStream.java | 37 ++------ .../webStream/UpdateStreamFactory.java | 8 +- .../examples/webStream/UpdateStreamTest.java | 53 ++--------- .../webStream/WebFirehoseFactory.java | 30 ++++-- .../webStream/WebFirehoseFactoryTest.java | 66 +++++++++---- .../examples/webStream/WebJsonSupplier.java | 1 - .../webStream/WebJsonSupplierTest.java | 36 ++++++++ 10 files changed, 328 insertions(+), 111 deletions(-) create mode 100644 examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java create mode 100644 examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java create mode 100644 examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java create mode 100644 examples/src/main/java/druid/examples/webStream/WebJsonSupplierTest.java diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java new file mode 100644 index 00000000000..922839f493e --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java @@ -0,0 +1,79 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.webStream; + +import com.google.common.collect.Maps; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class RenamingKeysUpdateStream implements Runnable +{ + + private final UpdateStream updateStream; + private Map renamedDimensions; + + public RenamingKeysUpdateStream( + UpdateStream updateStream, + Map renamedDimensions + ) + { + this.renamedDimensions = renamedDimensions; + this.updateStream = updateStream; + } + + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return renameKeys(updateStream.pollFromQueue(waitTime, unit)); + } + + + private Map renameKeys(Map update) + { + if (renamedDimensions != null) { + Map renamedMap = Maps.newHashMap(); + for (String key : renamedDimensions.keySet()) { + if (update.get(key) != null) { + Object obj = update.get(key); + renamedMap.put(renamedDimensions.get(key), obj); + } + } + return renamedMap; + } else { + return update; + } + } + + public String getTimeDimension() + { + if (renamedDimensions != null && renamedDimensions.get(updateStream.getTimeDimension()) != null) { + return renamedDimensions.get(updateStream.getTimeDimension()); + } + return updateStream.getTimeDimension(); + + } + + + @Override + public void run() + { + updateStream.run(); + } +} diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java new file mode 100644 index 00000000000..4dafc3becd3 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java @@ -0,0 +1,37 @@ +package druid.examples.webStream;/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +import java.util.Map; + +public class RenamingKeysUpdateStreamFactory +{ + private UpdateStreamFactory updateStreamFactory; + private Map renamedDimensions; + + public RenamingKeysUpdateStreamFactory(UpdateStreamFactory updateStreamFactory, Map renamedDimensions) + { + this.updateStreamFactory = updateStreamFactory; + this.renamedDimensions = renamedDimensions; + } + + public RenamingKeysUpdateStream build() + { + return new RenamingKeysUpdateStream(updateStreamFactory.build(), renamedDimensions); + } +} diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java new file mode 100644 index 00000000000..d4acd52fabc --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java @@ -0,0 +1,92 @@ +package druid.examples.webStream;/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +import com.google.common.io.InputSupplier; +import junit.framework.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class RenamingKeysUpdateStreamTest +{ + private final long waitTime = 15L; + private final TimeUnit unit = TimeUnit.SECONDS; + private InputSupplier testCaseSupplier; + String timeDimension; + + @BeforeClass + public void setUp() + { + timeDimension = "time"; + testCaseSupplier = new TestCaseSupplier( + "{\"item1\": \"value1\"," + + "\"item2\":2," + + "\"time\":1372121562 }" + ); + } + + @Test + public void testPollFromQueue() throws Exception + { + UpdateStream updateStream = new UpdateStream(testCaseSupplier, timeDimension); + Map renamedKeys = new HashMap(); + renamedKeys.put("item1", "i1"); + renamedKeys.put("item2", "i2"); + renamedKeys.put("time", "t"); + + RenamingKeysUpdateStream renamer = new RenamingKeysUpdateStream(updateStream, renamedKeys); + renamer.run(); + Map expectedAnswer = new HashMap(); + expectedAnswer.put("i1", "value1"); + expectedAnswer.put("i2", 2); + expectedAnswer.put("t", 1372121562); + + + Assert.assertEquals(expectedAnswer, renamer.pollFromQueue(waitTime, unit)); + } + + @Test + public void testGetTimeDimension() throws Exception + { + UpdateStream updateStream = new UpdateStream(testCaseSupplier, timeDimension); + Map renamedKeys = new HashMap(); + renamedKeys.put("item1", "i1"); + renamedKeys.put("item2", "i2"); + renamedKeys.put("time", "t"); + + RenamingKeysUpdateStream renamer = new RenamingKeysUpdateStream(updateStream, renamedKeys); + Assert.assertEquals("t", renamer.getTimeDimension()); + } + + @Test + public void testMissingTimeRename() throws Exception + { + UpdateStream updateStream = new UpdateStream(testCaseSupplier, timeDimension); + Map renamedKeys = new HashMap(); + renamedKeys.put("item1", "i1"); + renamedKeys.put("item2", "i2"); + RenamingKeysUpdateStream renamer = new RenamingKeysUpdateStream(updateStream, renamedKeys); + Assert.assertEquals("time", renamer.getTimeDimension()); + + } + +} diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 87e1a2ffffb..3a19e11b6b5 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -43,13 +43,10 @@ public class UpdateStream implements Runnable private final int QUEUE_SIZE = 10000; private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); private final ObjectMapper mapper = new DefaultObjectMapper(); - private final Map renamedDimensions; private final String timeDimension; - private final long waitTime = 15L; public UpdateStream( InputSupplier supplier, - Map renamedDimensions, String timeDimension ) { @@ -58,7 +55,6 @@ public class UpdateStream implements Runnable { }; this.timeDimension = timeDimension; - this.renamedDimensions = renamedDimensions; } private boolean isValid(String s) @@ -76,8 +72,7 @@ public class UpdateStream implements Runnable if (isValid(line)) { HashMap map = mapper.readValue(line, typeRef); if (map.get(timeDimension) != null) { - Map renamedMap = renameKeys(map); - queue.offer(renamedMap, queueWaitTime, TimeUnit.SECONDS); + queue.offer(map, queueWaitTime, TimeUnit.SECONDS); log.debug("Successfully added to queue"); } else { log.error("missing timestamp"); @@ -91,25 +86,10 @@ public class UpdateStream implements Runnable } - private Map renameKeys(Map update) - { - if (renamedDimensions != null) { - Map renamedMap = Maps.newHashMap(); - for (String key : renamedDimensions.keySet()) { - if (update.get(key) != null) { - Object obj = update.get(key); - renamedMap.put(renamedDimensions.get(key), obj); - } - } - return renamedMap; - } else { - return update; - } - } - public Map pollFromQueue() throws InterruptedException + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException { - return queue.poll(waitTime, TimeUnit.SECONDS); + return queue.poll(waitTime, unit); } public int getQueueSize() @@ -117,13 +97,8 @@ public class UpdateStream implements Runnable return queue.size(); } - public String getNewTimeDimension() - { - if (renamedDimensions != null) { - return renamedDimensions.get(timeDimension); - } else { - return timeDimension; - } - + public String getTimeDimension(){ + return timeDimension; } + } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java index 2bff55b3ce3..2a69eb42c1d 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java @@ -21,24 +21,20 @@ package druid.examples.webStream; import com.google.common.io.InputSupplier; -import java.util.Map; - public class UpdateStreamFactory { private final InputSupplier inputSupplier; - private final Map renamedDimensions; private final String timeDimension; - public UpdateStreamFactory(InputSupplier inputSupplier, Map renamedDimensions, String timeDimension) + public UpdateStreamFactory(InputSupplier inputSupplier, String timeDimension) { this.inputSupplier = inputSupplier; - this.renamedDimensions = renamedDimensions; this.timeDimension = timeDimension; } public UpdateStream build() { - return new UpdateStream(inputSupplier, renamedDimensions, timeDimension); + return new UpdateStream(inputSupplier, timeDimension); } } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java index 1aed4566213..a14993ff9a7 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java @@ -20,21 +20,21 @@ package druid.examples.webStream; import com.google.common.io.InputSupplier; -import com.metamx.druid.jackson.DefaultObjectMapper; import junit.framework.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; public class UpdateStreamTest { + private final long waitTime = 15L; + private final TimeUnit unit = TimeUnit.SECONDS; private final ArrayList dimensions = new ArrayList(); private InputSupplier testCaseSupplier; - DefaultObjectMapper mapper = new DefaultObjectMapper(); Map expectedAnswer = new HashMap(); String timeDimension; @@ -57,25 +57,16 @@ public class UpdateStreamTest expectedAnswer.put("time", 1372121562); } - @Test(expectedExceptions = UnknownHostException.class) - public void checkInvalidUrl() throws Exception - { - - String invalidURL = "http://invalid.url"; - WebJsonSupplier supplier = new WebJsonSupplier(invalidURL); - supplier.getInput(); - } @Test public void basicIngestionCheck() throws Exception { UpdateStream updateStream = new UpdateStream( testCaseSupplier, - null, timeDimension ); updateStream.run(); - Map insertedRow = updateStream.pollFromQueue(); + Map insertedRow = updateStream.pollFromQueue(waitTime, unit); Assert.assertEquals(expectedAnswer, insertedRow); } @@ -90,7 +81,6 @@ public class UpdateStreamTest UpdateStream updateStream = new UpdateStream( testCaseSupplier, - null, timeDimension ); updateStream.run(); @@ -105,40 +95,17 @@ public class UpdateStreamTest "{\"item1\": \"value1\"," + "\"time\":1372121562 }" ); + UpdateStream updateStream = new UpdateStream( + testCaseSupplier, + timeDimension + ); + updateStream.run(); + Map insertedRow = updateStream.pollFromQueue(waitTime, unit); Map expectedAnswer = new HashMap(); expectedAnswer.put("item1", "value1"); expectedAnswer.put("time", 1372121562); - UpdateStream updateStream = new UpdateStream( - testCaseSupplier, - null, - timeDimension - ); - updateStream.run(); - Map insertedRow = updateStream.pollFromQueue(); Assert.assertEquals(expectedAnswer, insertedRow); } - @Test - public void checkRenameKeys() throws Exception - { - Map expectedAnswer = new HashMap(); - Map renamedDimensions = new HashMap(); - renamedDimensions.put("item1", "i1"); - renamedDimensions.put("item2", "i2"); - renamedDimensions.put("time", "t"); - - expectedAnswer.put("i1", "value1"); - expectedAnswer.put("i2", 2); - expectedAnswer.put("t", 1372121562); - - UpdateStream updateStream = new UpdateStream( - testCaseSupplier, - renamedDimensions, - timeDimension - ); - updateStream.run(); - Map inputRow = updateStream.pollFromQueue(); - Assert.assertEquals(expectedAnswer, inputRow); - } } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index a174ba84654..a1285ee18e4 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -34,17 +34,19 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; @JsonTypeName("webstream") public class WebFirehoseFactory implements FirehoseFactory { private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); private final String timeFormat; - private final UpdateStreamFactory updateStreamFactory; - + private final RenamingKeysUpdateStreamFactory factory; + private final long queueWaitTime = 15L; @JsonCreator public WebFirehoseFactory( @@ -54,12 +56,17 @@ public class WebFirehoseFactory implements FirehoseFactory @JsonProperty("timeFormat") String timeFormat ) { - this(new UpdateStreamFactory(new WebJsonSupplier(url), renamedDimensions, timeDimension), timeFormat); + this( + new RenamingKeysUpdateStreamFactory( + new UpdateStreamFactory(new WebJsonSupplier(url), timeDimension), + renamedDimensions + ), timeFormat + ); } - public WebFirehoseFactory(UpdateStreamFactory updateStreamFactory, String timeFormat) + public WebFirehoseFactory(RenamingKeysUpdateStreamFactory factory, String timeFormat) { - this.updateStreamFactory = updateStreamFactory; + this.factory = factory; if (timeFormat == null) { this.timeFormat = "auto"; } else { @@ -71,9 +78,9 @@ public class WebFirehoseFactory implements FirehoseFactory public Firehose connect() throws IOException { - final UpdateStream updateStream = updateStreamFactory.build(); + final RenamingKeysUpdateStream renamingUpdateStream = factory.build(); final ExecutorService service = Executors.newSingleThreadExecutor(); - service.submit(updateStream); + service.submit(renamingUpdateStream); return new Firehose() { @@ -84,10 +91,10 @@ public class WebFirehoseFactory implements FirehoseFactory public boolean hasMore() { try { - map = updateStream.pollFromQueue(); + map = renamingUpdateStream.pollFromQueue(queueWaitTime, TimeUnit.SECONDS); return map != null; } - catch (Exception e) { + catch (InterruptedException e) { throw Throwables.propagate(e); } } @@ -98,7 +105,7 @@ public class WebFirehoseFactory implements FirehoseFactory { try { DateTime date = TimestampParser.createTimestampParser(timeFormat) - .apply(map.get(updateStream.getNewTimeDimension()).toString()); + .apply(map.get(renamingUpdateStream.getTimeDimension()).toString()); return new MapBasedInputRow( date.getMillis(), new ArrayList(map.keySet()), @@ -108,6 +115,9 @@ public class WebFirehoseFactory implements FirehoseFactory catch (Exception e) { throw Throwables.propagate(e); } + finally { + map = null; + } } @Override diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index 5494a500826..e51ef9e79ff 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -35,8 +35,6 @@ public class WebFirehoseFactoryTest { private List dimensions; private WebFirehoseFactory webbie; - private Firehose firehose; - private InputRow inputRow; private TestCaseSupplier testCaseSupplier; @BeforeClass @@ -51,21 +49,20 @@ public class WebFirehoseFactoryTest + "\"item2\":2," + "\"time\":1372121562 }" ); - - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); - webbie = new WebFirehoseFactory(updateStreamFactory, "posix"); - firehose = webbie.connect(); - if (firehose.hasMore()) { - inputRow = firehose.nextRow(); - } else { - throw new RuntimeException("queue is empty"); - } - } @Test public void testDimensions() throws Exception { + InputRow inputRow; + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); + Firehose firehose = webbie.connect(); + if (firehose.hasMore()) { + inputRow = firehose.nextRow(); + } else { + throw new RuntimeException("queue is empty"); + } List actualAnswer = inputRow.getDimensions(); Collections.sort(actualAnswer); Assert.assertEquals(actualAnswer, dimensions); @@ -74,6 +71,15 @@ public class WebFirehoseFactoryTest @Test public void testPosixTimeStamp() throws Exception { + InputRow inputRow; + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); + Firehose firehose = webbie.connect(); + if (firehose.hasMore()) { + inputRow = firehose.nextRow(); + } else { + throw new RuntimeException("queue is empty"); + } long expectedTime = 1372121562L * 1000L; Assert.assertEquals(expectedTime, inputRow.getTimestampFromEpoch()); } @@ -87,8 +93,8 @@ public class WebFirehoseFactoryTest + "\"time\":\"2013-07-08\"}" ); - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); - webbie = new WebFirehoseFactory(updateStreamFactory, "iso"); + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "iso"); Firehose firehose1 = webbie.connect(); if (firehose1.hasMore()) { long milliSeconds = firehose1.nextRow().getTimestampFromEpoch(); @@ -108,8 +114,8 @@ public class WebFirehoseFactoryTest + "\"time\":\"2013-07-08\"}" ); - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); - webbie = new WebFirehoseFactory(updateStreamFactory, null); + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), null); Firehose firehose2 = webbie.connect(); if (firehose2.hasMore()) { long milliSeconds = firehose2.nextRow().getTimestampFromEpoch(); @@ -129,8 +135,8 @@ public class WebFirehoseFactoryTest + "\"time\":1373241600000}" ); - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, null, "time"); - webbie = new WebFirehoseFactory(updateStreamFactory, null); + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), null); Firehose firehose3 = webbie.connect(); if (firehose3.hasMore()) { long milliSeconds = firehose3.nextRow().getTimestampFromEpoch(); @@ -142,16 +148,36 @@ public class WebFirehoseFactoryTest } @Test - public void testGetDimension() + public void testGetDimension() throws Exception { + InputRow inputRow; + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); + Firehose firehose = webbie.connect(); + if (firehose.hasMore()) { + inputRow = firehose.nextRow(); + } else { + throw new RuntimeException("queue is empty"); + } + List column1 = Lists.newArrayList(); column1.add("value1"); Assert.assertEquals(column1, inputRow.getDimension("item1")); } @Test - public void testGetFloatMetric() + public void testGetFloatMetric() throws Exception { + InputRow inputRow; + UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); + webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); + Firehose firehose = webbie.connect(); + if (firehose.hasMore()) { + inputRow = firehose.nextRow(); + } else { + throw new RuntimeException("queue is empty"); + } + Assert.assertEquals((float) 2.0, inputRow.getFloatMetric("item2")); } } diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java index b8a1cd5f7a0..105a51204b8 100644 --- a/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplier.java @@ -31,7 +31,6 @@ import java.util.List; public class WebJsonSupplier implements InputSupplier { - private List dimensions; private static final EmittingLogger log = new EmittingLogger(WebJsonSupplier.class); private String urlString; diff --git a/examples/src/main/java/druid/examples/webStream/WebJsonSupplierTest.java b/examples/src/main/java/druid/examples/webStream/WebJsonSupplierTest.java new file mode 100644 index 00000000000..c5b880b5f0a --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/WebJsonSupplierTest.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.webStream; + +import org.testng.annotations.Test; + +import java.net.UnknownHostException; + +public class WebJsonSupplierTest +{ + @Test(expectedExceptions = UnknownHostException.class) + public void checkInvalidUrl() throws Exception + { + + String invalidURL = "http://invalid.url"; + WebJsonSupplier supplier = new WebJsonSupplier(invalidURL); + supplier.getInput(); + } +} From 41cb115d6039b4e554954b0a7d58f0e4a3708b17 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 10:46:43 -0700 Subject: [PATCH 19/25] few more changes to RenamingKeysUpdateStreamFactory and test --- .../examples/webStream/RenamingKeysUpdateStreamFactory.java | 4 +++- .../examples/webStream/RenamingKeysUpdateStreamTest.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java index 4dafc3becd3..bd8fa957c44 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java @@ -1,4 +1,4 @@ -package druid.examples.webStream;/* +/* * Druid - a distributed column store. * Copyright (C) 2012 Metamarkets Group Inc. * @@ -17,6 +17,8 @@ package druid.examples.webStream;/* * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +package druid.examples.webStream; + import java.util.Map; public class RenamingKeysUpdateStreamFactory diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java index d4acd52fabc..24917b56374 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java @@ -1,4 +1,4 @@ -package druid.examples.webStream;/* +/* * Druid - a distributed column store. * Copyright (C) 2012 Metamarkets Group Inc. * @@ -17,6 +17,8 @@ package druid.examples.webStream;/* * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +package druid.examples.webStream; + import com.google.common.io.InputSupplier; import junit.framework.Assert; import org.testng.annotations.BeforeClass; From 6d000fc4c283ea71316913c8c103066008dd8c54 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 15:31:03 -0700 Subject: [PATCH 20/25] interfaces added and tests simplified --- .../webStream/InputSupplierUpdateStream.java | 103 +++++++++++ .../InputSupplierUpdateStreamFactory.java | 42 +++++ ...ava => InputSupplierUpdateStreamTest.java} | 10 +- .../webStream/RenamingKeysUpdateStream.java | 6 +- .../RenamingKeysUpdateStreamFactory.java | 6 +- .../RenamingKeysUpdateStreamTest.java | 6 +- .../examples/webStream/UpdateStream.java | 80 +------- .../webStream/UpdateStreamFactory.java | 20 +- .../webStream/WebFirehoseFactory.java | 15 +- .../webStream/WebFirehoseFactoryTest.java | 172 +++++++++++++----- 10 files changed, 300 insertions(+), 160 deletions(-) create mode 100644 examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java create mode 100644 examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamFactory.java rename examples/src/main/java/druid/examples/webStream/{UpdateStreamTest.java => InputSupplierUpdateStreamTest.java} (91%) diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java new file mode 100644 index 00000000000..c51121ac9c9 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java @@ -0,0 +1,103 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.webStream; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.io.InputSupplier; +import com.metamx.druid.jackson.DefaultObjectMapper; +import com.metamx.emitter.EmittingLogger; + +import java.io.BufferedReader; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +public class InputSupplierUpdateStream implements UpdateStream +{ + private static final EmittingLogger log = new EmittingLogger(InputSupplierUpdateStream.class); + private static final long queueWaitTime = 15L; + private final TypeReference> typeRef; + private final InputSupplier supplier; + private final int QUEUE_SIZE = 10000; + private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); + private final ObjectMapper mapper = new DefaultObjectMapper(); + private final String timeDimension; + + public InputSupplierUpdateStream( + InputSupplier supplier, + String timeDimension + ) + { + this.supplier = supplier; + this.typeRef = new TypeReference>() + { + }; + this.timeDimension = timeDimension; + } + + private boolean isValid(String s) + { + return !(s.isEmpty()); + } + + @Override + public void run() + { + try { + BufferedReader reader = supplier.getInput(); + String line; + while ((line = reader.readLine()) != null) { + if (isValid(line)) { + HashMap map = mapper.readValue(line, typeRef); + if (map.get(timeDimension) != null) { + queue.offer(map, queueWaitTime, TimeUnit.SECONDS); + log.debug("Successfully added to queue"); + } else { + log.error("missing timestamp"); + } + } + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + } + + + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return queue.poll(waitTime, unit); + } + + public int getQueueSize() + { + return queue.size(); + } + + public String getTimeDimension(){ + return timeDimension; + } + +} diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamFactory.java new file mode 100644 index 00000000000..1b8bb2473e8 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamFactory.java @@ -0,0 +1,42 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.webStream; + +import com.google.common.io.InputSupplier; + +import java.io.BufferedReader; + +public class InputSupplierUpdateStreamFactory implements UpdateStreamFactory +{ + private final InputSupplier inputSupplier; + private final String timeDimension; + + public InputSupplierUpdateStreamFactory(InputSupplier inputSupplier, String timeDimension) + { + this.inputSupplier = inputSupplier; + this.timeDimension = timeDimension; + } + + public InputSupplierUpdateStream build() + { + return new InputSupplierUpdateStream(inputSupplier, timeDimension); + } + +} diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java similarity index 91% rename from examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java rename to examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java index a14993ff9a7..654a8e26a73 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStreamTest.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java @@ -29,9 +29,9 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -public class UpdateStreamTest +public class InputSupplierUpdateStreamTest { - private final long waitTime = 15L; + private final long waitTime = 1L; private final TimeUnit unit = TimeUnit.SECONDS; private final ArrayList dimensions = new ArrayList(); private InputSupplier testCaseSupplier; @@ -61,7 +61,7 @@ public class UpdateStreamTest @Test public void basicIngestionCheck() throws Exception { - UpdateStream updateStream = new UpdateStream( + InputSupplierUpdateStream updateStream = new InputSupplierUpdateStream( testCaseSupplier, timeDimension ); @@ -79,7 +79,7 @@ public class UpdateStreamTest + "\"item2\":2}" ); - UpdateStream updateStream = new UpdateStream( + InputSupplierUpdateStream updateStream = new InputSupplierUpdateStream( testCaseSupplier, timeDimension ); @@ -95,7 +95,7 @@ public class UpdateStreamTest "{\"item1\": \"value1\"," + "\"time\":1372121562 }" ); - UpdateStream updateStream = new UpdateStream( + InputSupplierUpdateStream updateStream = new InputSupplierUpdateStream( testCaseSupplier, timeDimension ); diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java index 922839f493e..c7ed5c8113a 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java @@ -24,14 +24,14 @@ import com.google.common.collect.Maps; import java.util.Map; import java.util.concurrent.TimeUnit; -public class RenamingKeysUpdateStream implements Runnable +public class RenamingKeysUpdateStream implements UpdateStream { - private final UpdateStream updateStream; + private final InputSupplierUpdateStream updateStream; private Map renamedDimensions; public RenamingKeysUpdateStream( - UpdateStream updateStream, + InputSupplierUpdateStream updateStream, Map renamedDimensions ) { diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java index bd8fa957c44..ef5d2e6e3f1 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamFactory.java @@ -21,12 +21,12 @@ package druid.examples.webStream; import java.util.Map; -public class RenamingKeysUpdateStreamFactory +public class RenamingKeysUpdateStreamFactory implements UpdateStreamFactory { - private UpdateStreamFactory updateStreamFactory; + private InputSupplierUpdateStreamFactory updateStreamFactory; private Map renamedDimensions; - public RenamingKeysUpdateStreamFactory(UpdateStreamFactory updateStreamFactory, Map renamedDimensions) + public RenamingKeysUpdateStreamFactory(InputSupplierUpdateStreamFactory updateStreamFactory, Map renamedDimensions) { this.updateStreamFactory = updateStreamFactory; this.renamedDimensions = renamedDimensions; diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java index 24917b56374..449ad4b4a90 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java @@ -49,7 +49,7 @@ public class RenamingKeysUpdateStreamTest @Test public void testPollFromQueue() throws Exception { - UpdateStream updateStream = new UpdateStream(testCaseSupplier, timeDimension); + InputSupplierUpdateStream updateStream = new InputSupplierUpdateStream(testCaseSupplier, timeDimension); Map renamedKeys = new HashMap(); renamedKeys.put("item1", "i1"); renamedKeys.put("item2", "i2"); @@ -69,7 +69,7 @@ public class RenamingKeysUpdateStreamTest @Test public void testGetTimeDimension() throws Exception { - UpdateStream updateStream = new UpdateStream(testCaseSupplier, timeDimension); + InputSupplierUpdateStream updateStream = new InputSupplierUpdateStream(testCaseSupplier, timeDimension); Map renamedKeys = new HashMap(); renamedKeys.put("item1", "i1"); renamedKeys.put("item2", "i2"); @@ -82,7 +82,7 @@ public class RenamingKeysUpdateStreamTest @Test public void testMissingTimeRename() throws Exception { - UpdateStream updateStream = new UpdateStream(testCaseSupplier, timeDimension); + InputSupplierUpdateStream updateStream = new InputSupplierUpdateStream(testCaseSupplier, timeDimension); Map renamedKeys = new HashMap(); renamedKeys.put("item1", "i1"); renamedKeys.put("item2", "i2"); diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 3a19e11b6b5..45818eb64a9 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -16,89 +16,15 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - package druid.examples.webStream; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Throwables; -import com.google.common.collect.Maps; -import com.google.common.io.InputSupplier; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.emitter.EmittingLogger; - -import java.io.BufferedReader; -import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; -public class UpdateStream implements Runnable +public interface UpdateStream extends Runnable { - private static final EmittingLogger log = new EmittingLogger(UpdateStream.class); - private static final long queueWaitTime = 15L; - private final TypeReference> typeRef; - private final InputSupplier supplier; - private final int QUEUE_SIZE = 10000; - private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); - private final ObjectMapper mapper = new DefaultObjectMapper(); - private final String timeDimension; + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException; + public String getTimeDimension(); - public UpdateStream( - InputSupplier supplier, - String timeDimension - ) - { - this.supplier = supplier; - this.typeRef = new TypeReference>() - { - }; - this.timeDimension = timeDimension; - } - - private boolean isValid(String s) - { - return !(s.isEmpty()); - } - - @Override - public void run() - { - try { - BufferedReader reader = supplier.getInput(); - String line; - while ((line = reader.readLine()) != null) { - if (isValid(line)) { - HashMap map = mapper.readValue(line, typeRef); - if (map.get(timeDimension) != null) { - queue.offer(map, queueWaitTime, TimeUnit.SECONDS); - log.debug("Successfully added to queue"); - } else { - log.error("missing timestamp"); - } - } - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - - } - - - public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException - { - return queue.poll(waitTime, unit); - } - - public int getQueueSize() - { - return queue.size(); - } - - public String getTimeDimension(){ - return timeDimension; - } } diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java b/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java index 2a69eb42c1d..f2ce1ccae89 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStreamFactory.java @@ -16,25 +16,9 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - package druid.examples.webStream; -import com.google.common.io.InputSupplier; - -public class UpdateStreamFactory +public interface UpdateStreamFactory { - private final InputSupplier inputSupplier; - private final String timeDimension; - - public UpdateStreamFactory(InputSupplier inputSupplier, String timeDimension) - { - this.inputSupplier = inputSupplier; - this.timeDimension = timeDimension; - } - - public UpdateStream build() - { - return new UpdateStream(inputSupplier, timeDimension); - } - + public UpdateStream build(); } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index a1285ee18e4..24aff866cfe 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -34,7 +34,6 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,7 +44,7 @@ public class WebFirehoseFactory implements FirehoseFactory { private static final EmittingLogger log = new EmittingLogger(WebFirehoseFactory.class); private final String timeFormat; - private final RenamingKeysUpdateStreamFactory factory; + private final UpdateStreamFactory factory; private final long queueWaitTime = 15L; @JsonCreator @@ -58,13 +57,13 @@ public class WebFirehoseFactory implements FirehoseFactory { this( new RenamingKeysUpdateStreamFactory( - new UpdateStreamFactory(new WebJsonSupplier(url), timeDimension), + new InputSupplierUpdateStreamFactory(new WebJsonSupplier(url), timeDimension), renamedDimensions ), timeFormat ); } - public WebFirehoseFactory(RenamingKeysUpdateStreamFactory factory, String timeFormat) + public WebFirehoseFactory(UpdateStreamFactory factory, String timeFormat) { this.factory = factory; if (timeFormat == null) { @@ -78,9 +77,9 @@ public class WebFirehoseFactory implements FirehoseFactory public Firehose connect() throws IOException { - final RenamingKeysUpdateStream renamingUpdateStream = factory.build(); + final UpdateStream updateStream = factory.build(); final ExecutorService service = Executors.newSingleThreadExecutor(); - service.submit(renamingUpdateStream); + service.submit(updateStream); return new Firehose() { @@ -91,7 +90,7 @@ public class WebFirehoseFactory implements FirehoseFactory public boolean hasMore() { try { - map = renamingUpdateStream.pollFromQueue(queueWaitTime, TimeUnit.SECONDS); + map = updateStream.pollFromQueue(queueWaitTime, TimeUnit.SECONDS); return map != null; } catch (InterruptedException e) { @@ -105,7 +104,7 @@ public class WebFirehoseFactory implements FirehoseFactory { try { DateTime date = TimestampParser.createTimestampParser(timeFormat) - .apply(map.get(renamingUpdateStream.getTimeDimension()).toString()); + .apply(map.get(updateStream.getTimeDimension()).toString()); return new MapBasedInputRow( date.getMillis(), new ArrayList(map.keySet()), diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index e51ef9e79ff..9b63f361380 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -20,6 +20,7 @@ package druid.examples.webStream; import com.beust.jcommander.internal.Lists; +import com.google.common.collect.ImmutableMap; import com.metamx.druid.input.InputRow; import com.metamx.druid.realtime.firehose.Firehose; import junit.framework.Assert; @@ -27,36 +28,91 @@ import org.joda.time.DateTime; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; public class WebFirehoseFactoryTest { - private List dimensions; + private List dimensions = Lists.newArrayList(); private WebFirehoseFactory webbie; - private TestCaseSupplier testCaseSupplier; + private WebFirehoseFactory webbie1; @BeforeClass public void setUp() throws Exception { - dimensions = new ArrayList(); dimensions.add("item1"); dimensions.add("item2"); dimensions.add("time"); - testCaseSupplier = new TestCaseSupplier( - "{\"item1\":\"value1\"," - + "\"item2\":2," - + "\"time\":1372121562 }" + webbie = new WebFirehoseFactory( + new UpdateStreamFactory() + { + @Override + public UpdateStream build() + { + return new UpdateStream() + { + @Override + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return ImmutableMap.of("item1", "value1", "item2", 2, "time", "1372121562"); + } + + @Override + public String getTimeDimension() + { + return "time"; + } + + @Override + public void run() + { + + } + }; + } + }, + "posix" ); + + webbie1 = new WebFirehoseFactory( + new UpdateStreamFactory() + { + @Override + public UpdateStream build() + { + return new UpdateStream() + { + @Override + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return ImmutableMap.of("item1", "value1", "item2", 2, "time", "1373241600000"); + } + + @Override + public String getTimeDimension() + { + return "time"; + } + + @Override + public void run() + { + + } + }; + } + }, + "auto" + ); + } @Test public void testDimensions() throws Exception { InputRow inputRow; - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); Firehose firehose = webbie.connect(); if (firehose.hasMore()) { inputRow = firehose.nextRow(); @@ -72,8 +128,6 @@ public class WebFirehoseFactoryTest public void testPosixTimeStamp() throws Exception { InputRow inputRow; - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); Firehose firehose = webbie.connect(); if (firehose.hasMore()) { inputRow = firehose.nextRow(); @@ -87,15 +141,37 @@ public class WebFirehoseFactoryTest @Test public void testISOTimeStamp() throws Exception { - testCaseSupplier = new TestCaseSupplier( - "{\"item1\": \"value1\"," - + "\"item2\":2," - + "\"time\":\"2013-07-08\"}" - ); + WebFirehoseFactory webbie4 = new WebFirehoseFactory( + new UpdateStreamFactory() + { + @Override + public UpdateStream build() + { + return new UpdateStream() + { + @Override + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return ImmutableMap.of("item1", "value1", "item2", 2, "time", "2013-07-08"); + } - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "iso"); - Firehose firehose1 = webbie.connect(); + @Override + public String getTimeDimension() + { + return "time"; + } + + @Override + public void run() + { + + } + }; + } + }, + "auto" + ); + Firehose firehose1 = webbie4.connect(); if (firehose1.hasMore()) { long milliSeconds = firehose1.nextRow().getTimestampFromEpoch(); DateTime date = new DateTime("2013-07-08"); @@ -108,15 +184,37 @@ public class WebFirehoseFactoryTest @Test public void testAutoIsoTimeStamp() throws Exception { - testCaseSupplier = new TestCaseSupplier( - "{\"item1\": \"value1\"," - + "\"item2\":2," - + "\"time\":\"2013-07-08\"}" - ); + WebFirehoseFactory webbie5 = new WebFirehoseFactory( + new UpdateStreamFactory() + { + @Override + public UpdateStream build() + { + return new UpdateStream() + { + @Override + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return ImmutableMap.of("item1", "value1", "item2", 2, "time", "2013-07-08"); + } - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), null); - Firehose firehose2 = webbie.connect(); + @Override + public String getTimeDimension() + { + return "time"; + } + + @Override + public void run() + { + + } + }; + } + }, + null + ); + Firehose firehose2 = webbie5.connect(); if (firehose2.hasMore()) { long milliSeconds = firehose2.nextRow().getTimestampFromEpoch(); DateTime date = new DateTime("2013-07-08"); @@ -129,15 +227,7 @@ public class WebFirehoseFactoryTest @Test public void testAutoMilliSecondsTimeStamp() throws Exception { - testCaseSupplier = new TestCaseSupplier( - "{\"item1\": \"value1\"," - + "\"item2\":2," - + "\"time\":1373241600000}" - ); - - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), null); - Firehose firehose3 = webbie.connect(); + Firehose firehose3 = webbie1.connect(); if (firehose3.hasMore()) { long milliSeconds = firehose3.nextRow().getTimestampFromEpoch(); DateTime date = new DateTime("2013-07-08"); @@ -151,9 +241,7 @@ public class WebFirehoseFactoryTest public void testGetDimension() throws Exception { InputRow inputRow; - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); - Firehose firehose = webbie.connect(); + Firehose firehose = webbie1.connect(); if (firehose.hasMore()) { inputRow = firehose.nextRow(); } else { @@ -169,9 +257,7 @@ public class WebFirehoseFactoryTest public void testGetFloatMetric() throws Exception { InputRow inputRow; - UpdateStreamFactory updateStreamFactory = new UpdateStreamFactory(testCaseSupplier, "time"); - webbie = new WebFirehoseFactory(new RenamingKeysUpdateStreamFactory(updateStreamFactory, null), "posix"); - Firehose firehose = webbie.connect(); + Firehose firehose = webbie1.connect(); if (firehose.hasMore()) { inputRow = firehose.nextRow(); } else { From 3250c698bba9e1dd88c1db772b39e0c16db46c26 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 16:34:55 -0700 Subject: [PATCH 21/25] fixed thread stuff and made tests cleaner --- .../webStream/InputSupplierUpdateStream.java | 46 ++++--- .../InputSupplierUpdateStreamTest.java | 6 +- .../webStream/RenamingKeysUpdateStream.java | 10 +- .../RenamingKeysUpdateStreamTest.java | 2 +- .../examples/webStream/StoppableThread.java | 27 ++++ .../examples/webStream/UpdateStream.java | 5 +- .../webStream/WebFirehoseFactory.java | 7 +- .../webStream/WebFirehoseFactoryTest.java | 122 ++++++------------ 8 files changed, 108 insertions(+), 117 deletions(-) create mode 100644 examples/src/main/java/druid/examples/webStream/StoppableThread.java diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java index c51121ac9c9..8a1066bcdea 100644 --- a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java @@ -43,7 +43,7 @@ public class InputSupplierUpdateStream implements UpdateStream private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); private final ObjectMapper mapper = new DefaultObjectMapper(); private final String timeDimension; - + private StoppableThread addToQueueThread; public InputSupplierUpdateStream( InputSupplier supplier, String timeDimension @@ -61,30 +61,40 @@ public class InputSupplierUpdateStream implements UpdateStream return !(s.isEmpty()); } - @Override - public void run() + public void start() { - try { - BufferedReader reader = supplier.getInput(); - String line; - while ((line = reader.readLine()) != null) { - if (isValid(line)) { - HashMap map = mapper.readValue(line, typeRef); - if (map.get(timeDimension) != null) { - queue.offer(map, queueWaitTime, TimeUnit.SECONDS); - log.debug("Successfully added to queue"); - } else { - log.error("missing timestamp"); + addToQueueThread = new StoppableThread(){ + public void run(){ + while(!finished) + try { + BufferedReader reader = supplier.getInput(); + String line; + while ((line = reader.readLine()) != null) { + if (isValid(line)) { + HashMap map = mapper.readValue(line, typeRef); + if (map.get(timeDimension) != null) { + queue.offer(map, queueWaitTime, TimeUnit.SECONDS); + log.debug("Successfully added to queue"); + } else { + log.error("missing timestamp"); + } + } } } + catch (Exception e) { + throw Throwables.propagate(e); + } } - } - catch (Exception e) { - throw Throwables.propagate(e); - } + }; + addToQueueThread.setDaemon(true); + addToQueueThread.start(); } + public void stop(){ + addToQueueThread.stopMe(); + } + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException { diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java index 654a8e26a73..8499661de2d 100644 --- a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStreamTest.java @@ -65,7 +65,7 @@ public class InputSupplierUpdateStreamTest testCaseSupplier, timeDimension ); - updateStream.run(); + updateStream.start(); Map insertedRow = updateStream.pollFromQueue(waitTime, unit); Assert.assertEquals(expectedAnswer, insertedRow); } @@ -83,7 +83,7 @@ public class InputSupplierUpdateStreamTest testCaseSupplier, timeDimension ); - updateStream.run(); + updateStream.start(); Assert.assertEquals(updateStream.getQueueSize(), 0); } @@ -99,7 +99,7 @@ public class InputSupplierUpdateStreamTest testCaseSupplier, timeDimension ); - updateStream.run(); + updateStream.start(); Map insertedRow = updateStream.pollFromQueue(waitTime, unit); Map expectedAnswer = new HashMap(); expectedAnswer.put("item1", "value1"); diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java index c7ed5c8113a..3b6596e344e 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStream.java @@ -70,10 +70,12 @@ public class RenamingKeysUpdateStream implements UpdateStream } - - @Override - public void run() + public void start() { - updateStream.run(); + updateStream.start(); + } + + public void stop(){ + updateStream.stop(); } } diff --git a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java index 449ad4b4a90..53ed20a238e 100644 --- a/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java +++ b/examples/src/main/java/druid/examples/webStream/RenamingKeysUpdateStreamTest.java @@ -56,7 +56,7 @@ public class RenamingKeysUpdateStreamTest renamedKeys.put("time", "t"); RenamingKeysUpdateStream renamer = new RenamingKeysUpdateStream(updateStream, renamedKeys); - renamer.run(); + renamer.start(); Map expectedAnswer = new HashMap(); expectedAnswer.put("i1", "value1"); expectedAnswer.put("i2", 2); diff --git a/examples/src/main/java/druid/examples/webStream/StoppableThread.java b/examples/src/main/java/druid/examples/webStream/StoppableThread.java new file mode 100644 index 00000000000..4b2f054af70 --- /dev/null +++ b/examples/src/main/java/druid/examples/webStream/StoppableThread.java @@ -0,0 +1,27 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package druid.examples.webStream; + +public class StoppableThread extends Thread +{ + volatile boolean finished=false; + public void stopMe(){ + finished=true; + } +} diff --git a/examples/src/main/java/druid/examples/webStream/UpdateStream.java b/examples/src/main/java/druid/examples/webStream/UpdateStream.java index 45818eb64a9..bd73c4e05e4 100644 --- a/examples/src/main/java/druid/examples/webStream/UpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/UpdateStream.java @@ -21,10 +21,11 @@ package druid.examples.webStream; import java.util.Map; import java.util.concurrent.TimeUnit; -public interface UpdateStream extends Runnable +public interface UpdateStream { public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException; public String getTimeDimension(); - + public void start(); + public void stop(); } diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java index 24aff866cfe..12c80012f28 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactory.java @@ -35,8 +35,6 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayList; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @JsonTypeName("webstream") @@ -78,8 +76,7 @@ public class WebFirehoseFactory implements FirehoseFactory { final UpdateStream updateStream = factory.build(); - final ExecutorService service = Executors.newSingleThreadExecutor(); - service.submit(updateStream); + updateStream.start(); return new Firehose() { @@ -129,7 +126,7 @@ public class WebFirehoseFactory implements FirehoseFactory @Override public void close() throws IOException { - service.shutdown(); + updateStream.stop(); } }; diff --git a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java index 9b63f361380..55018934fdf 100644 --- a/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java +++ b/examples/src/main/java/druid/examples/webStream/WebFirehoseFactoryTest.java @@ -51,26 +51,7 @@ public class WebFirehoseFactoryTest @Override public UpdateStream build() { - return new UpdateStream() - { - @Override - public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException - { - return ImmutableMap.of("item1", "value1", "item2", 2, "time", "1372121562"); - } - - @Override - public String getTimeDimension() - { - return "time"; - } - - @Override - public void run() - { - - } - }; + return new MyUpdateStream(ImmutableMap.of("item1", "value1", "item2", 2, "time", "1372121562")); } }, "posix" @@ -82,26 +63,7 @@ public class WebFirehoseFactoryTest @Override public UpdateStream build() { - return new UpdateStream() - { - @Override - public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException - { - return ImmutableMap.of("item1", "value1", "item2", 2, "time", "1373241600000"); - } - - @Override - public String getTimeDimension() - { - return "time"; - } - - @Override - public void run() - { - - } - }; + return new MyUpdateStream(ImmutableMap.of("item1", "value1", "item2", 2, "time", "1373241600000")); } }, "auto" @@ -141,37 +103,18 @@ public class WebFirehoseFactoryTest @Test public void testISOTimeStamp() throws Exception { - WebFirehoseFactory webbie4 = new WebFirehoseFactory( + WebFirehoseFactory webbie3 = new WebFirehoseFactory( new UpdateStreamFactory() { @Override public UpdateStream build() { - return new UpdateStream() - { - @Override - public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException - { - return ImmutableMap.of("item1", "value1", "item2", 2, "time", "2013-07-08"); - } - - @Override - public String getTimeDimension() - { - return "time"; - } - - @Override - public void run() - { - - } - }; + return new MyUpdateStream(ImmutableMap.of("item1", "value1", "item2", 2, "time", "2013-07-08")); } }, "auto" ); - Firehose firehose1 = webbie4.connect(); + Firehose firehose1 = webbie3.connect(); if (firehose1.hasMore()) { long milliSeconds = firehose1.nextRow().getTimestampFromEpoch(); DateTime date = new DateTime("2013-07-08"); @@ -184,37 +127,18 @@ public class WebFirehoseFactoryTest @Test public void testAutoIsoTimeStamp() throws Exception { - WebFirehoseFactory webbie5 = new WebFirehoseFactory( + WebFirehoseFactory webbie2 = new WebFirehoseFactory( new UpdateStreamFactory() { @Override public UpdateStream build() { - return new UpdateStream() - { - @Override - public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException - { - return ImmutableMap.of("item1", "value1", "item2", 2, "time", "2013-07-08"); - } - - @Override - public String getTimeDimension() - { - return "time"; - } - - @Override - public void run() - { - - } - }; + return new MyUpdateStream(ImmutableMap.of("item1", "value1", "item2", 2, "time", "2013-07-08")); } }, null ); - Firehose firehose2 = webbie5.connect(); + Firehose firehose2 = webbie2.connect(); if (firehose2.hasMore()) { long milliSeconds = firehose2.nextRow().getTimestampFromEpoch(); DateTime date = new DateTime("2013-07-08"); @@ -266,4 +190,34 @@ public class WebFirehoseFactoryTest Assert.assertEquals((float) 2.0, inputRow.getFloatMetric("item2")); } + + private static class MyUpdateStream implements UpdateStream + { + private static ImmutableMap map; + public MyUpdateStream(ImmutableMap map){ + this.map=map; + } + + @Override + public Map pollFromQueue(long waitTime, TimeUnit unit) throws InterruptedException + { + return map; + } + + @Override + public String getTimeDimension() + { + return "time"; + } + + @Override + public void start() + { + } + + @Override + public void stop() + { + } + } } From b3157c275294fea7b752a94637814a8d2c2243d7 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 16:42:35 -0700 Subject: [PATCH 22/25] made thread final --- .../webStream/InputSupplierUpdateStream.java | 66 +++++++++++-------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java index 8a1066bcdea..b26f241414c 100644 --- a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java @@ -43,12 +43,42 @@ public class InputSupplierUpdateStream implements UpdateStream private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); private final ObjectMapper mapper = new DefaultObjectMapper(); private final String timeDimension; - private StoppableThread addToQueueThread; + private final StoppableThread addToQueueThread; + public InputSupplierUpdateStream( - InputSupplier supplier, - String timeDimension + final InputSupplier supplier, + final String timeDimension ) { + addToQueueThread = new StoppableThread() + { + public void run() + { + while (!finished) { + try { + BufferedReader reader = supplier.getInput(); + String line; + while ((line = reader.readLine()) != null) { + if (isValid(line)) { + HashMap map = mapper.readValue(line, typeRef); + if (map.get(timeDimension) != null) { + queue.offer(map, queueWaitTime, TimeUnit.SECONDS); + log.debug("Successfully added to queue"); + } else { + log.error("missing timestamp"); + } + } + } + } + + catch (Exception e) { + throw Throwables.propagate(e); + } + } + } + }; + addToQueueThread.setDaemon(true); + this.supplier = supplier; this.typeRef = new TypeReference>() { @@ -63,35 +93,12 @@ public class InputSupplierUpdateStream implements UpdateStream public void start() { - addToQueueThread = new StoppableThread(){ - public void run(){ - while(!finished) - try { - BufferedReader reader = supplier.getInput(); - String line; - while ((line = reader.readLine()) != null) { - if (isValid(line)) { - HashMap map = mapper.readValue(line, typeRef); - if (map.get(timeDimension) != null) { - queue.offer(map, queueWaitTime, TimeUnit.SECONDS); - log.debug("Successfully added to queue"); - } else { - log.error("missing timestamp"); - } - } - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - }; - addToQueueThread.setDaemon(true); addToQueueThread.start(); } - public void stop(){ + public void stop() + { addToQueueThread.stopMe(); } @@ -106,7 +113,8 @@ public class InputSupplierUpdateStream implements UpdateStream return queue.size(); } - public String getTimeDimension(){ + public String getTimeDimension() + { return timeDimension; } From 9f7284c8010d6845e28e402909456294e2c1a1e7 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 16:59:49 -0700 Subject: [PATCH 23/25] fixed thread stuff --- .../webStream/InputSupplierUpdateStream.java | 8 +++--- .../examples/webStream/StoppableThread.java | 27 ------------------- 2 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 examples/src/main/java/druid/examples/webStream/StoppableThread.java diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java index b26f241414c..0d89ffd3322 100644 --- a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java @@ -43,18 +43,18 @@ public class InputSupplierUpdateStream implements UpdateStream private final BlockingQueue> queue = new ArrayBlockingQueue>(QUEUE_SIZE); private final ObjectMapper mapper = new DefaultObjectMapper(); private final String timeDimension; - private final StoppableThread addToQueueThread; + private final Thread addToQueueThread; public InputSupplierUpdateStream( final InputSupplier supplier, final String timeDimension ) { - addToQueueThread = new StoppableThread() + addToQueueThread = new Thread() { public void run() { - while (!finished) { + while (!isInterrupted()) { try { BufferedReader reader = supplier.getInput(); String line; @@ -99,7 +99,7 @@ public class InputSupplierUpdateStream implements UpdateStream public void stop() { - addToQueueThread.stopMe(); + addToQueueThread.interrupt(); } diff --git a/examples/src/main/java/druid/examples/webStream/StoppableThread.java b/examples/src/main/java/druid/examples/webStream/StoppableThread.java deleted file mode 100644 index 4b2f054af70..00000000000 --- a/examples/src/main/java/druid/examples/webStream/StoppableThread.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package druid.examples.webStream; - -public class StoppableThread extends Thread -{ - volatile boolean finished=false; - public void stopMe(){ - finished=true; - } -} From ba484fca5cb794f0e770ae5a8a8850d517a0e18b Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 17:17:21 -0700 Subject: [PATCH 24/25] catch error --- .../druid/examples/webStream/InputSupplierUpdateStream.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java index 0d89ffd3322..d5138773ed2 100644 --- a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java @@ -21,7 +21,6 @@ package druid.examples.webStream; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Throwables; import com.google.common.io.InputSupplier; import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.emitter.EmittingLogger; @@ -65,14 +64,14 @@ public class InputSupplierUpdateStream implements UpdateStream queue.offer(map, queueWaitTime, TimeUnit.SECONDS); log.debug("Successfully added to queue"); } else { - log.error("missing timestamp"); + log.info("missing timestamp"); } } } } catch (Exception e) { - throw Throwables.propagate(e); + log.info(e,e.getMessage()); } } } From bd6dcd397386c4f915335e6c1468a2340a034513 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Tue, 9 Jul 2013 17:39:19 -0700 Subject: [PATCH 25/25] interruption! --- .../webStream/InputSupplierUpdateStream.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java index d5138773ed2..dc846261f9e 100644 --- a/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java +++ b/examples/src/main/java/druid/examples/webStream/InputSupplierUpdateStream.java @@ -19,13 +19,16 @@ package druid.examples.webStream; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.InputSupplier; import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.emitter.EmittingLogger; import java.io.BufferedReader; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; @@ -70,8 +73,18 @@ public class InputSupplierUpdateStream implements UpdateStream } } - catch (Exception e) { - log.info(e,e.getMessage()); + catch (InterruptedException e){ + log.info(e, "Thread adding events to the queue interrupted"); + return; + } + catch (JsonMappingException e) { + log.info(e, "Error in converting json to map"); + } + catch (JsonParseException e) { + log.info(e, "Error in parsing json"); + } + catch (IOException e) { + log.info(e, "Error in connecting to InputStream"); } } }