From 748695bf974cfdff30cd75118f7e8c7de698c2e2 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Thu, 20 Jun 2013 14:59:52 -0700 Subject: [PATCH] trying to recover my files on intellij --- .DS_Store | Bin 0 -> 15364 bytes .../com/metamx/druid/client/DruidServer.java | 1 + common/.DS_Store | Bin 0 -> 6148 bytes common/src/.DS_Store | Bin 0 -> 6148 bytes common/src/main/.DS_Store | Bin 0 -> 6148 bytes common/src/main/java/.DS_Store | Bin 0 -> 6148 bytes common/src/main/java/com/.DS_Store | Bin 0 -> 6148 bytes common/src/main/java/com/metamx/.DS_Store | Bin 0 -> 6148 bytes examples/.DS_Store | Bin 0 -> 6148 bytes examples/query.body | 12 + examples/search_query.body | 13 + .../main/java/druid/examples/HttpTest.java | 55 + .../druid/examples/twitter/Configuration.java | 59 + .../twitter/ConfigurationContext.java | 57 + .../twitter/ConfigurationFactory.java | 42 + .../twitter/HttpClientConfiguration.java | 46 + .../HttpClientWrapperConfiguration.java | 13 + .../druid/examples/twitter/HttpParameter.java | 288 ++++ .../druid/examples/twitter/HttpRequest.java | 120 ++ .../druid/examples/twitter/HttpResponse.java | 239 +++ .../examples/twitter/HttpResponseEvent.java | 92 ++ .../examples/twitter/HttpResponseImpl.java | 75 + .../twitter/HttpResponseListener.java | 10 + .../druid/examples/twitter/JSONArray.java | 726 ++++++++++ .../druid/examples/twitter/JSONObject.java | 1285 +++++++++++++++++ .../druid/examples/twitter/JSONTokener.java | 359 +++++ .../java/druid/examples/twitter/Logger.java | 180 +++ .../druid/examples/twitter/LoggerFactory.java | 34 + .../twitter/PropertyConfiguration.java | 325 +++++ .../twitter/PropertyConfigurationFactory.java | 61 + .../druid/examples/twitter/RequestMethod.java | 26 + .../twitter/StreamingGZipInputStream.java | 53 + .../TwitterSpritzerFirehoseFactory.java | 2 +- .../twitter/USGovFirehoseFactory.java | 173 +++ .../java/druid/examples/twitter/WebBase.java | 15 + .../druid/examples/twitter/WebBaseImpl.java | 170 +++ .../twitter/WebConfigurationBase.java | 494 +++++++ .../examples/twitter/WebStatusStreamBase.java | 176 +++ .../examples/twitter/WebStreamFactory.java | 111 ++ .../druid/examples/twitter/WebStreamImpl.java | 750 ++++++++++ .../twitter/z_InternalStringUtil.java | 111 ++ examples/twitter4j.properties | 5 + examples/twitter_realtime.spec | 44 + indexing-common/.DS_Store | Bin 0 -> 6148 bytes indexing-common/src/.DS_Store | Bin 0 -> 6148 bytes indexing-common/src/main/.DS_Store | Bin 0 -> 6148 bytes indexing-common/src/main/java/.DS_Store | Bin 0 -> 6148 bytes install/.DS_Store | Bin 0 -> 6148 bytes realtime/.DS_Store | Bin 0 -> 6148 bytes .../realtime/firehose/WebFirehoseFactory.java | 85 ++ server/.DS_Store | Bin 0 -> 6148 bytes services/.DS_Store | Bin 0 -> 6148 bytes 52 files changed, 6306 insertions(+), 1 deletion(-) create mode 100644 .DS_Store create mode 100644 common/.DS_Store create mode 100644 common/src/.DS_Store create mode 100644 common/src/main/.DS_Store create mode 100644 common/src/main/java/.DS_Store create mode 100644 common/src/main/java/com/.DS_Store create mode 100644 common/src/main/java/com/metamx/.DS_Store create mode 100644 examples/.DS_Store create mode 100644 examples/query.body create mode 100644 examples/search_query.body create mode 100644 examples/src/main/java/druid/examples/HttpTest.java create mode 100644 examples/src/main/java/druid/examples/twitter/Configuration.java create mode 100644 examples/src/main/java/druid/examples/twitter/ConfigurationContext.java create mode 100644 examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpParameter.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpRequest.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponse.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java create mode 100644 examples/src/main/java/druid/examples/twitter/HttpResponseListener.java create mode 100644 examples/src/main/java/druid/examples/twitter/JSONArray.java create mode 100644 examples/src/main/java/druid/examples/twitter/JSONObject.java create mode 100644 examples/src/main/java/druid/examples/twitter/JSONTokener.java create mode 100644 examples/src/main/java/druid/examples/twitter/Logger.java create mode 100644 examples/src/main/java/druid/examples/twitter/LoggerFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java create mode 100644 examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/RequestMethod.java create mode 100644 examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java create mode 100644 examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebBase.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebBaseImpl.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebStreamFactory.java create mode 100644 examples/src/main/java/druid/examples/twitter/WebStreamImpl.java create mode 100644 examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java create mode 100644 examples/twitter4j.properties create mode 100644 examples/twitter_realtime.spec create mode 100644 indexing-common/.DS_Store create mode 100644 indexing-common/src/.DS_Store create mode 100644 indexing-common/src/main/.DS_Store create mode 100644 indexing-common/src/main/java/.DS_Store create mode 100644 install/.DS_Store create mode 100644 realtime/.DS_Store create mode 100644 realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java create mode 100644 server/.DS_Store create mode 100644 services/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ee1df41e8fb05f1e8db6280875eaa40f45860a2c GIT binary patch literal 15364 zcmeI1&rcIU6vy9CXtxm~7mVIEo;VgCd?BXQo)|cp#|ztebe)+32k{G1cZPP5CTF#2wVpO_|E2H-3p{!g@6zc0z(44 zKG>)%i$)Ftsh191^cDbe5|4GkJ@x_G#x}BOfD(w zxFbqa6}m3fs7YPk>xXoIjL`L|fsaqksIJ<&Ot*5}X&JxYDlO0LbaT0%Y9ci`mCjh% ztTny6<2D`tc-22_I-S?0x_?}9om!>bexvWZ^%L$G(Qw#DrFPTvn+I%!ngV8@sxcV9@Fom73$3;e3m>Yu0O-XmfKQ!@{|MDVf655`WxhB8qP}V- zU0OIAs?i?p>s~eBsf&*b{|D$-Z}bQwzAV`HVAsGXJluDp<50{n+;&1glvLlZAgS*6 z`}w89GP0_0X;yJkeMH|;F3hUWc$pYaEVJ%tnvSLvaD5og!ze9|y0Su1|AV#Dsd^R6 z9Zumm;jEEo!eg(Ze0hX!{P5hm$crEszBIp8<7t z#y}~i9ZG>4xUR$o6N`Tab9@uCbAw(|i8i7bb-a6_cxUwb&=@`Uh=Uf^PSY0ru3%+& z5%;VE)C1}XgWN(WYdSbQzZ;2wa)?-e>LUHC4^lpl1^-c@FpkAPG5lDV(t07}Y)8(I z2+}FgOd2#1-K)nk-tWZoNseL1kXS<`E>^6>%C57Kv809>bMDLN-)uCA;m7b`S5Z<*_H>i+{(BtB{NJ=gB;30$x9)%Qou%d|(8wg2BQhTV;;4}ChzKzf0 z%M5lQ@mCrI#(zsJQZH+0u`r`!L9_ z6)%HwQFuYFJeNT^1CQe4EK7h-P4py=%Gr(GS*`kx)0Mr!pz6xia0vXQe_B8qkQ5*VNC8rS6i^D7148VP z9;5&%Kni?Q0QUz81<^HFYgAhYRCs+xe;W}EIKCwir9szVtr0w6T&Du+RBoOaT&F|7 zG;yxMTBA;9T+IyQn30tS5T5&#X{M>@)>{SrCExdv;EI12WwbU?ZY MC_$(q1%5$+PkJjxzW@LL literal 0 HcmV?d00001 diff --git a/common/src/.DS_Store b/common/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..73307bf9352f15b2d73524ac7d4be4c294bd50f0 GIT binary patch literal 6148 zcmeHK!AiqG5Z!H~Ca92$(BtB{NJ=gB;30$x9)%Qou%ZbO8;D71QhTV;;5Yam{*B+` z%0Qb%X6wBNT7SeMO6f^KhKF+cP_*6$v;;5Ki+nx2g?>Jr28w|>>SPzH5kNYQOX_=;F zznTY!N2llG;IVvqu9S`(WR|wCv4$5QpT~Ic7ip5EcQ9AS*KvfT04YEUkOHJYeL0xB zqy;HJ3XlTd6u{z9KtpsjRtjZZ2_2Tt=r<72fa6;NQD}5ERtmub##JhyO6B^A!BslU z3mxZbtQ4wr#^uhij^4R`yl}ZY%nKRLxT}zQQh*d_D$rI#8}I)!_{nU1kp*nQg0J5nPLnt<9Os=kn#RuVo@rU;!FUv>QMUB5c^c){{v=!aadaOB zvm42apqS@gFq59EpqPM1@?n-Gz{fgz6i3D6)^4p&)r&(b7IA7HMIuj2?w0aAbzAO%Q)`f@P$ zNefbd6d(n@DS*YJfQIO5tQ5++5;`oO(QhE40mrulqR{AStQ3L=jH^^YmCE%KgR69y z7dp<>SSeKLjLV&29ldk?c;Rw)m=`jfaaSSrqyQ7k-wQjBT|4A z_-6`mW8e=wIF!Cye;g0*S^@128XCq`NWg$^6ag^6|B;q-D!)JuajwQnA+Cb+G98c# N0VxP|q`)sI@By;bM|=PP literal 0 HcmV?d00001 diff --git a/common/src/main/java/.DS_Store b/common/src/main/java/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..3c38389f4b6ce453a087e350f406df53d2a5baad GIT binary patch literal 6148 zcmeHK%}T>S5Z-O8Ca92$(BtB{NJ=gB;30$x9)%Qou%d|(8;IG`r1nsw!DsM2d>fy~ zncWRo?O8--VE6lWXJ$A1LH36+#{EUyWz1oWSx^u;k`0F48(j?(jL30}d={lLg4TO9 zGrqqL`0Y)0%OrbUe^4C+SMGBAt z|4aei8U{lj7G=)XAIrnDRzSIdf`)Mw5-{LvmjGzsKGIfB?H9-)&NEnP#8I$cr32DM NKnX$>Dewyld;;U3My&t< literal 0 HcmV?d00001 diff --git a/common/src/main/java/com/.DS_Store b/common/src/main/java/com/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7274933556578f5532ba45ef704dae81dded9531 GIT binary patch literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~v94NJ=gB;30$x9)%Qou%ZbmHjpNzNv%+$!DsM2d>fy~ zncWQ-%vr?F!0b0WKfBovvOkP5?yQ3zV+LbP14ZO$)Cjt3Lk*RT$i)~7_T+EUr5`1J z%|L(Ah2P$03pQgJ^8mkp|A)Vf!Zfq2Pu^&@_S*ZJu4{+WNf-yo#!1$3klwiSWaEax z%=cEe!p^*GojTr1M6SJT4iRCGlOzIrs!0Vj zu4%gQt{tD8UR;g6=ltchko<9=TF8;Y65hb5EalN%#ZeMJfWNFMa|ww7Vt^PR26mVM zbtq^DJInzsofsen>KVZO!30IL6&4c3)&UJ(pE2G-L;)M$5{RZkTVWv)A|PCr0?JaZ zo)}z~gWptnw!%W9EN5KI4CAPo%f|~BvxDCh>5SVFsU-%8fhq${*){O|zl2|=_K{yL zp&l_n4E!?&cx&X299Wb(TYoGM&sr1o9ux)h3N%2#u3Q43gZoHF1ZCVr8}e+0g+v?$ S{h}O@E&_rOYKVbfVBiZyhe?V6 literal 0 HcmV?d00001 diff --git a/common/src/main/java/com/metamx/.DS_Store b/common/src/main/java/com/metamx/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..de97bb0c5e9924efedd9e0823865ec36191fac9d GIT binary patch literal 6148 zcmeHKOH0E*5Z>)W6I94W=rMRMl2S`OcnG0_McZy)Zn86s+KoL0_4TA2*P)j8vaxq5Wud<~dC4R#& zf76BE-emzxS;0cK`Tbx1G77WYvc7q%-RT}2YPzl+&u3vABr7K=;vlKUQI<0S5Z-NTO;I5ip~uB@krt(`;30$x9)%Qou%ZbmZ6G8iN$sIVgU{f5_%=R| zGrOBZFlP}v1GC@k{Oo2w$o??KxV`ZQjG2rv0~C>?Q6uQC4K;K!B9~(zyQ7DRr$wc}0Xr}xe(-a3K5 z@Z97<+L@beQrk^rodk%h$vv=TLMvNv@{k9Ap*ixDWEFl z`ia3+IrxRnvosb8RXO8wW*A4$Ts~g7oE`i^rZa9Sq?Q;U2C57+RoBGx{|0`U+DCr1 zgnGmPG4RhA;H`->v0+jAZ2hr3JZlB$6DSJi6=;BfUAY862ltV#463+58}clTg+d$! S{jwa8E&`GeYKVbfVBj0ZI7x*7 literal 0 HcmV?d00001 diff --git a/examples/query.body b/examples/query.body new file mode 100644 index 00000000000..e0607aa1554 --- /dev/null +++ b/examples/query.body @@ -0,0 +1,12 @@ +{ + "queryType": "groupBy", + "dataSource": "twitterstream", + "granularity": "all", + "dimensions": ["lang", "utc_offset"], + "aggregations":[ + { "type": "count", "name": "rows"}, + { "type": "doubleSum", "fieldName": "tweets", "name": "tweets"} + ], + "filter": { "type": "selector", "dimension": "lang", "value": "en" }, + "intervals":["2012-10-01T00:00/2020-01-01T00"] +} diff --git a/examples/search_query.body b/examples/search_query.body new file mode 100644 index 00000000000..f2feb27d4fe --- /dev/null +++ b/examples/search_query.body @@ -0,0 +1,13 @@ +{ + "queryType": "search", + "dataSource": "twitterstream", + "granularity": "all", + "searchDimensions": ["htags"], + "limit": 1, + "query": { + "type": "fragment", + "values": ["men"], + "sort": { "type": "strlen" } + }, + "intervals":["2012-10-01T00:00/2020-01-01T00"] +} diff --git a/examples/src/main/java/druid/examples/HttpTest.java b/examples/src/main/java/druid/examples/HttpTest.java new file mode 100644 index 00000000000..ca39df38bf2 --- /dev/null +++ b/examples/src/main/java/druid/examples/HttpTest.java @@ -0,0 +1,55 @@ +package druid.examples; + +import org.apache.http.*; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.HashMap; + +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/20/13 + * Time: 12:13 PM + * To change this template use File | Settings | File Templates. + */ +public class HttpTest +{ + public static void main(String[] args) throws Exception{ + URL url = new URL("http://developer.usa.gov/1usagov"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setDoInput(true); + connection.setDoOutput(true); + connection.setRequestMethod("GET"); + + BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); + String line; + ObjectMapper mapper = new ObjectMapper(); + TypeReference> typeRef = new TypeReference> () {}; + try{ + while ((line = reader.readLine())!= null){ + try{ + HashMap map=mapper.readValue(line, typeRef); + System.out.println(map); + } + catch (Exception e){ + System.out.println(e); + } + } + } + catch (IOException e){ + + } + } +} diff --git a/examples/src/main/java/druid/examples/twitter/Configuration.java b/examples/src/main/java/druid/examples/twitter/Configuration.java new file mode 100644 index 00000000000..4c6cc76f202 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/Configuration.java @@ -0,0 +1,59 @@ +package druid.examples.twitter; + +import druid.examples.twitter.HttpClientWrapperConfiguration.HttpClientWrapperConfiguration; + +import java.util.Map; + + +public interface Configuration extends HttpClientConfiguration + , HttpClientWrapperConfiguration + , java.io.Serializable { + + + boolean isDebugEnabled(); + + + Map getRequestHeaders(); + + // methods for HttpClientConfiguration + + String getHttpProxyHost(); + + String getHttpProxyUser(); + + String getHttpProxyPassword(); + + int getHttpProxyPort(); + + int getHttpConnectionTimeout(); + + int getHttpReadTimeout(); + + int getHttpStreamingReadTimeout(); + + int getHttpRetryCount(); + + int getHttpRetryIntervalSeconds(); + + int getHttpMaxTotalConnections(); + + int getHttpDefaultMaxPerRoute(); + + + String getStreamBaseURL(); + + + boolean isJSONStoreEnabled(); + + + boolean isStallWarningsEnabled(); + + int getAsyncNumThreads(); + + long getContributingTo(); + + String getDispatcherImpl(); + + String getLoggerFactory(); + +} diff --git a/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java b/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java new file mode 100644 index 00000000000..15ba09c7f6c --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/ConfigurationContext.java @@ -0,0 +1,57 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/** +* Static factory of Configuration. This class wraps ConfigurationFactory implementations.
+* By default, twitter4j.conf.PropertyConfigurationFactory will be used and can be changed with -Dtwitter4j.configurationFactory system property. +* +*/ +public final class ConfigurationContext { + public static final String DEFAULT_CONFIGURATION_FACTORY = "twitter4j.conf.PropertyConfigurationFactory"; + public static final String CONFIGURATION_IMPL = "twitter4j.configurationFactory"; + private static final ConfigurationFactory factory; + + static { + String CONFIG_IMPL; + try { + CONFIG_IMPL = System.getProperty(CONFIGURATION_IMPL, DEFAULT_CONFIGURATION_FACTORY); + } catch (SecurityException ignore) { + // Unsigned applets are not allowed to access System properties + CONFIG_IMPL = DEFAULT_CONFIGURATION_FACTORY; + } + + try { + factory = (ConfigurationFactory) Class.forName(CONFIG_IMPL).newInstance(); + } catch (ClassNotFoundException cnfe) { + throw new AssertionError(cnfe); + } catch (InstantiationException ie) { + throw new AssertionError(ie); + } catch (IllegalAccessException iae) { + throw new AssertionError(iae); + } + } + + + public static Configuration getInstance() { + return factory.getInstance(); + } + + public static Configuration getInstance(String configTreePath) { + return factory.getInstance(configTreePath); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java b/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java new file mode 100644 index 00000000000..d4c1acdf69e --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/ConfigurationFactory.java @@ -0,0 +1,42 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public interface ConfigurationFactory { + /** + * returns the root configuration + * + * @return root configuration + */ + Configuration getInstance(); + + /** + * returns the configuration specified by the path + * + * @param configTreePath the path + * @return the configuratoin + */ + Configuration getInstance(String configTreePath); + + /** + * clean up resources acquired by this factory. + */ + void dispose(); +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java b/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java new file mode 100644 index 00000000000..882ca03091a --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpClientConfiguration.java @@ -0,0 +1,46 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +public interface HttpClientConfiguration { + + String getHttpProxyHost(); + + int getHttpProxyPort(); + + String getHttpProxyUser(); + + String getHttpProxyPassword(); + + int getHttpConnectionTimeout(); + + int getHttpReadTimeout(); + + int getHttpRetryCount(); + + int getHttpRetryIntervalSeconds(); + + int getHttpMaxTotalConnections(); + + int getHttpDefaultMaxPerRoute(); + + boolean isPrettyDebugEnabled(); + + boolean isGZIPEnabled(); +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java b/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java new file mode 100644 index 00000000000..e419f6da6e0 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpClientWrapperConfiguration/HttpClientWrapperConfiguration.java @@ -0,0 +1,13 @@ +package druid.examples.twitter.HttpClientWrapperConfiguration; + +import druid.examples.twitter.HttpClientConfiguration; + +import java.util.Map; + +public interface HttpClientWrapperConfiguration extends HttpClientConfiguration +{ + /** + * @return request headers + */ + Map getRequestHeaders(); +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpParameter.java b/examples/src/main/java/druid/examples/twitter/HttpParameter.java new file mode 100644 index 00000000000..0ca5643df1b --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpParameter.java @@ -0,0 +1,288 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +import java.io.File; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +/** + * A data class representing HTTP Post parameter + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public final class HttpParameter implements Comparable, java.io.Serializable { + private String name = null; + private String value = null; + private File file = null; + private InputStream fileBody = null; + private static final long serialVersionUID = -8708108746980739212L; + + public HttpParameter(String name, String value) { + this.name = name; + this.value = value; + } + + public HttpParameter(String name, File file) { + this.name = name; + this.file = file; + } + + public HttpParameter(String name, String fileName, InputStream fileBody) { + this.name = name; + this.file = new File(fileName); + this.fileBody = fileBody; + } + + public HttpParameter(String name, int value) { + this.name = name; + this.value = String.valueOf(value); + } + + public HttpParameter(String name, long value) { + this.name = name; + this.value = String.valueOf(value); + } + + public HttpParameter(String name, double value) { + this.name = name; + this.value = String.valueOf(value); + } + + public HttpParameter(String name, boolean value) { + this.name = name; + this.value = String.valueOf(value); + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + public File getFile() { + return file; + } + + public InputStream getFileBody() { + return fileBody; + } + + public boolean isFile() { + return file != null; + } + + public boolean hasFileBody() { + return fileBody != null; + } + + private static final String JPEG = "image/jpeg"; + private static final String GIF = "image/gif"; + private static final String PNG = "image/png"; + private static final String OCTET = "application/octet-stream"; + + /** + * @return content-type + */ + public String getContentType() { + if (!isFile()) { + throw new IllegalStateException("not a file"); + } + String contentType; + String extensions = file.getName(); + int index = extensions.lastIndexOf("."); + if (-1 == index) { + // no extension + contentType = OCTET; + } else { + extensions = extensions.substring(extensions.lastIndexOf(".") + 1).toLowerCase(); + if (extensions.length() == 3) { + if ("gif".equals(extensions)) { + contentType = GIF; + } else if ("png".equals(extensions)) { + contentType = PNG; + } else if ("jpg".equals(extensions)) { + contentType = JPEG; + } else { + contentType = OCTET; + } + } else if (extensions.length() == 4) { + if ("jpeg".equals(extensions)) { + contentType = JPEG; + } else { + contentType = OCTET; + } + } else { + contentType = OCTET; + } + } + return contentType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof HttpParameter)) return false; + + HttpParameter that = (HttpParameter) o; + + if (file != null ? !file.equals(that.file) : that.file != null) + return false; + if (fileBody != null ? !fileBody.equals(that.fileBody) : that.fileBody != null) + return false; + if (!name.equals(that.name)) return false; + if (value != null ? !value.equals(that.value) : that.value != null) + return false; + + return true; + } + + public static boolean containsFile(HttpParameter[] params) { + boolean containsFile = false; + if (null == params) { + return false; + } + for (HttpParameter param : params) { + if (param.isFile()) { + containsFile = true; + break; + } + } + return containsFile; + } + + /*package*/ + static boolean containsFile(List params) { + boolean containsFile = false; + for (HttpParameter param : params) { + if (param.isFile()) { + containsFile = true; + break; + } + } + return containsFile; + } + + public static HttpParameter[] getParameterArray(String name, String value) { + return new HttpParameter[]{new HttpParameter(name, value)}; + } + + public static HttpParameter[] getParameterArray(String name, int value) { + return getParameterArray(name, String.valueOf(value)); + } + + public static HttpParameter[] getParameterArray(String name1, String value1 + , String name2, String value2) { + return new HttpParameter[]{new HttpParameter(name1, value1) + , new HttpParameter(name2, value2)}; + } + + public static HttpParameter[] getParameterArray(String name1, int value1 + , String name2, int value2) { + return getParameterArray(name1, String.valueOf(value1), name2, String.valueOf(value2)); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + (value != null ? value.hashCode() : 0); + result = 31 * result + (file != null ? file.hashCode() : 0); + result = 31 * result + (fileBody != null ? fileBody.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "PostParameter{" + + "name='" + name + '\'' + + ", value='" + value + '\'' + + ", file=" + file + + ", fileBody=" + fileBody + + '}'; + } + + @Override + public int compareTo(Object o) { + int compared; + HttpParameter that = (HttpParameter) o; + compared = name.compareTo(that.name); + if (0 == compared) { + compared = value.compareTo(that.value); + } + return compared; + } + + public static String encodeParameters(HttpParameter[] httpParams) { + if (null == httpParams) { + return ""; + } + StringBuilder buf = new StringBuilder(); + for (int j = 0; j < httpParams.length; j++) { + if (httpParams[j].isFile()) { + throw new IllegalArgumentException("parameter [" + httpParams[j].name + "]should be text"); + } + if (j != 0) { + buf.append("&"); + } + buf.append(encode(httpParams[j].name)) + .append("=").append(encode(httpParams[j].value)); + } + return buf.toString(); + } + + /** + * @param value string to be encoded + * @return encoded string + * @see OAuth / TestCases + * @see Space encoding - OAuth | Google Groups + * @see RFC 3986 - Uniform Resource Identifier (URI): Generic Syntax - 2.1. Percent-Encoding + */ + public static String encode(String value) { + String encoded = null; + try { + encoded = URLEncoder.encode(value, "UTF-8"); + } catch (UnsupportedEncodingException ignore) { + } + StringBuilder buf = new StringBuilder(encoded.length()); + char focus; + for (int i = 0; i < encoded.length(); i++) { + focus = encoded.charAt(i); + if (focus == '*') { + buf.append("%2A"); + } else if (focus == '+') { + buf.append("%20"); + } else if (focus == '%' && (i + 1) < encoded.length() + && encoded.charAt(i + 1) == '7' && encoded.charAt(i + 2) == 'E') { + buf.append('~'); + i += 2; + } else { + buf.append(focus); + } + } + return buf.toString(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpRequest.java b/examples/src/main/java/druid/examples/twitter/HttpRequest.java new file mode 100644 index 00000000000..9d233a423f8 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpRequest.java @@ -0,0 +1,120 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.util.Arrays; +import java.util.Map; + + +import java.util.Arrays; +import java.util.Map; + +/** + * HTTP Request parameter object + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public final class HttpRequest implements java.io.Serializable { + + private final RequestMethod method; + + private final String url; + + private final HttpParameter[] parameters; + + + private Map requestHeaders; + + private static final long serialVersionUID = -3463594029098858381L; + + + private static final HttpParameter[] NULL_PARAMETERS = new HttpParameter[0]; + + /** + * @param method Specifies the HTTP method + * @param url the request to request + * @param parameters parameters + * @param requestHeaders + */ + public HttpRequest(RequestMethod method, String url, HttpParameter[] parameters + , Map requestHeaders) { + this.method = method; + if (method != RequestMethod.POST && parameters != null && parameters.length != 0) { + this.url = url + "?" + HttpParameter.encodeParameters(parameters); + this.parameters = NULL_PARAMETERS; + } else { + this.url = url; + this.parameters = parameters; + } + this.requestHeaders = requestHeaders; + } + + public RequestMethod getMethod() { + return method; + } + + public HttpParameter[] getParameters() { + return parameters; + } + + public String getURL() { + return url; + } + + + public Map getRequestHeaders() { + return requestHeaders; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + HttpRequest that = (HttpRequest) o; + + if (!Arrays.equals(parameters, that.parameters)) return false; + if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) + return false; + if (method != null ? !method.equals(that.method) : that.method != null) + return false; + if (url != null ? !url.equals(that.url) : that.url != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = method != null ? method.hashCode() : 0; + result = 31 * result + (url != null ? url.hashCode() : 0); + result = 31 * result + (parameters != null ? Arrays.hashCode(parameters) : 0); + result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "HttpRequest{" + + "requestMethod=" + method + + ", url='" + url + '\'' + + ", postParams=" + (parameters == null ? null : Arrays.asList(parameters)) + + ", requestHeaders=" + requestHeaders + + '}'; + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponse.java b/examples/src/main/java/druid/examples/twitter/HttpResponse.java new file mode 100644 index 00000000000..ae2bc6f27d4 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponse.java @@ -0,0 +1,239 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import org.codehaus.jettison.json.JSONException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.List; +import java.util.Map; + +import java.io.*; +import java.util.List; +import java.util.Map; + +/** +* A data class representing HTTP Response +* +* @author Yusuke Yamamoto - yusuke at mac.com +*/ +public abstract class HttpResponse { + private static final Logger logger = Logger.getLogger(HttpResponseImpl.class); + protected final HttpClientConfiguration CONF; + + HttpResponse() { + this.CONF = ConfigurationContext.getInstance(); + } + + public HttpResponse(HttpClientConfiguration conf) { + this.CONF = conf; + } + + protected int statusCode; + protected String responseAsString = null; + protected InputStream is; + private boolean streamConsumed = false; + + public int getStatusCode() { + return statusCode; + } + + public abstract String getResponseHeader(String name); + + public abstract Map> getResponseHeaderFields(); + + /** + * Returns the response stream.
+ * This method cannot be called after calling asString() or asDcoument()
+ * It is suggested to call disconnect() after consuming the stream. + *

+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body stream + * @see #disconnect() + */ + public InputStream asStream() { + if (streamConsumed) { + throw new IllegalStateException("Stream has already been consumed."); + } + return is; + } + + /** + * Returns the response body as string.
+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body + */ + public String asString() throws RuntimeException { + if (null == responseAsString) { + BufferedReader br = null; + InputStream stream = null; + try { + stream = asStream(); + if (null == stream) { + return null; + } + br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); + StringBuilder buf = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + buf.append(line).append("\n"); + } + this.responseAsString = buf.toString(); + //logger.debug(responseAsString); + stream.close(); + streamConsumed = true; + } catch (IOException ioe) { + throw new RuntimeException(ioe.getMessage(), ioe); + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException ignore) { + } + } + if (br != null) { + try { + br.close(); + } catch (IOException ignore) { + } + } + disconnectForcibly(); + } + } + return responseAsString; + } + + private JSONObject json = null; + + /** + * Returns the response body as twitter4j.internal.org.json.JSONObject.
+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body as JSONObject + */ + public JSONObject asJSONObject() throws RuntimeException { + if (json == null) { + Reader reader = null; + try { + if (responseAsString == null) { + reader = asReader(); + json = new JSONObject(new JSONTokener(reader)); + } else { + json = new JSONObject(responseAsString); + } + if (CONF.isPrettyDebugEnabled()) { + //logger.debug(json.toString(1)); + } else { + //logger.debug(responseAsString != null ? responseAsString : +// json.toString()); + } + } catch (JSONException jsone) { + if (responseAsString == null) { + throw new RuntimeException(jsone.getMessage(), jsone); + } else { + throw new RuntimeException(jsone.getMessage() + ":" + this.responseAsString, jsone); + } + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException ignore) { + } + } + disconnectForcibly(); + } + } + return json; + } + + private JSONArray jsonArray = null; + + /** + * Returns the response body as twitter4j.internal.org.json.JSONArray.
+ * Disconnects the internal HttpURLConnection silently. + * + * @return response body as twitter4j.internal.org.json.JSONArray + * @throws RuntimeException + */ + public JSONArray asJSONArray() throws RuntimeException { + if (jsonArray == null) { + Reader reader = null; + try { + if (responseAsString == null) { + reader = asReader(); + jsonArray = new JSONArray(new JSONTokener(reader)); + } else { + jsonArray = new JSONArray(responseAsString); + } + if (CONF.isPrettyDebugEnabled()) { + //logger.debug(jsonArray.toString(1)); + } else { + //logger.debug(responseAsString != null ? responseAsString : +// jsonArray.toString()); + } + } catch (JSONException jsone) { +// if (logger.isDebugEnabled()) { +// throw new RuntimeException(jsone.getMessage() + ":" + this.responseAsString, jsone); +// } else { +// throw new RuntimeException(jsone.getMessage(), jsone); +// } + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException ignore) { + } + } + disconnectForcibly(); + } + } + return jsonArray; + } + + public Reader asReader() { + try { + return new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException uee) { + return new InputStreamReader(is); + } + } + + private void disconnectForcibly() { + try { + disconnect(); + } catch (Exception ignore) { + } + } + + public abstract void disconnect() throws IOException; + + @Override + public String toString() { + return "HttpResponse{" + + "statusCode=" + statusCode + + ", responseAsString='" + responseAsString + '\'' + + ", is=" + is + + ", streamConsumed=" + streamConsumed + + '}'; + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java b/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java new file mode 100644 index 00000000000..2f92bae3d7d --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponseEvent.java @@ -0,0 +1,92 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/** +* @author Andrew Hedges - andrew.hedges at gmail.com +*/ +public final class HttpResponseEvent { + + private HttpRequest request; + + private HttpResponse response; + + private RuntimeException e; + + HttpResponseEvent(HttpRequest request, HttpResponse response, RuntimeException e) { + this.request = request; + this.response = response; + this.e = e; + } + + /** + * returns the request associated with the event + * + * @return the request associated with the event + */ + public HttpRequest getRequest() { + return request; + } + + /** + * returns the response associated with the event + * + * @return the response associated with the event + */ + public HttpResponse getResponse() { + return response; + } + + /** + * returns the TwitterException associated with the event + * + * @return the TwitterException associated with the event + */ + public RuntimeException getRuntimeException() { + return this.e; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + HttpResponseEvent that = (HttpResponseEvent) o; + + if (request != null ? !request.equals(that.request) : that.request != null) + return false; + if (response != null ? !response.equals(that.response) : that.response != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = request != null ? request.hashCode() : 0; + result = 31 * result + (response != null ? response.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "HttpResponseEvent{" + + "request=" + request + + ", response=" + response + + '}'; + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java b/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java new file mode 100644 index 00000000000..0faf59f50c9 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponseImpl.java @@ -0,0 +1,75 @@ +package druid.examples.twitter; + +/* +* Copyright 2007 Yusuke Yamamoto +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; + + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; + +/** +* @author Yusuke Yamamoto - yusuke at mac.com +* @since Twitter4J 2.1.2 +*/ +public class HttpResponseImpl extends HttpResponse +{ + private HttpURLConnection con; + + HttpResponseImpl(HttpURLConnection con, HttpClientConfiguration conf) throws IOException + { + super(conf); + this.con = con; + this.statusCode = con.getResponseCode(); + if (null == (is = con.getErrorStream())) { + is = con.getInputStream(); + } + if (is != null && "gzip".equals(con.getContentEncoding())) { + // the response is gzipped + is = new StreamingGZIPInputStream(is); + } + } + + // for test purpose + /*package*/ HttpResponseImpl(String content) { + super(); + this.responseAsString = content; + } + + @Override + public String getResponseHeader(String name) { + return con.getHeaderField(name); + } + + @Override + public Map> getResponseHeaderFields() { + return con.getHeaderFields(); + } + + /** + * {@inheritDoc} + */ + @Override + public void disconnect() { + con.disconnect(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java b/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java new file mode 100644 index 00000000000..5d1d639bcae --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/HttpResponseListener.java @@ -0,0 +1,10 @@ +package druid.examples.twitter; + +/** + * @author Andrew Hedges - andrew.hedges at gmail.com + */ +public interface HttpResponseListener { + + public void httpResponseReceived(HttpResponseEvent event); + +} diff --git a/examples/src/main/java/druid/examples/twitter/JSONArray.java b/examples/src/main/java/druid/examples/twitter/JSONArray.java new file mode 100644 index 00000000000..897d9825826 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/JSONArray.java @@ -0,0 +1,726 @@ +package druid.examples.twitter; + +import org.codehaus.jettison.json.JSONException; + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +/** + * A JSONArray is an ordered sequence of values. Its external text form is a + * string wrapped in square brackets with commas separating the values. The + * internal form is an object having get and opt + * methods for accessing the values by index, and put methods for + * adding or replacing values. The values can be any of these types: + * Boolean, JSONArray, JSONObject, + * Number, String, or the + * JSONObject.NULL object. + *

+ * The constructor can convert a JSON text into a Java object. The + * toString method converts to JSON text. + *

+ * A get method returns a value if one can be found, and throws an + * exception if one cannot be found. An opt method returns a + * default value instead of throwing an exception, and so is useful for + * obtaining optional values. + *

+ * The generic get() and opt() methods return an + * object which you can cast or query for type. There are also typed + * get and opt methods that do type checking and type + * coercion for you. + *

+ * The texts produced by the toString methods strictly conform to + * JSON syntax rules. The constructors are more forgiving in the texts they will + * accept: + *

    + *
  • An extra , (comma) may appear just + * before the closing bracket.
    • + *
  • The null value will be inserted when there + * is , (comma) elision.
    • + *
  • Strings may be quoted with ' (single + * quote).
    • + *
  • Strings do not need to be quoted at all if they do not begin with a quote + * or single quote, and if they do not contain leading or trailing spaces, + * and if they do not contain any of these characters: + * { } [ ] / \ : , = ; # and if they do not look like numbers + * and if they are not the reserved words true, + * false, or null.
    • + *
  • Values can be separated by ; (semicolon) as + * well as by , (comma).
    • + *
  • Numbers may have the + * 0x- (hex) prefix.
    • + *
+ * + * @author JSON.org + * @version 2010-12-28 + */ +public class JSONArray { + + + /** + * The arrayList where the JSONArray's properties are kept. + */ + private ArrayList myArrayList; + + + /** + * Construct an empty JSONArray. + */ + public JSONArray() { + this.myArrayList = new ArrayList(); + } + + /** + * Construct a JSONArray from a JSONTokener. + * + * @param x A JSONTokener + * @throws JSONException If there is a syntax error. + */ + public JSONArray(JSONTokener x) throws JSONException + { + this(); + if (x.nextClean() != '[') { + throw x.syntaxError("A JSONArray text must start with '['"); + } + if (x.nextClean() != ']') { + x.back(); + for (; ; ) { + if (x.nextClean() == ',') { + x.back(); + this.myArrayList.add(twitter4j.internal.org.json.JSONObject.NULL); + } else { + x.back(); + this.myArrayList.add(x.nextValue()); + } + switch (x.nextClean()) { + case ';': + case ',': + if (x.nextClean() == ']') { + return; + } + x.back(); + break; + case ']': + return; + default: + throw x.syntaxError("Expected a ',' or ']'"); + } + } + } + } + + + /** + * Construct a JSONArray from a source JSON text. + * + * @param source A string that begins with + * [ (left bracket) + * and ends with ] (right bracket). + * @throws JSONException If there is a syntax error. + */ + public JSONArray(String source) throws JSONException { + this(new twitter4j.internal.org.json.JSONTokener(source)); + } + + + /** + * Construct a JSONArray from a Collection. + * + * @param collection A Collection. + */ + public JSONArray(Collection collection) { + this.myArrayList = new ArrayList(); + if (collection != null) { + for (Object aCollection : collection) { + this.myArrayList.add(twitter4j.internal.org.json.JSONObject.wrap(aCollection)); + } + } + } + + + /** + * Construct a JSONArray from an array + * + * @throws JSONException If not an array. + */ + public JSONArray(Object array) throws JSONException { + this(); + if (array.getClass().isArray()) { + int length = Array.getLength(array); + for (int i = 0; i < length; i += 1) { + this.put(twitter4j.internal.org.json.JSONObject.wrap(Array.get(array, i))); + } + } else { + throw new JSONException( + "JSONArray initial value should be a string or collection or array."); + } + } + + + /** + * Get the object value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return An object value. + * @throws JSONException If there is no value for the index. + */ + public Object get(int index) throws JSONException { + Object object = opt(index); + if (object == null) { + throw new JSONException("JSONArray[" + index + "] not found."); + } + return object; + } + + + /** + * Get the boolean value associated with an index. + * The string values "true" and "false" are converted to boolean. + * + * @param index The index must be between 0 and length() - 1. + * @return The truth. + * @throws JSONException If there is no value for the index or if the + * value is not convertible to boolean. + */ + public boolean getBoolean(int index) throws JSONException { + Object object = get(index); + if (object.equals(Boolean.FALSE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("false"))) { + return false; + } else if (object.equals(Boolean.TRUE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("true"))) { + return true; + } + throw new JSONException("JSONArray[" + index + "] is not a boolean."); + } + + + /** + * Get the double value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + */ + public double getDouble(int index) throws JSONException { + Object object = get(index); + try { + return object instanceof Number ? + ((Number) object).doubleValue() : + Double.parseDouble((String) object); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the int value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value is not a number. + */ + public int getInt(int index) throws JSONException { + Object object = get(index); + try { + return object instanceof Number ? + ((Number) object).intValue() : + Integer.parseInt((String) object); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the JSONArray associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return A JSONArray value. + * @throws JSONException If there is no value for the index. or if the + * value is not a JSONArray + */ + public JSONArray getJSONArray(int index) throws JSONException { + Object object = get(index); + if (object instanceof JSONArray) { + return (JSONArray) object; + } + throw new JSONException("JSONArray[" + index + + "] is not a JSONArray."); + } + + + /** + * Get the JSONObject associated with an index. + * + * @param index subscript + * @return A JSONObject value. + * @throws JSONException If there is no value for the index or if the + * value is not a JSONObject + */ + public twitter4j.internal.org.json.JSONObject getJSONObject(int index) throws JSONException { + Object object = get(index); + if (object instanceof twitter4j.internal.org.json.JSONObject) { + return (twitter4j.internal.org.json.JSONObject) object; + } + throw new JSONException("JSONArray[" + index + + "] is not a JSONObject."); + } + + + /** + * Get the long value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + */ + public long getLong(int index) throws JSONException { + Object object = get(index); + try { + return object instanceof Number ? + ((Number) object).longValue() : + Long.parseLong((String) object); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the string associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return A string value. + * @throws JSONException If there is no value for the index. + */ + public String getString(int index) throws JSONException { + Object object = get(index); + return object == twitter4j.internal.org.json.JSONObject.NULL ? null : object.toString(); + } + + + /** + * Determine if the value is null. + * + * @param index The index must be between 0 and length() - 1. + * @return true if the value at the index is null, or if there is no value. + */ + public boolean isNull(int index) { + return twitter4j.internal.org.json.JSONObject.NULL.equals(opt(index)); + } + + + /** + * Make a string from the contents of this JSONArray. The + * separator string is inserted between each element. + * Warning: This method assumes that the data structure is acyclical. + * + * @param separator A string that will be inserted between the elements. + * @return a string. + * @throws JSONException If the array contains an invalid number. + */ + public String join(String separator) throws JSONException { + int len = length(); + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < len; i += 1) { + if (i > 0) { + sb.append(separator); + } + sb.append(JSONObject.valueToString(this.myArrayList.get(i))); + } + return sb.toString(); + } + + + /** + * Get the number of elements in the JSONArray, included nulls. + * + * @return The length (or size). + */ + public int length() { + return this.myArrayList.size(); + } + + + /** + * Get the optional object value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return An object value, or null if there is no + * object at that index. + */ + public Object opt(int index) { + return (index < 0 || index >= length()) ? + null : this.myArrayList.get(index); + } + + /** + * Append a boolean value. This increases the array's length by one. + * + * @param value A boolean value. + * @return this. + */ + public JSONArray put(boolean value) { + put(value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONArray which is produced from a Collection. + * + * @param value A Collection value. + * @return this. + */ + public JSONArray put(Collection value) { + put(new JSONArray(value)); + return this; + } + + /** + * Append an int value. This increases the array's length by one. + * + * @param value An int value. + * @return this. + */ + public JSONArray put(int value) { + put(new Integer(value)); + return this; + } + + + /** + * Append an long value. This increases the array's length by one. + * + * @param value A long value. + * @return this. + */ + public JSONArray put(long value) { + put(new Long(value)); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONObject which is produced from a Map. + * + * @param value A Map value. + * @return this. + */ + public JSONArray put(Map value) { + put(new twitter4j.internal.org.json.JSONObject(value)); + return this; + } + + + /** + * Append an object value. This increases the array's length by one. + * + * @param value An object value. The value should be a + * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the + * JSONObject.NULL object. + * @return this. + */ + public JSONArray put(Object value) { + this.myArrayList.add(value); + return this; + } + + + /** + * Put or replace a boolean value in the JSONArray. If the index is greater + * than the length of the JSONArray, then null elements will be added as + * necessary to pad it out. + * + * @param index The subscript. + * @param value A boolean value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, boolean value) throws JSONException { + put(index, value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONArray which is produced from a Collection. + * + * @param index The subscript. + * @param value A Collection value. + * @return this. + * @throws JSONException If the index is negative or if the value is + * not finite. + */ + public JSONArray put(int index, Collection value) throws JSONException { + put(index, new JSONArray(value)); + return this; + } + + + /** + * Put or replace a double value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * + * @param index The subscript. + * @param value A double value. + * @return this. + * @throws JSONException If the index is negative or if the value is + * not finite. + */ + public JSONArray put(int index, double value) throws JSONException { + put(index, new Double(value)); + return this; + } + + + /** + * Put or replace an int value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * + * @param index The subscript. + * @param value An int value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, int value) throws JSONException { + put(index, new Integer(value)); + return this; + } + + + /** + * Put or replace a long value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * + * @param index The subscript. + * @param value A long value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, long value) throws JSONException { + put(index, new Long(value)); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONObject which is produced from a Map. + * + * @param index The subscript. + * @param value The Map value. + * @return this. + * @throws JSONException If the index is negative or if the the value is + * an invalid number. + */ + public JSONArray put(int index, Map value) throws JSONException { + put(index, new twitter4j.internal.org.json.JSONObject(value)); + return this; + } + + + /** + * Put or replace an object value in the JSONArray. If the index is greater + * than the length of the JSONArray, then null elements will be added as + * necessary to pad it out. + * + * @param index The subscript. + * @param value The value to put into the array. The value should be a + * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the + * JSONObject.NULL object. + * @return this. + * @throws JSONException If the index is negative or if the the value is + * an invalid number. + */ + public JSONArray put(int index, Object value) throws JSONException { + JSONObject.testValidity(value); + if (index < 0) { + throw new JSONException("JSONArray[" + index + "] not found."); + } + if (index < length()) { + this.myArrayList.set(index, value); + } else { + while (index != length()) { + put(twitter4j.internal.org.json.JSONObject.NULL); + } + put(value); + } + return this; + } + + /** + * Make a JSON text of this JSONArray. For compactness, no + * unnecessary whitespace is added. If it is not possible to produce a + * syntactically correct JSON text then null will be returned instead. This + * could occur if the array contains an invalid number. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return a printable, displayable, transmittable + * representation of the array. + */ + public String toString() { + try { + return '[' + join(",") + ']'; + } catch (Exception e) { + return null; + } + } + + + /** + * Make a prettyprinted JSON text of this JSONArray. + * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with [ (left bracket) and ending + * with ] (right bracket). + * @throws JSONException + */ + public String toString(int indentFactor) throws JSONException { + return toString(indentFactor, 0); + } + + + /** + * Make a prettyprinted JSON text of this JSONArray. + * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indention of the top level. + * @return a printable, displayable, transmittable + * representation of the array. + * @throws JSONException + */ + String toString(int indentFactor, int indent) throws JSONException { + int len = length(); + if (len == 0) { + return "[]"; + } + int i; + StringBuilder sb = new StringBuilder("["); + if (len == 1) { + sb.append( + JSONObject.valueToString( + this.myArrayList.get(0), + indentFactor, indent + )); + } else { + int newindent = indent + indentFactor; + sb.append('\n'); + for (i = 0; i < len; i += 1) { + if (i > 0) { + sb.append(",\n"); + } + for (int j = 0; j < newindent; j += 1) { + sb.append(' '); + } + sb.append( + JSONObject.valueToString( + this.myArrayList.get(i), + indentFactor, newindent + )); + } + sb.append('\n'); + for (i = 0; i < indent; i += 1) { + sb.append(' '); + } + } + sb.append(']'); + return sb.toString(); + } + + + /** + * Write the contents of the JSONArray as JSON text to a writer. + * For compactness, no whitespace is added. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return The writer. + * @throws JSONException + */ + public Writer write(Writer writer) throws JSONException { + try { + boolean b = false; + int len = length(); + + writer.write('['); + + for (int i = 0; i < len; i += 1) { + if (b) { + writer.write(','); + } + Object v = this.myArrayList.get(i); + if (v instanceof twitter4j.internal.org.json.JSONObject) { + ((JSONObject) v).write(writer); + } else if (v instanceof JSONArray) { + ((JSONArray) v).write(writer); + } else { + writer.write(JSONObject.valueToString(v)); + } + b = true; + } + writer.write(']'); + return writer; + } catch (IOException e) { + throw new JSONException(e); + } + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/JSONObject.java b/examples/src/main/java/druid/examples/twitter/JSONObject.java new file mode 100644 index 00000000000..373dcd10726 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/JSONObject.java @@ -0,0 +1,1285 @@ +package druid.examples.twitter; + + +import org.codehaus.jettison.json.JSONException; + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.ResourceBundle; +import java.util.TreeSet; + + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.*; + +/** + * A JSONObject is an unordered collection of name/value pairs. Its + * external form is a string wrapped in curly braces with colons between the + * names and values, and commas between the values and names. The internal form + * is an object having get and opt methods for + * accessing the values by name, and put methods for adding or + * replacing values by name. The values can be any of these types: + * Boolean, JSONArray, JSONObject, + * Number, String, or the JSONObject.NULL + * object. A JSONObject constructor can be used to convert an external form + * JSON text into an internal form whose values can be retrieved with the + * get and opt methods, or to convert values into a + * JSON text using the put and toString methods. + * A get method returns a value if one can be found, and throws an + * exception if one cannot be found. An opt method returns a + * default value instead of throwing an exception, and so is useful for + * obtaining optional values. + *

+ * The generic get() and opt() methods return an + * object, which you can cast or query for type. There are also typed + * get and opt methods that do type checking and type + * coercion for you. The opt methods differ from the get methods in that they + * do not throw. Instead, they return a specified value, such as null. + *

+ * The put methods add or replace values in an object. For example, + * 

myString = new JSONObject().put("JSON", "Hello, World!").toString();
+ * produces the string {"JSON": "Hello, World"}. + *

+ * The texts produced by the toString methods strictly conform to + * the JSON syntax rules. + * The constructors are more forgiving in the texts they will accept: + *

    + *
  • An extra , (comma) may appear just + * before the closing brace.
    • + *
  • Strings may be quoted with ' (single + * quote).
    • + *
  • Strings do not need to be quoted at all if they do not begin with a quote + * or single quote, and if they do not contain leading or trailing spaces, + * and if they do not contain any of these characters: + * { } [ ] / \ : , = ; # and if they do not look like numbers + * and if they are not the reserved words true, + * false, or null.
    • + *
  • Keys can be followed by = or => as well as + * by :.
    • + *
  • Values can be followed by ; (semicolon) as + * well as by , (comma).
    • + *
  • Numbers may have the 0x- (hex) prefix.
    • + *
+ * + * @author JSON.org + * @version 2010-12-28 + */ +public class JSONObject { + + /** + * JSONObject.NULL is equivalent to the value that JavaScript calls null, + * whilst Java's null is equivalent to the value that JavaScript calls + * undefined. + */ + private static final class Null { + + /** + * There is only intended to be a single instance of the NULL object, + * so the clone method returns itself. + * + * @return NULL. + */ + protected final Object clone() { + return this; + } + + /** + * A Null object is equal to the null value and to itself. + * + * @param object An object to test for nullness. + * @return true if the object parameter is the JSONObject.NULL object + * or null. + */ + public boolean equals(Object object) { + return object == null || object == this; + } + + /** + * Get the "null" string value. + * + * @return The string "null". + */ + public String toString() { + return "null"; + } + } + + + /** + * The map where the JSONObject's properties are kept. + */ + private Map map; + + + /** + * It is sometimes more convenient and less ambiguous to have a + * NULL object than to use Java's null value. + * JSONObject.NULL.equals(null) returns true. + * JSONObject.NULL.toString() returns "null". + */ + public static final Object NULL = new Null(); + + + /** + * Construct an empty JSONObject. + */ + public JSONObject() { + this.map = new HashMap(); + } + + + /** + * Construct a JSONObject from a subset of another JSONObject. + * An array of strings is used to identify the keys that should be copied. + * Missing keys are ignored. + * + * @param jo A JSONObject. + * @param names An array of strings. + */ + public JSONObject(JSONObject jo, String[] names) { + this(); + for (int i = 0; i < names.length; i += 1) { + try { + putOnce(names[i], jo.opt(names[i])); + } catch (Exception ignore) { + } + } + } + + + /** + * Construct a JSONObject from a JSONTokener. + * + * @param x A JSONTokener object containing the source string. + * @throws twitter4j.internal.org.json.JSONException If there is a syntax error in the source string + * or a duplicated key. + */ + public JSONObject(JSONTokener x) throws JSONException + { + this(); + char c; + String key; + + if (x.nextClean() != '{') { + throw x.syntaxError("A JSONObject text must begin with '{' found:" + x.nextClean()); + } + for (; ; ) { + c = x.nextClean(); + switch (c) { + case 0: + throw x.syntaxError("A JSONObject text must end with '}'"); + case '}': + return; + default: + x.back(); + key = x.nextValue().toString(); + } + +// The key is followed by ':'. We will also tolerate '=' or '=>'. + + c = x.nextClean(); + if (c == '=') { + if (x.next() != '>') { + x.back(); + } + } else if (c != ':') { + throw x.syntaxError("Expected a ':' after a key"); + } + putOnce(key, x.nextValue()); + +// Pairs are separated by ','. We will also tolerate ';'. + + switch (x.nextClean()) { + case ';': + case ',': + if (x.nextClean() == '}') { + return; + } + x.back(); + break; + case '}': + return; + default: + throw x.syntaxError("Expected a ',' or '}'"); + } + } + } + + + /** + * Construct a JSONObject from a Map. + * + * @param map A map object that can be used to initialize the contents of + * the JSONObject. + * @throws JSONException + */ + public JSONObject(Map map) { + this.map = new HashMap(); + if (map != null) { + Iterator i = map.entrySet().iterator(); + while (i.hasNext()) { + Map.Entry e = (Map.Entry) i.next(); + Object value = e.getValue(); + if (value != null) { + this.map.put(e.getKey(), wrap(value)); + } + } + } + } + + + /** + * Construct a JSONObject from an Object using bean getters. + * It reflects on all of the public methods of the object. + * For each of the methods with no parameters and a name starting + * with "get" or "is" followed by an uppercase letter, + * the method is invoked, and a key and the value returned from the getter method + * are put into the new JSONObject. + *

+ * The key is formed by removing the "get" or "is" prefix. + * If the second remaining character is not upper case, then the first + * character is converted to lower case. + *

+ * For example, if an object has a method named "getName", and + * if the result of calling object.getName() is "Larry Fine", + * then the JSONObject will contain "name": "Larry Fine". + * + * @param bean An object that has getter methods that should be used + * to make a JSONObject. + */ + public JSONObject(Object bean) { + this(); + populateMap(bean); + } + + + /** + * Construct a JSONObject from an Object, using reflection to find the + * public members. The resulting JSONObject's keys will be the strings + * from the names array, and the values will be the field values associated + * with those keys in the object. If a key is not found or not visible, + * then it will not be copied into the new JSONObject. + * + * @param object An object that has fields that should be used to make a + * JSONObject. + * @param names An array of strings, the names of the fields to be obtained + * from the object. + */ + public JSONObject(Object object, String names[]) { + this(); + Class c = object.getClass(); + for (int i = 0; i < names.length; i += 1) { + String name = names[i]; + try { + putOpt(name, c.getField(name).get(object)); + } catch (Exception ignore) { + } + } + } + + + /** + * Construct a JSONObject from a source JSON text string. + * This is the most commonly used JSONObject constructor. + * + * @param source A string beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If there is a syntax error in the source + * string or a duplicated key. + */ + public JSONObject(String source) throws JSONException { + this(new twitter4j.internal.org.json.JSONTokener(source)); + } + + + /** + * Construct a JSONObject from a ResourceBundle. + * + * @param baseName The ResourceBundle base name. + * @param locale The Locale to load the ResourceBundle for. + * @throws JSONException If any JSONExceptions are detected. + */ + public JSONObject(String baseName, Locale locale) throws JSONException { + this(); + ResourceBundle r = ResourceBundle.getBundle(baseName, locale, + Thread.currentThread().getContextClassLoader()); + +// Iterate through the keys in the bundle. + + Enumeration keys = r.getKeys(); + while (keys.hasMoreElements()) { + Object key = keys.nextElement(); + if (key instanceof String) { + +// Go through the path, ensuring that there is a nested JSONObject for each +// segment except the last. Add the value using the last segment's name into +// the deepest nested JSONObject. + + String[] path = ((String) key).split("\\."); + int last = path.length - 1; + JSONObject target = this; + for (int i = 0; i < last; i += 1) { + String segment = path[i]; + Object object = target.opt(segment); + JSONObject nextTarget = object instanceof JSONObject ? (JSONObject) object : null; + if (nextTarget == null) { + nextTarget = new JSONObject(); + target.put(segment, nextTarget); + } + target = nextTarget; + } + target.put(path[last], r.getString((String) key)); + } + } + } + + /** + * Append values to the array under a key. If the key does not exist in the + * JSONObject, then the key is put in the JSONObject with its value being a + * JSONArray containing the value parameter. If the key was already + * associated with a JSONArray, then the value parameter is appended to it. + * + * @param key A key string. + * @param value An object to be accumulated under the key. + * @return this. + * @throws JSONException If the key is null or if the current value + * associated with the key is not a JSONArray. + */ + public JSONObject append(String key, Object value) throws JSONException { + testValidity(value); + Object object = opt(key); + if (object == null) { + put(key, new JSONArray().put(value)); + } else if (object instanceof JSONArray) { + put(key, ((JSONArray) object).put(value)); + } else { + throw new JSONException("JSONObject[" + key + + "] is not a JSONArray."); + } + return this; + } + + /** + * Get the value object associated with a key. + * + * @param key A key string. + * @return The object associated with the key. + * @throws JSONException if the key is not found. + */ + public Object get(String key) throws JSONException { + if (key == null) { + throw new JSONException("Null key."); + } + Object object = opt(key); + if (object == null) { + throw new JSONException("JSONObject[" + quote(key) + + "] not found."); + } + return object; + } + + + /** + * Get the boolean value associated with a key. + * + * @param key A key string. + * @return The truth. + * @throws JSONException if the value is not a Boolean or the String "true" or "false". + */ + public boolean getBoolean(String key) throws JSONException { + Object object = get(key); + if (object.equals(Boolean.FALSE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("false"))) { + return false; + } else if (object.equals(Boolean.TRUE) || + (object instanceof String && + ((String) object).equalsIgnoreCase("true"))) { + return true; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a Boolean."); + } + + /** + * Get the int value associated with a key. + * + * @param key A key string. + * @return The integer value. + * @throws JSONException if the key is not found or if the value cannot + * be converted to an integer. + */ + public int getInt(String key) throws JSONException { + Object object = get(key); + try { + return object instanceof Number ? + ((Number) object).intValue() : + Integer.parseInt((String) object); + } catch (Exception e) { + throw new JSONException("JSONObject[" + quote(key) + + "] is not an int."); + } + } + + + /** + * Get the JSONArray value associated with a key. + * + * @param key A key string. + * @return A JSONArray which is the value. + * @throws JSONException if the key is not found or + * if the value is not a JSONArray. + */ + public JSONArray getJSONArray(String key) throws JSONException { + Object object = get(key); + if (object instanceof JSONArray) { + return (JSONArray) object; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a JSONArray."); + } + + + /** + * Get the JSONObject value associated with a key. + * + * @param key A key string. + * @return A JSONObject which is the value. + * @throws JSONException if the key is not found or + * if the value is not a JSONObject. + */ + public JSONObject getJSONObject(String key) throws JSONException { + Object object = get(key); + if (object instanceof JSONObject) { + return (JSONObject) object; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a JSONObject."); + } + + + /** + * Get the long value associated with a key. + * + * @param key A key string. + * @return The long value. + * @throws JSONException if the key is not found or if the value cannot + * be converted to a long. + */ + public long getLong(String key) throws JSONException { + Object object = get(key); + try { + return object instanceof Number ? + ((Number) object).longValue() : + Long.parseLong((String) object); + } catch (Exception e) { + throw new JSONException("JSONObject[" + quote(key) + + "] is not a long."); + } + } + + /** + * Get the string associated with a key. + * + * @param key A key string. + * @return A string which is the value. + * @throws JSONException if the key is not found. + */ + public String getString(String key) throws JSONException { + Object object = get(key); + return object == NULL ? null : object.toString(); + } + + + /** + * Determine if the JSONObject contains a specific key. + * + * @param key A key string. + * @return true if the key exists in the JSONObject. + */ + public boolean has(String key) { + return this.map.containsKey(key); + } + + /** + * Determine if the value associated with the key is null or if there is + * no value. + * + * @param key A key string. + * @return true if there is no value associated with the key or if + * the value is the JSONObject.NULL object. + */ + public boolean isNull(String key) { + return JSONObject.NULL.equals(opt(key)); + } + + + /** + * Get an enumeration of the keys of the JSONObject. + * + * @return An iterator of the keys. + */ + public Iterator keys() { + return this.map.keySet().iterator(); + } + + + /** + * Get the number of keys stored in the JSONObject. + * + * @return The number of keys in the JSONObject. + */ + public int length() { + return this.map.size(); + } + + + /** + * Produce a JSONArray containing the names of the elements of this + * JSONObject. + * + * @return A JSONArray containing the key strings, or null if the JSONObject + * is empty. + */ + public JSONArray names() { + JSONArray ja = new JSONArray(); + Iterator keys = keys(); + while (keys.hasNext()) { + ja.put(keys.next()); + } + return ja.length() == 0 ? null : ja; + } + + /** + * Produce a string from a Number. + * + * @param number A Number + * @return A String. + * @throws JSONException If n is a non-finite number. + */ + public static String numberToString(Number number) + throws JSONException { + if (number == null) { + throw new JSONException("Null pointer"); + } + testValidity(number); + +// Shave off trailing zeros and decimal point, if possible. + + String string = number.toString(); + if (string.indexOf('.') > 0 && string.indexOf('e') < 0 && + string.indexOf('E') < 0) { + while (string.endsWith("0")) { + string = string.substring(0, string.length() - 1); + } + if (string.endsWith(".")) { + string = string.substring(0, string.length() - 1); + } + } + return string; + } + + + /** + * Get an optional value associated with a key. + * + * @param key A key string. + * @return An object which is the value, or null if there is no value. + */ + public Object opt(String key) { + return key == null ? null : this.map.get(key); + } + + + private void populateMap(Object bean) { + Class klass = bean.getClass(); + +// If klass is a System class then set includeSuperClass to false. + + boolean includeSuperClass = klass.getClassLoader() != null; + + Method[] methods = (includeSuperClass) ? + klass.getMethods() : klass.getDeclaredMethods(); + for (int i = 0; i < methods.length; i += 1) { + try { + Method method = methods[i]; + if (Modifier.isPublic(method.getModifiers())) { + String name = method.getName(); + String key = ""; + if (name.startsWith("get")) { + if (name.equals("getClass") || + name.equals("getDeclaringClass")) { + key = ""; + } else { + key = name.substring(3); + } + } else if (name.startsWith("is")) { + key = name.substring(2); + } + if (key.length() > 0 && + Character.isUpperCase(key.charAt(0)) && + method.getParameterTypes().length == 0) { + if (key.length() == 1) { + key = key.toLowerCase(); + } else if (!Character.isUpperCase(key.charAt(1))) { + key = key.substring(0, 1).toLowerCase() + + key.substring(1); + } + + Object result = method.invoke(bean, (Object[]) null); + if (result != null) { + map.put(key, wrap(result)); + } + } + } + } catch (Exception ignore) { + } + } + } + + + /** + * Put a key/boolean pair in the JSONObject. + * + * @param key A key string. + * @param value A boolean which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, boolean value) throws JSONException { + put(key, value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a key/value pair in the JSONObject, where the value will be a + * JSONArray which is produced from a Collection. + * + * @param key A key string. + * @param value A Collection value. + * @return this. + * @throws JSONException + */ + public JSONObject put(String key, Collection value) throws JSONException { + put(key, new JSONArray(value)); + return this; + } + + + /** + * Put a key/double pair in the JSONObject. + * + * @param key A key string. + * @param value A double which is the value. + * @return this. + * @throws JSONException If the key is null or if the number is invalid. + */ + public JSONObject put(String key, double value) throws JSONException { + put(key, new Double(value)); + return this; + } + + + /** + * Put a key/int pair in the JSONObject. + * + * @param key A key string. + * @param value An int which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, int value) throws JSONException { + put(key, new Integer(value)); + return this; + } + + + /** + * Put a key/long pair in the JSONObject. + * + * @param key A key string. + * @param value A long which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, long value) throws JSONException { + put(key, new Long(value)); + return this; + } + + + /** + * Put a key/value pair in the JSONObject, where the value will be a + * JSONObject which is produced from a Map. + * + * @param key A key string. + * @param value A Map value. + * @return this. + * @throws JSONException + */ + public JSONObject put(String key, Map value) throws JSONException { + put(key, new JSONObject(value)); + return this; + } + + + /** + * Put a key/value pair in the JSONObject. If the value is null, + * then the key will be removed from the JSONObject if it is present. + * + * @param key A key string. + * @param value An object which is the value. It should be of one of these + * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, + * or the JSONObject.NULL object. + * @return this. + * @throws JSONException If the value is non-finite number + * or if the key is null. + */ + public JSONObject put(String key, Object value) throws JSONException { + if (key == null) { + throw new JSONException("Null key."); + } + if (value != null) { + testValidity(value); + this.map.put(key, value); + } else { + remove(key); + } + return this; + } + + + /** + * Put a key/value pair in the JSONObject, but only if the key and the + * value are both non-null, and only if there is not already a member + * with that name. + * + * @param key + * @param value + * @return his. + * @throws JSONException if the key is a duplicate + */ + public JSONObject putOnce(String key, Object value) throws JSONException { + if (key != null && value != null) { + if (opt(key) != null) { + throw new JSONException("Duplicate key \"" + key + "\""); + } + put(key, value); + } + return this; + } + + + /** + * Put a key/value pair in the JSONObject, but only if the + * key and the value are both non-null. + * + * @param key A key string. + * @param value An object which is the value. It should be of one of these + * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, + * or the JSONObject.NULL object. + * @return this. + * @throws JSONException If the value is a non-finite number. + */ + public JSONObject putOpt(String key, Object value) throws JSONException { + if (key != null && value != null) { + put(key, value); + } + return this; + } + + + /** + * Produce a string in double quotes with backslash sequences in all the + * right places. A backslash will be inserted within = '\u0080' && c < '\u00a0') || + (c >= '\u2000' && c < '\u2100')) { + hhhh = "000" + Integer.toHexString(c); + sb.append("\\u").append(hhhh.substring(hhhh.length() - 4)); + } else { + sb.append(c); + } + } + } + sb.append('"'); + return sb.toString(); + } + + /** + * Remove a name and its value, if present. + * + * @param key The name to be removed. + * @return The value that was associated with the name, + * or null if there was no value. + */ + public Object remove(String key) { + return this.map.remove(key); + } + + /** + * Get an enumeration of the keys of the JSONObject. + * The keys will be sorted alphabetically. + * + * @return An iterator of the keys. + */ + public Iterator sortedKeys() { + return new TreeSet(this.map.keySet()).iterator(); + } + + /** + * Try to convert a string into a number, boolean, or null. If the string + * can't be converted, return the string. + * + * @param string A String. + * @return A simple JSON value. + */ + public static Object stringToValue(String string) { + if (string.equals("")) { + return string; + } + if (string.equalsIgnoreCase("true")) { + return Boolean.TRUE; + } + if (string.equalsIgnoreCase("false")) { + return Boolean.FALSE; + } + if (string.equalsIgnoreCase("null")) { + return JSONObject.NULL; + } + + /* + * If it might be a number, try converting it. + * We support the non-standard 0x- convention. + * If a number cannot be produced, then the value will just + * be a string. Note that the 0x-, plus, and implied string + * conventions are non-standard. A JSON parser may accept + * non-JSON forms as long as it accepts all correct JSON forms. + */ + + char b = string.charAt(0); + if ((b >= '0' && b <= '9') || b == '.' || b == '-' || b == '+') { + if (b == '0' && string.length() > 2 && + (string.charAt(1) == 'x' || string.charAt(1) == 'X')) { + try { + return Integer.parseInt(string.substring(2), 16); + } catch (Exception ignore) { + } + } + try { + if (string.indexOf('.') > -1 || + string.indexOf('e') > -1 || string.indexOf('E') > -1) { + return Double.valueOf(string); + } else { + Long myLong = new Long(string); + if (myLong == myLong.intValue()) { + return myLong.intValue(); + } else { + return myLong; + } + } + } catch (Exception ignore) { + } + } + return string; + } + + + /** + * Throw an exception if the object is a NaN or infinite number. + * + * @param o The object to test. + * @throws JSONException If o is a non-finite number. + */ + public static void testValidity(Object o) throws JSONException { + if (o != null) { + if (o instanceof Double) { + if (((Double) o).isInfinite() || ((Double) o).isNaN()) { + throw new JSONException( + "JSON does not allow non-finite numbers."); + } + } else if (o instanceof Float) { + if (((Float) o).isInfinite() || ((Float) o).isNaN()) { + throw new JSONException( + "JSON does not allow non-finite numbers."); + } + } + } + } + + /** + * Make a JSON text of this JSONObject. For compactness, no whitespace + * is added. If this would not result in a syntactically correct JSON text, + * then null will be returned instead. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return a printable, displayable, portable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + */ + public String toString() { + try { + Iterator keys = keys(); + StringBuilder sb = new StringBuilder("{"); + + while (keys.hasNext()) { + if (sb.length() > 1) { + sb.append(','); + } + Object o = keys.next(); + sb.append(quote(o.toString())); + sb.append(':'); + sb.append(valueToString(this.map.get(o))); + } + sb.append('}'); + return sb.toString(); + } catch (Exception e) { + return null; + } + } + + + /** + * Make a prettyprinted JSON text of this JSONObject. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @return a printable, displayable, portable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + public String toString(int indentFactor) throws JSONException { + return toString(indentFactor, 0); + } + + + /** + * Make a prettyprinted JSON text of this JSONObject. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indentation of the top level. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + String toString(int indentFactor, int indent) throws JSONException { + int i; + int length = this.length(); + if (length == 0) { + return "{}"; + } + Iterator keys = sortedKeys(); + int newindent = indent + indentFactor; + Object object; + StringBuilder sb = new StringBuilder("{"); + if (length == 1) { + object = keys.next(); + sb.append(quote(object.toString())); + sb.append(": "); + sb.append(valueToString(this.map.get(object), indentFactor, + indent)); + } else { + while (keys.hasNext()) { + object = keys.next(); + if (sb.length() > 1) { + sb.append(",\n"); + } else { + sb.append('\n'); + } + for (i = 0; i < newindent; i += 1) { + sb.append(' '); + } + sb.append(quote(object.toString())); + sb.append(": "); + sb.append(valueToString(this.map.get(object), indentFactor, + newindent)); + } + if (sb.length() > 1) { + sb.append('\n'); + for (i = 0; i < indent; i += 1) { + sb.append(' '); + } + } + } + sb.append('}'); + return sb.toString(); + } + + + /** + * Make a JSON text of an Object value. If the object has an + * value.toJSONString() method, then that method will be used to produce + * the JSON text. The method is required to produce a strictly + * conforming text. If the object does not contain a toJSONString + * method (which is the most common case), then a text will be + * produced by other means. If the value is an array or Collection, + * then a JSONArray will be made from it and its toJSONString method + * will be called. If the value is a MAP, then a JSONObject will be made + * from it and its toJSONString method will be called. Otherwise, the + * value's toString method will be called, and the result will be quoted. + *

+ *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param value The value to be serialized. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the value is or contains an invalid number. + */ + public static String valueToString(Object value) throws JSONException { + if (value == null || value.equals(null)) { + return "null"; + } + if (value instanceof Number) { + return numberToString((Number) value); + } + if (value instanceof Boolean || value instanceof JSONObject || + value instanceof JSONArray) { + return value.toString(); + } + if (value instanceof Map) { + return new JSONObject((Map) value).toString(); + } + if (value instanceof Collection) { + return new JSONArray((Collection) value).toString(); + } + if (value.getClass().isArray()) { + return new JSONArray(value).toString(); + } + return quote(value.toString()); + } + + + /** + * Make a prettyprinted JSON text of an object value. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @param value The value to be serialized. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indentation of the top level. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + static String valueToString(Object value, int indentFactor, int indent) + throws JSONException { + if (value == null || value.equals(null)) { + return "null"; + } + if (value instanceof Number) { + return numberToString((Number) value); + } + if (value instanceof Boolean) { + return value.toString(); + } + if (value instanceof JSONObject) { + return ((JSONObject) value).toString(indentFactor, indent); + } + if (value instanceof JSONArray) { + return ((JSONArray) value).toString(indentFactor, indent); + } + if (value instanceof Map) { + return new JSONObject((Map) value).toString(indentFactor, indent); + } + if (value instanceof Collection) { + return new JSONArray((Collection) value).toString(indentFactor, indent); + } + if (value.getClass().isArray()) { + return new JSONArray(value).toString(indentFactor, indent); + } + return quote(value.toString()); + } + + + /** + * Wrap an object, if necessary. If the object is null, return the NULL + * object. If it is an array or collection, wrap it in a JSONArray. If + * it is a map, wrap it in a JSONObject. If it is a standard property + * (Double, String, et al) then it is already wrapped. Otherwise, if it + * comes from one of the java packages, turn it into a string. And if + * it doesn't, try to wrap it in a JSONObject. If the wrapping fails, + * then null is returned. + * + * @param object The object to wrap + * @return The wrapped value + */ + public static Object wrap(Object object) { + try { + if (object == null) { + return NULL; + } + if (object instanceof JSONObject || object instanceof JSONArray || + NULL.equals(object) || + object instanceof Byte || object instanceof Character || + object instanceof Short || object instanceof Integer || + object instanceof Long || object instanceof Boolean || + object instanceof Float || object instanceof Double || + object instanceof String) { + return object; + } + + if (object instanceof Collection) { + return new JSONArray((Collection) object); + } + if (object.getClass().isArray()) { + return new JSONArray(object); + } + if (object instanceof Map) { + return new JSONObject((Map) object); + } + Package objectPackage = object.getClass().getPackage(); + String objectPackageName = (objectPackage != null ? objectPackage.getName() : ""); + if (objectPackageName.startsWith("java.") || + objectPackageName.startsWith("javax.") || + object.getClass().getClassLoader() == null) { + return object.toString(); + } + return new JSONObject(object); + } catch (Exception exception) { + return null; + } + } + + + /** + * Write the contents of the JSONObject as JSON text to a writer. + * For compactness, no whitespace is added. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return The writer. + * @throws JSONException + */ + public Writer write(Writer writer) throws JSONException { + try { + boolean commanate = false; + Iterator keys = keys(); + writer.write('{'); + + while (keys.hasNext()) { + if (commanate) { + writer.write(','); + } + Object key = keys.next(); + writer.write(quote(key.toString())); + writer.write(':'); + Object value = this.map.get(key); + if (value instanceof JSONObject) { + ((JSONObject) value).write(writer); + } else if (value instanceof JSONArray) { + ((JSONArray) value).write(writer); + } else { + writer.write(valueToString(value)); + } + commanate = true; + } + writer.write('}'); + return writer; + } catch (IOException exception) { + throw new JSONException(exception); + } + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/JSONTokener.java b/examples/src/main/java/druid/examples/twitter/JSONTokener.java new file mode 100644 index 00000000000..db694a335a1 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/JSONTokener.java @@ -0,0 +1,359 @@ +package druid.examples.twitter; + +import org.codehaus.jettison.json.JSONException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.StringReader; + + +import java.io.*; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/** + * A JSONTokener takes a source string and extracts characters and tokens from + * it. It is used by the JSONObject and JSONArray constructors to parse + * JSON source strings. + * + * @author JSON.org + * @version 2010-12-24 + */ +public class JSONTokener { + + private int character; + private boolean eof; + private int index; + private int line; + private char previous; + private Reader reader; + private boolean usePrevious; + + + /** + * Construct a JSONTokener from a Reader. + * + * @param reader A reader. + */ + public JSONTokener(Reader reader) { + this.reader = reader.markSupported() ? + reader : new BufferedReader(reader); + this.eof = false; + this.usePrevious = false; + this.previous = 0; + this.index = 0; + this.character = 1; + this.line = 1; + } + + + /** + * Construct a JSONTokener from an InputStream. + */ + public JSONTokener(InputStream inputStream) throws JSONException + { + this(new InputStreamReader(inputStream)); + } + + + /** + * Construct a JSONTokener from a string. + * + * @param s A source string. + */ + public JSONTokener(String s) { + this(new StringReader(s)); + } + + + /** + * Back up one character. This provides a sort of lookahead capability, + * so that you can test for a digit or letter before attempting to parse + * the next number or identifier. + */ + public void back() throws JSONException { + if (usePrevious || index <= 0) { + throw new JSONException("Stepping back two steps is not supported"); + } + this.index -= 1; + this.character -= 1; + this.usePrevious = true; + this.eof = false; + } + + + public boolean end() { + return eof && !usePrevious; + } + + + /** + * Determine if the source string still contains characters that next() + * can consume. + * + * @return true if not yet at the end of the source. + */ + public boolean more() throws JSONException { + next(); + if (end()) { + return false; + } + back(); + return true; + } + + + /** + * Get the next character in the source string. + * + * @return The next character, or 0 if past the end of the source string. + */ + public char next() throws JSONException { + int c; + if (this.usePrevious) { + this.usePrevious = false; + c = this.previous; + } else { + try { + c = this.reader.read(); + } catch (IOException exception) { + throw new JSONException(exception); + } + + if (c <= 0) { // End of stream + this.eof = true; + c = 0; + } + } + this.index += 1; + if (this.previous == '\r') { + this.line += 1; + this.character = c == '\n' ? 0 : 1; + } else if (c == '\n') { + this.line += 1; + this.character = 0; + } else { + this.character += 1; + } + this.previous = (char) c; + return this.previous; + } + + + /** + * Consume the next character, and check that it matches a specified + * character. + * + * @param c The character to match. + * @return The character. + * @throws JSONException if the character does not match. + */ + public char next(char c) throws JSONException { + char n = next(); + if (n != c) { + throw syntaxError("Expected '" + c + "' and instead saw '" + + n + "'"); + } + return n; + } + + + /** + * Get the next n characters. + * + * @param n The number of characters to take. + * @return A string of n characters. + * @throws JSONException Substring bounds error if there are not + * n characters remaining in the source string. + */ + public String next(int n) throws JSONException { + if (n == 0) { + return ""; + } + + char[] chars = new char[n]; + int pos = 0; + + while (pos < n) { + chars[pos] = next(); + if (end()) { + throw syntaxError("Substring bounds error"); + } + pos += 1; + } + return new String(chars); + } + + + /** + * Get the next char in the string, skipping whitespace. + * + * @return A character, or 0 if there are no more characters. + * @throws JSONException + */ + public char nextClean() throws JSONException { + for (; ; ) { + char c = next(); + if (c == 0 || c > ' ') { + return c; + } + } + } + + + /** + * Return the characters up to the next close quote character. + * Backslash processing is done. The formal JSON format does not + * allow strings in single quotes, but an implementation is allowed to + * accept them. + * + * @param quote The quoting character, either + * " (double quote) or + * ' (single quote). + * @return A String. + * @throws JSONException Unterminated string. + */ + public String nextString(char quote) throws JSONException { + char c; + StringBuilder sb = new StringBuilder(); + for (; ; ) { + c = next(); + switch (c) { + case 0: + case '\n': + case '\r': + throw syntaxError("Unterminated string"); + case '\\': + c = next(); + switch (c) { + case 'b': + sb.append('\b'); + break; + case 't': + sb.append('\t'); + break; + case 'n': + sb.append('\n'); + break; + case 'f': + sb.append('\f'); + break; + case 'r': + sb.append('\r'); + break; + case 'u': + sb.append((char) Integer.parseInt(next(4), 16)); + break; + case '"': + case '\'': + case '\\': + case '/': + sb.append(c); + break; + default: + throw syntaxError("Illegal escape."); + } + break; + default: + if (c == quote) { + return sb.toString(); + } + sb.append(c); + } + } + } + + /** + * Get the next value. The value can be a Boolean, Double, Integer, + * JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object. + * + * @return An object. + * @throws JSONException If syntax error. + */ + public Object nextValue() throws JSONException { + char c = nextClean(); + String string; + + switch (c) { + case '"': + case '\'': + return nextString(c); + case '{': + back(); + return new JSONObject(this); + case '[': + back(); + return new JSONArray(this); + } + + /* + * Handle unquoted text. This could be the values true, false, or + * null, or it can be a number. An implementation (such as this one) + * is allowed to also accept non-standard forms. + * + * Accumulate characters until we reach the end of the text or a + * formatting character. + */ + + StringBuilder sb = new StringBuilder(); + while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) { + sb.append(c); + c = next(); + } + back(); + + string = sb.toString().trim(); + if (string.equals("")) { + throw syntaxError("Missing value"); + } + return JSONObject.stringToValue(string); + } + + + + /** + * Make a JSONException to signal a syntax error. + * + * @param message The error message. + * @return A JSONException object, suitable for throwing + */ + public JSONException syntaxError(String message) { + return new JSONException(message + toString()); + } + + + /** + * Make a printable string of this JSONTokener. + * + * @return " at {index} [character {character} line {line}]" + */ + public String toString() { + return " at " + index + " [character " + this.character + " line " + + this.line + "]"; + } +} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/twitter/Logger.java b/examples/src/main/java/druid/examples/twitter/Logger.java new file mode 100644 index 00000000000..0f9a4eb3769 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/Logger.java @@ -0,0 +1,180 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + + + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + * @since Twitter4J 2.1.0 + */ +public abstract class Logger { + private static final LoggerFactory LOGGER_FACTORY; + private static final String LOGGER_FACTORY_IMPLEMENTATION = "twitter4j.loggerFactory"; + + static { + LoggerFactory loggerFactory = null; + // -Dtwitter4j.debug=true -Dtwitter4j.loggerFactory=twitter4j.internal.logging.StdOutLoggerFactory + String loggerFactoryImpl = System.getProperty(LOGGER_FACTORY_IMPLEMENTATION); + if (loggerFactoryImpl != null) { + loggerFactory = getLoggerFactoryIfAvailable(loggerFactoryImpl, loggerFactoryImpl); + } + + Configuration conf = ConfigurationContext.getInstance(); + // configuration in twitter4j.properties + // loggerFactory=twitter4j.internal.logging.StdOutLoggerFactory + loggerFactoryImpl = conf.getLoggerFactory(); + if (loggerFactoryImpl != null) { + loggerFactory = getLoggerFactoryIfAvailable(loggerFactoryImpl, loggerFactoryImpl); + } + // use SLF4J if it's found in the classpath + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("org.slf4j.impl.StaticLoggerBinder", "twitter4j.internal.logging.SLF4JLoggerFactory"); + } + // otherwise, use commons-logging if it's found in the classpath + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("org.apache.commons.logging.Log", "twitter4j.internal.logging.CommonsLoggingLoggerFactory"); + } + // otherwise, use log4j if it's found in the classpath + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("org.apache.log4j.Logger", "twitter4j.internal.logging.Log4JLoggerFactory"); + } + // on Google App Engine, use java.util.logging + if (null == loggerFactory) { + loggerFactory = getLoggerFactoryIfAvailable("com.google.appengine.api.urlfetch.URLFetchService", "twitter4j.internal.logging.JULLoggerFactory"); + } + // otherwise, use the default logger +// if (null == loggerFactory) { +// loggerFactory = new StdOutLoggerFactory(); +// } + LOGGER_FACTORY = loggerFactory; + + try { + Method method = conf.getClass().getMethod("dumpConfiguration", new Class[]{}); + method.setAccessible(true); + method.invoke(conf); + } catch (IllegalAccessException ignore) { + } catch (InvocationTargetException ignore) { + } catch (NoSuchMethodException ignore) { + } + } + + private static LoggerFactory getLoggerFactoryIfAvailable(String checkClassName, String implementationClass) { + try { + Class.forName(checkClassName); + return (LoggerFactory) Class.forName(implementationClass).newInstance(); + } catch (ClassNotFoundException ignore) { + } catch (InstantiationException e) { + throw new AssertionError(e); + } catch (SecurityException ignore) { + // Unsigned applets are not allowed to access System properties + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } + return null; + } + + /** + * Returns a Logger instance associated with the specified class. + * + * @param clazz class + * @return logger instance + */ + public static Logger getLogger(Class clazz) { + return LOGGER_FACTORY.getLogger(clazz); + } + + /** + * tests if debug level logging is enabled + * + * @return if debug level logging is enabled + */ + public abstract boolean isDebugEnabled(); + + /** + * tests if info level logging is enabled + * + * @return if info level logging is enabled + */ + public abstract boolean isInfoEnabled(); + + /** + * tests if warn level logging is enabled + * + * @return if warn level logging is enabled + */ + public abstract boolean isWarnEnabled(); + + /** + * tests if error level logging is enabled + * + * @return if error level logging is enabled + */ + public abstract boolean isErrorEnabled(); + + /** + * @param message message + */ + public abstract void debug(String message); + + /** + * @param message message + * @param message2 message2 + */ + public abstract void debug(String message, String message2); + + /** + * @param message message + */ + public abstract void info(String message); + + /** + * @param message message + * @param message2 message2 + */ + public abstract void info(String message, String message2); + + /** + * @param message message + */ + public abstract void warn(String message); + + /** + * @param message message + * @param message2 message2 + */ + public abstract void warn(String message, String message2); + + /** + * @param message message + */ + public abstract void error(String message); + + /** + * @param message message + * @param th throwable + */ + public abstract void error(String message, Throwable th); + +} diff --git a/examples/src/main/java/druid/examples/twitter/LoggerFactory.java b/examples/src/main/java/druid/examples/twitter/LoggerFactory.java new file mode 100644 index 00000000000..5dece70aedc --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/LoggerFactory.java @@ -0,0 +1,34 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import twitter4j.internal.logging.*; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + * @since Twitter4J 2.1.1 + */ +public abstract class LoggerFactory { + + /** + * Returns a logger associated with the specified class. + * + * @param clazz class + * @return a logger instance + */ + public abstract Logger getLogger(Class clazz); +} diff --git a/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java b/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java new file mode 100644 index 00000000000..a69da55a7e1 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/PropertyConfiguration.java @@ -0,0 +1,325 @@ +package druid.examples.twitter; +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectStreamException; +import java.util.ArrayList; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import java.io.*; +import java.util.ArrayList; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + */ +public final class PropertyConfiguration extends WebConfigurationBase implements java.io.Serializable { + + public static final String DEBUG = "debug"; + public static final String HTTP_USER_AGENT = "http.userAgent"; + + public static final String HTTP_USE_SSL = "http.useSSL"; + public static final String HTTP_PRETTY_DEBUG = "http.prettyDebug"; + public static final String HTTP_GZIP = "http.gzip"; + public static final String HTTP_PROXY_HOST = "http.proxyHost"; + public static final String HTTP_PROXY_HOST_FALLBACK = "http.proxyHost"; + public static final String HTTP_PROXY_USER = "http.proxyUser"; + public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword"; + public static final String HTTP_PROXY_PORT = "http.proxyPort"; + public static final String HTTP_PROXY_PORT_FALLBACK = "http.proxyPort"; + public static final String HTTP_CONNECTION_TIMEOUT = "http.connectionTimeout"; + public static final String HTTP_READ_TIMEOUT = "http.readTimeout"; + + public static final String HTTP_STREAMING_READ_TIMEOUT = "http.streamingReadTimeout"; + + public static final String HTTP_RETRY_COUNT = "http.retryCount"; + public static final String HTTP_RETRY_INTERVAL_SECS = "http.retryIntervalSecs"; + + public static final String HTTP_MAX_TOTAL_CONNECTIONS = "http.maxTotalConnections"; + public static final String HTTP_DEFAULT_MAX_PER_ROUTE = "http.defaultMaxPerRoute"; + + + public static final String STREAM_BASE_URL = "streamBaseURL"; + + public static final String ASYNC_NUM_THREADS = "async.numThreads"; + public static final String CONTRIBUTING_TO = "contributingTo"; + public static final String ASYNC_DISPATCHER_IMPL = "async.dispatcherImpl"; + public static final String LOGGER_FACTORY = "loggerFactory"; + public static final String JSON_STORE_ENABLED = "jsonStoreEnabled"; + public static final String STREAM_STALL_WARNINGS_ENABLED = "stream.enableStallWarnings"; + + // hidden portion + private static final long serialVersionUID = 6458764415636588373L; + + public PropertyConfiguration(InputStream is) { + super(); + Properties props = new Properties(); + loadProperties(props, is); + setFieldsWithTreePath(props, "/"); + } + + public PropertyConfiguration(Properties props) { + this(props, "/"); + } + + public PropertyConfiguration(Properties props, String treePath) { + super(); + setFieldsWithTreePath(props, treePath); + } + + PropertyConfiguration(String treePath) { + super(); + Properties props; + // load from system properties + try { + props = (Properties) System.getProperties().clone(); + try { + Map envMap = System.getenv(); + for(String key :envMap.keySet()){ + props.setProperty(key, envMap.get(key)); + } + }catch(SecurityException ignore){} + normalize(props); + } catch (SecurityException ignore) { + // Unsigned applets are not allowed to access System properties + props = new Properties(); + } +// final String TWITTER4J_PROPERTIES = "twitter4j.properties"; +// // override System properties with ./twitter4j.properties in the classpath +// loadProperties(props, "." + File.separatorChar + TWITTER4J_PROPERTIES); +// // then, override with /twitter4j.properties in the classpath +// loadProperties(props, twitter4j.conf.Configuration.class.getResourceAsStream("/" + TWITTER4J_PROPERTIES)); +// // then, override with /WEB/INF/twitter4j.properties in the classpath +// loadProperties(props, twitter4j.conf.Configuration.class.getResourceAsStream("/WEB-INF/" + TWITTER4J_PROPERTIES)); +// // for Google App Engine +// try { +// loadProperties(props, new FileInputStream("WEB-INF/" + TWITTER4J_PROPERTIES)); +// } catch (SecurityException ignore) { +// } catch (FileNotFoundException ignore) { +// } + + setFieldsWithTreePath(props, treePath); + } + + /** + * Creates a root PropertyConfiguration. This constructor is equivalent to new PropertyConfiguration("/"). + */ + PropertyConfiguration() { + this("/"); + } + + private boolean notNull(Properties props, String prefix, String name) { + return props.getProperty(prefix + name) != null; + } + + private boolean loadProperties(Properties props, String path) { + FileInputStream fis = null; + try { + File file = new File(path); + if (file.exists() && file.isFile()) { + fis = new FileInputStream(file); + props.load(fis); + normalize(props); + return true; + } + } catch (Exception ignore) { + } finally { + try { + if (fis != null) { + fis.close(); + } + } catch (IOException ignore) { + + } + } + return false; + } + + private boolean loadProperties(Properties props, InputStream is) { + try { + props.load(is); + normalize(props); + return true; + } catch (Exception ignore) { + } + return false; + } + + private void normalize(Properties props) { + Set keys = props.keySet(); + ArrayList toBeNormalized = new ArrayList(10); + for (Object key : keys) { + String keyStr = (String) key; + if (-1 != (keyStr.indexOf("twitter4j."))) { + toBeNormalized.add(keyStr); + } + } + for (String keyStr : toBeNormalized) { + String property = props.getProperty(keyStr); + int index = keyStr.indexOf("twitter4j."); + String newKey = keyStr.substring(0, index) + keyStr.substring(index + 10); + props.setProperty(newKey, property); + } + } + + /** + * passing "/foo/bar" as treePath will result:
+ * 1. load [twitter4j.]restBaseURL
+ * 2. override the value with foo.[twitter4j.]restBaseURL
+ * 3. override the value with foo.bar.[twitter4j.]restBaseURL
+ * + * @param props properties to be loaded + * @param treePath the path + */ + private void setFieldsWithTreePath(Properties props, String treePath) { + setFieldsWithPrefix(props, ""); + String[] splitArray = z_InternalStringUtil.split(treePath, "/"); + String prefix = null; + for (String split : splitArray) { + if (!"".equals(split)) { + if (null == prefix) { + prefix = split + "."; + } else { + prefix += split + "."; + } + setFieldsWithPrefix(props, prefix); + } + } + } + + private void setFieldsWithPrefix(Properties props, String prefix) { + if (notNull(props, prefix, DEBUG)) { + setDebug(getBoolean(props, prefix, DEBUG)); + } + + if (notNull(props, prefix, HTTP_USE_SSL)) { + setUseSSL(getBoolean(props, prefix, HTTP_USE_SSL)); + } + if (notNull(props, prefix, HTTP_PRETTY_DEBUG)) { + setPrettyDebugEnabled(getBoolean(props, prefix, HTTP_PRETTY_DEBUG)); + } + if (notNull(props, prefix, HTTP_GZIP)) { + setGZIPEnabled(getBoolean(props, prefix, HTTP_GZIP)); + } + if (notNull(props, prefix, HTTP_PROXY_HOST)) { + setHttpProxyHost(getString(props, prefix, HTTP_PROXY_HOST)); + } else if (notNull(props, prefix, HTTP_PROXY_HOST_FALLBACK)) { + setHttpProxyHost(getString(props, prefix, HTTP_PROXY_HOST_FALLBACK)); + } + if (notNull(props, prefix, HTTP_PROXY_USER)) { + setHttpProxyUser(getString(props, prefix, HTTP_PROXY_USER)); + } + if (notNull(props, prefix, HTTP_PROXY_PASSWORD)) { + setHttpProxyPassword(getString(props, prefix, HTTP_PROXY_PASSWORD)); + } + if (notNull(props, prefix, HTTP_PROXY_PORT)) { + setHttpProxyPort(getIntProperty(props, prefix, HTTP_PROXY_PORT)); + } else if (notNull(props, prefix, HTTP_PROXY_PORT_FALLBACK)) { + setHttpProxyPort(getIntProperty(props, prefix, HTTP_PROXY_PORT_FALLBACK)); + } + if (notNull(props, prefix, HTTP_CONNECTION_TIMEOUT)) { + setHttpConnectionTimeout(getIntProperty(props, prefix, HTTP_CONNECTION_TIMEOUT)); + } + if (notNull(props, prefix, HTTP_READ_TIMEOUT)) { + setHttpReadTimeout(getIntProperty(props, prefix, HTTP_READ_TIMEOUT)); + } + if (notNull(props, prefix, HTTP_STREAMING_READ_TIMEOUT)) { + setHttpStreamingReadTimeout(getIntProperty(props, prefix, HTTP_STREAMING_READ_TIMEOUT)); + } + if (notNull(props, prefix, HTTP_RETRY_COUNT)) { + setHttpRetryCount(getIntProperty(props, prefix, HTTP_RETRY_COUNT)); + } + if (notNull(props, prefix, HTTP_RETRY_INTERVAL_SECS)) { + setHttpRetryIntervalSeconds(getIntProperty(props, prefix, HTTP_RETRY_INTERVAL_SECS)); + } + if (notNull(props, prefix, HTTP_MAX_TOTAL_CONNECTIONS)) { + setHttpMaxTotalConnections(getIntProperty(props, prefix, HTTP_MAX_TOTAL_CONNECTIONS)); + } + if (notNull(props, prefix, HTTP_DEFAULT_MAX_PER_ROUTE)) { + setHttpDefaultMaxPerRoute(getIntProperty(props, prefix, HTTP_DEFAULT_MAX_PER_ROUTE)); + } + if (notNull(props, prefix, ASYNC_NUM_THREADS)) { + setAsyncNumThreads(getIntProperty(props, prefix, ASYNC_NUM_THREADS)); + } + if (notNull(props, prefix, CONTRIBUTING_TO)) { + setContributingTo(getLongProperty(props, prefix, CONTRIBUTING_TO)); + } + if (notNull(props, prefix, ASYNC_DISPATCHER_IMPL)) { + setDispatcherImpl(getString(props, prefix, ASYNC_DISPATCHER_IMPL)); + } + if (notNull(props, prefix, HTTP_USER_AGENT)) { + setUserAgent(getString(props, prefix, HTTP_USER_AGENT)); + } + + + if (notNull(props, prefix, STREAM_BASE_URL)) { + setStreamBaseURL(getString(props, prefix, STREAM_BASE_URL)); + } + if (notNull(props, prefix, LOGGER_FACTORY)) { + setLoggerFactory(getString(props, prefix, LOGGER_FACTORY)); + } + if (notNull(props, prefix, JSON_STORE_ENABLED)) { + setJSONStoreEnabled(getBoolean(props, prefix, JSON_STORE_ENABLED)); + } + if (notNull(props, prefix, STREAM_STALL_WARNINGS_ENABLED)) { + setStallWarningsEnabled(getBoolean(props, prefix, STREAM_STALL_WARNINGS_ENABLED)); + } + + cacheInstance(); + } + + protected boolean getBoolean(Properties props, String prefix, String name) { + String value = props.getProperty(prefix + name); + return Boolean.valueOf(value); + } + + protected int getIntProperty(Properties props, String prefix, String name) { + String value = props.getProperty(prefix + name); + try { + return Integer.parseInt(value); + } catch (NumberFormatException nfe) { + return -1; + } + } + + protected long getLongProperty(Properties props, String prefix, String name) { + String value = props.getProperty(prefix + name); + try { + return Long.parseLong(value); + } catch (NumberFormatException nfe) { + return -1L; + } + } + + protected String getString(Properties props, String prefix, String name) { + return props.getProperty(prefix + name); + } + + // assures equality after deserialization + protected Object readResolve() throws ObjectStreamException + { + return super.readResolve(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java b/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java new file mode 100644 index 00000000000..7926874be2a --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/PropertyConfigurationFactory.java @@ -0,0 +1,61 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * ConfigurationFactory implementation for PropertyConfiguration. + * Currently getInstance calls concrete constructor each time. No caching at all. + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +class PropertyConfigurationFactory implements ConfigurationFactory { + private static final PropertyConfiguration ROOT_CONFIGURATION; + + static { + ROOT_CONFIGURATION = new PropertyConfiguration(); + // calling ROOT_CONFIGURATION.dumpConfiguration() will cause ExceptionInInitializerError as Logger has not been initialized. + // as a quick and dirty solution, static initializer of twitter4j.internal.logging.Logger will call dumpConfiguration() on behalf. + } + + /** + * {@inheritDoc} + */ + @Override + public Configuration getInstance() { + return ROOT_CONFIGURATION; + } + + // It may be preferable to cache the config instance + + /** + * {@inheritDoc} + */ + @Override + public Configuration getInstance(String configTreePath) { + PropertyConfiguration conf = new PropertyConfiguration(configTreePath); + conf.dumpConfiguration(); + return conf; + } + + /** + * {@inheritDoc} + */ + @Override + public void dispose() { + // nothing to do for property based configuration + } +} diff --git a/examples/src/main/java/druid/examples/twitter/RequestMethod.java b/examples/src/main/java/druid/examples/twitter/RequestMethod.java new file mode 100644 index 00000000000..16fea4c21b0 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/RequestMethod.java @@ -0,0 +1,26 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @author Dan Checkoway - dcheckoway at gmail.com + */ +public enum RequestMethod { + GET, POST, DELETE, HEAD, PUT +} + diff --git a/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java b/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java new file mode 100644 index 00000000000..ecbd9e9b1f0 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/StreamingGZipInputStream.java @@ -0,0 +1,53 @@ +package druid.examples.twitter; +/* + * Copyright 2007 Yusuke Yamamoto + * Copyright 2012 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +final class StreamingGZIPInputStream extends GZIPInputStream +{ + + private final InputStream wrapped; + + public StreamingGZIPInputStream(InputStream is) throws IOException + { + super(is); + wrapped = is; + } + + /** + * Overrides behavior of GZIPInputStream which assumes we have all the data available + * which is not true for streaming. We instead rely on the underlying stream to tell us + * how much data is available. + *

+ * Programs should not count on this method to return the actual number + * of bytes that could be read without blocking. + * + * @return - whatever the wrapped InputStream returns + * @throws IOException if an I/O error occurs. + */ + @Override + public int available() throws IOException { + return wrapped.available(); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java index e87b5e44de2..ea1d8331924 100644 --- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java @@ -139,7 +139,7 @@ public class TwitterSpritzerFirehoseFactory implements FirehoseFactory { public void onStatus(Status status) { // time to stop? - if (Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { diff --git a/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java new file mode 100644 index 00000000000..7f880d23959 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/USGovFirehoseFactory.java @@ -0,0 +1,173 @@ +//package druid.examples.twitter; +// +//import com.fasterxml.jackson.annotation.JsonCreator; +//import com.google.common.collect.Maps; +//import com.metamx.common.logger.Logger; +//import com.metamx.druid.input.InputRow; +//import com.metamx.druid.input.MapBasedInputRow; +//import com.metamx.druid.realtime.firehose.Firehose; +//import com.metamx.druid.realtime.firehose.FirehoseFactory; +//import org.codehaus.jackson.annotate.JsonProperty; +//import org.codehaus.jackson.annotate.JsonTypeName; +//import twitter4j.ConnectionLifeCycleListener; +//import twitter4j.Status; +// +//import java.io.IOException; +//import java.util.LinkedList; +//import java.util.Map; +//import java.util.concurrent.ArrayBlockingQueue; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.TimeUnit; +// +//import static java.lang.Thread.sleep; +// +///** +//* Created with IntelliJ IDEA. +//* User: dhruvparthasarathy +//* Date: 6/18/13 +//* Time: 4:52 PM +//* To change this template use File | Settings | File Templates. +//*/ +//@JsonTypeName("Gov") +// +//public class USGovFirehoseFactory implements FirehoseFactory +//{ +// private static final Logger log = new Logger(TwitterSpritzerFirehoseFactory.class); +// +// private final int maxEventCount; +// +// private final int rowCount; +// +// @JsonCreator +// public USGovFirehoseFactory( +// @JsonProperty("maxEventCount") Integer maxEventCount, +// @JsonProperty("rowCount") Integer rowCount +// ) +// { +// this.maxEventCount=maxEventCount; +// this.rowCount=rowCount; +// log.info("maxEventCount=" + ((maxEventCount <= 0) ? "no limit" : maxEventCount)); +// log.info("rowCount=" + ((rowCount <= 0) ? "no limit" : rowCount)); +// } +// +// @Override +// public Firehose connect() throws IOException +// { +// final LinkedList dimensions = new LinkedList(); +// final int QUEUE_SIZE = 2000; +// final BlockingQueue queue = new ArrayBlockingQueue(QUEUE_SIZE); +// dimensions.add("device"); +// dimensions.add("country_code"); +// dimensions.add("known_user"); +// dimensions.add("base_url"); +// dimensions.add("referring_url"); +// dimensions.add("full_url"); +// dimensions.add("timestamp"); +// dimensions.add("city"); +// dimensions.add("tz"); +// +// WebListener listener = new WebListener() { +// @Override +// public void onUpdate(Update update) +// { +// if (Thread.currentThread().isInterrupted()) { +// throw new RuntimeException("Interrupted, time to stop"); +// } +// try { +// boolean success = queue.offer(update, 15L, TimeUnit.SECONDS); +// if (!success){ +// log.warn("queue too slow!"); +// } +// } +// catch (InterruptedException e){ +// throw new RuntimeException("InterruptedException", e); +// } +// } +// }; +// return new Firehose() +// { +// final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { +// @Override +// public void onConnect() +// { +// log.info("Connected_to_stream"); +// } +// +// @Override +// public void onDisconnect() +// { +// log.info("Disconnected_from_stream"); +// } +// +// @Override +// public void onCleanUp() +// { +// log.info("Cleanup_stream"); +// } +// }; +// private final Runnable doNothingRunnable = new Runnable() { +// public void run() +// { +// } +// }; +// private boolean waitIfmax = true; +// @Override +// public boolean hasMore() +// { +// if (maxEventCount >=0 && rowCount >= maxEventCount){ +// return waitIfmax; +// } +// else +// { +// return true; +// } +// } +// +// @Override +// public InputRow nextRow() +// { +// if (maxEventCount >=0 && rowCount >=maxEventCount && waitIfmax){ +// try { +// sleep(2000000000L); +// } +// catch (InterruptedException e) { +// throw new RuntimeException("InterruptedException"); +// } +// } +// Update udpate; +// try{ +// update=queue.take(); +// } +// catch (InterruptedException e) { +// throw new RuntimeException("InterruptedException", e); +// } +// final Map theMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); +// theMap.put("device", update.device()); +// theMap.put("country_code", update.country()); +// theMap.put("known_user", update.knownUser()); +// theMap.put("base_url", update.baseUrl()); +// theMap.put("referring_url", update.referringUrl()); +// theMap.put("full_url", update.fullUrl()); +// theMap.put("timestamp", update.timestamp()); +// theMap.put("city", update.city()); +// theMap.put("tz", update.tz()); +// +// return new MapBasedInputRow(update.timestamp,dimensions,theMap); //To change body of implemented methods use File | Settings | File Templates. +// } +// +// @Override +// public Runnable commit() +// { +// return doNothingRunnable; //To change body of implemented methods use File | Settings | File Templates. +// } +// +// @Override +// public void close() throws IOException +// { +// //To change body of implemented methods use File | Settings | File Templates. +// } +// +// }; +// +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebBase.java b/examples/src/main/java/druid/examples/twitter/WebBase.java new file mode 100644 index 00000000000..4e223b8ebd9 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebBase.java @@ -0,0 +1,15 @@ +package druid.examples.twitter; + +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/19/13 + * Time: 4:15 PM + * To change this template use File | Settings | File Templates. + */ +public interface WebBase +{ + Configuration getConfiguration(); + + void shutdown(); +} diff --git a/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java b/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java new file mode 100644 index 00000000000..30310e89739 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebBaseImpl.java @@ -0,0 +1,170 @@ +//package druid.examples.twitter; +// +///* +// * Copyright 2007 Yusuke Yamamoto +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +// +//import java.io.IOException; +//import java.io.ObjectInputStream; +//import java.util.ArrayList; +//import java.util.List; +// +// +//import java.io.IOException; +//import java.io.ObjectInputStream; +//import java.util.ArrayList; +//import java.util.List; +// +///** +// * Base class of Twitter / AsyncTwitter / TwitterStream supports OAuth. +// * +// * @author Yusuke Yamamoto - yusuke at mac.com +// */ +//abstract class WebBaseImpl implements WebBase, java.io.Serializable, HttpResponseListener +//{ +// protected Configuration conf; +// protected transient String screenName = null; +// protected transient long id = 0; +// +// protected transient HttpClientWrapper http; +// +// protected z_T4JInternalFactory factory; +// +// private static final long serialVersionUID = -3812176145960812140L; +// +// /*package*/ WebBaseImpl(Configuration conf) { +// this.conf = conf; +// init(); +// } +// +// private void init() { +// +// http = new HttpClientWrapper(conf); +// http.setHttpResponseListener(this); +// setFactory(); +// } +// +// protected void setFactory() { +// factory = new z_T4JInternalJSONImplFactory(conf); +// } +// +// /** +// * {@inheritDoc} +// */ +// +// @Override +// public void httpResponseReceived(HttpResponseEvent event) { +// if (rateLimitStatusListeners.size() != 0) { +// HttpResponse res = event.getResponse(); +// TwitterException te = event.getTwitterException(); +// RateLimitStatus rateLimitStatus; +// int statusCode; +// if (te != null) { +// rateLimitStatus = te.getRateLimitStatus(); +// statusCode = te.getStatusCode(); +// } else { +// rateLimitStatus = z_T4JInternalJSONImplFactory.createRateLimitStatusFromResponseHeader(res); +// statusCode = res.getStatusCode(); +// } +// if (rateLimitStatus != null) { +// RateLimitStatusEvent statusEvent +// = new RateLimitStatusEvent(this, rateLimitStatus, event.isAuthenticated()); +// if (statusCode == ENHANCE_YOUR_CLAIM +// || statusCode == SERVICE_UNAVAILABLE) { +// // EXCEEDED_RATE_LIMIT_QUOTA is returned by Rest API +// // SERVICE_UNAVAILABLE is returned by Search API +// for (RateLimitStatusListener listener : rateLimitStatusListeners) { +// listener.onRateLimitStatus(statusEvent); +// listener.onRateLimitReached(statusEvent); +// } +// } else { +// for (RateLimitStatusListener listener : rateLimitStatusListeners) { +// listener.onRateLimitStatus(statusEvent); +// } +// } +// } +// } +// } +// +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public Configuration getConfiguration() { +// return this.conf; +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void shutdown() { +// if (http != null) http.shutdown(); +// } +// +// +// private void writeObject(java.io.ObjectOutputStream out) throws IOException +// { +// // http://docs.oracle.com/javase/6/docs/platform/serialization/spec/output.html#861 +// out.putFields(); +// out.writeFields(); +// +// out.writeObject(conf); +// } +// +// private void readObject(ObjectInputStream stream) +// throws IOException, ClassNotFoundException { +// // http://docs.oracle.com/javase/6/docs/platform/serialization/spec/input.html#2971 +// stream.readFields(); +// +// conf = (Configuration) stream.readObject(); +// http = new HttpClientWrapper(conf); +// http.setHttpResponseListener(this); +// setFactory(); +// } +// +// +// +// @Override +// public boolean equals(Object o) { +// if (this == o) return true; +// if (!(o instanceof WebBaseImpl)) return false; +// +// WebBaseImpl that = (WebBaseImpl) o; +// +// if (!conf.equals(that.conf)) return false; +// if (http != null ? !http.equals(that.http) : that.http != null) +// return false; +// +// return true; +// } +// +// @Override +// public int hashCode() { +// int result = conf.hashCode(); +// result = 31 * result + (http != null ? http.hashCode() : 0); +// return result; +// } +// +// @Override +// public String toString() { +// return "TwitterBase{" + +// "conf=" + conf + +// ", http=" + http + +// '}'; +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java b/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java new file mode 100644 index 00000000000..64854151f31 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebConfigurationBase.java @@ -0,0 +1,494 @@ +package druid.examples.twitter; +/** + * Created with IntelliJ IDEA. + * User: dhruvparthasarathy + * Date: 6/19/13 + * Time: 2:02 PM + * To change this template use File | Settings | File Templates. + */ + +import druid.examples.twitter.Configuration; +import twitter4j.internal.util.z_T4JInternalStringUtil; + +import java.io.ObjectStreamException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + + +import java.io.ObjectStreamException; +import java.lang.reflect.Field; +import java.util.*; +import java.util.logging.Logger; + +/** + * Configuration base class with default settings. + * + * @author Yusuke Yamamoto - yusuke at mac.com + */ +class WebConfigurationBase implements Configuration, java.io.Serializable { + private boolean debug; + private boolean useSSL; + private boolean prettyDebug; + private boolean gzipEnabled; + private String httpProxyHost; + private String httpProxyUser; + private String httpProxyPassword; + private String userAgent; + private int httpProxyPort; + private int httpConnectionTimeout; + private int httpReadTimeout; + + private int httpStreamingReadTimeout; + private int httpRetryCount; + private int httpRetryIntervalSeconds; + private int maxTotalConnections; + private int defaultMaxPerRoute; + + private String streamBaseURL; + + private String dispatcherImpl; + private String loggerFactory; + + private int asyncNumThreads; + + private long contributingTo; + private boolean includeEntitiesEnabled = true; + + private boolean jsonStoreEnabled; + + + private boolean stallWarningsEnabled; + + + + private static final String DEFAULT_STREAM_BASE_URL = "http://developer.usa.gov/1usagov/"; + + private static final long serialVersionUID = -6610497517837844232L; + + + protected WebConfigurationBase() { + setDebug(false); + setUseSSL(false); + setPrettyDebugEnabled(false); + setGZIPEnabled(true); + setHttpProxyHost(null); + setHttpProxyUser(null); + setHttpProxyPassword(null); + setUserAgent(null); + setHttpProxyPort(-1); + setHttpConnectionTimeout(20000); + setHttpReadTimeout(120000); + setHttpStreamingReadTimeout(40 * 1000); + setHttpRetryCount(0); + setHttpRetryIntervalSeconds(5); + setHttpMaxTotalConnections(20); + setHttpDefaultMaxPerRoute(2); + setAsyncNumThreads(1); + setContributingTo(-1L); + + setJSONStoreEnabled(false); + + + setStreamBaseURL(DEFAULT_STREAM_BASE_URL); + + setDispatcherImpl("twitter4j.internal.async.DispatcherImpl"); + setLoggerFactory(null); + + setStallWarningsEnabled(true); + + } + + public void dumpConfiguration() { + if (debug) { +// Field[] fields = WebConfigurationBase.class.getDeclaredFields(); +// for (Field field : fields) { +// try { +// Object value = field.get(this); +// String strValue = String.valueOf(value); +// if (value != null && field.getName().matches("oAuthConsumerSecret|oAuthAccessTokenSecret|password")) { +// strValue = z_T4JInternalStringUtil.maskString(String.valueOf(value)); +// } +// log.debug(field.getName() + ": " + strValue); +// } catch (IllegalAccessException ignore) { +// } +// } + } + } + + @Override + public final boolean isDebugEnabled() { + return debug; + } + + protected final void setDebug(boolean debug) { + this.debug = debug; + } + + + @Override + public boolean isPrettyDebugEnabled() { + return prettyDebug; + } + + protected final void setUseSSL(boolean useSSL) { + this.useSSL = useSSL; + } + + protected final void setPrettyDebugEnabled(boolean prettyDebug) { + this.prettyDebug = prettyDebug; + } + + protected final void setGZIPEnabled(boolean gzipEnabled) { + this.gzipEnabled = gzipEnabled; + initRequestHeaders(); + } + + @Override + public boolean isGZIPEnabled() { + return gzipEnabled; + } + + // method for HttpRequestFactoryConfiguration + Map requestHeaders; + + private void initRequestHeaders() { +// requestHeaders = new HashMap(); +// requestHeaders.put("X-Twitter-Client-Version", getClientVersion()); +// requestHeaders.put("X-Twitter-Client-URL", getClientURL()); +// requestHeaders.put("X-Twitter-Client", "Twitter4J"); +// +// requestHeaders.put("User-Agent", getUserAgent()); +// if (gzipEnabled) { +// requestHeaders.put("Accept-Encoding", "gzip"); +// } +// if (IS_DALVIK) { +// requestHeaders.put("Connection", "close"); +// } + } + + @Override + public Map getRequestHeaders() { + return requestHeaders; + } + + // methods for HttpClientConfiguration + + @Override + public final String getHttpProxyHost() { + return httpProxyHost; + } + + protected final void setHttpProxyHost(String proxyHost) { + this.httpProxyHost = proxyHost; + } + + @Override + public final String getHttpProxyUser() { + return httpProxyUser; + } + + protected final void setHttpProxyUser(String proxyUser) { + this.httpProxyUser = proxyUser; + } + + @Override + public final String getHttpProxyPassword() { + return httpProxyPassword; + } + + protected final void setHttpProxyPassword(String proxyPassword) { + this.httpProxyPassword = proxyPassword; + } + + @Override + public final int getHttpProxyPort() { + return httpProxyPort; + } + + protected final void setHttpProxyPort(int proxyPort) { + this.httpProxyPort = proxyPort; + } + + @Override + public final int getHttpConnectionTimeout() { + return httpConnectionTimeout; + } + + protected final void setHttpConnectionTimeout(int connectionTimeout) { + this.httpConnectionTimeout = connectionTimeout; + } + + @Override + public final int getHttpReadTimeout() { + return httpReadTimeout; + } + + protected final void setHttpReadTimeout(int readTimeout) { + this.httpReadTimeout = readTimeout; + } + + @Override + public int getHttpStreamingReadTimeout() { + return httpStreamingReadTimeout; + } + + protected final void setHttpStreamingReadTimeout(int httpStreamingReadTimeout) { + this.httpStreamingReadTimeout = httpStreamingReadTimeout; + } + + + @Override + public final int getHttpRetryCount() { + return httpRetryCount; + } + + protected final void setHttpRetryCount(int retryCount) { + this.httpRetryCount = retryCount; + } + + @Override + public final int getHttpRetryIntervalSeconds() { + return httpRetryIntervalSeconds; + } + + protected final void setHttpRetryIntervalSeconds(int retryIntervalSeconds) { + this.httpRetryIntervalSeconds = retryIntervalSeconds; + } + + @Override + public final int getHttpMaxTotalConnections() { + return maxTotalConnections; + } + + protected final void setHttpMaxTotalConnections(int maxTotalConnections) { + this.maxTotalConnections = maxTotalConnections; + } + + @Override + public final int getHttpDefaultMaxPerRoute() { + return defaultMaxPerRoute; + } + + protected final void setHttpDefaultMaxPerRoute(int defaultMaxPerRoute) { + this.defaultMaxPerRoute = defaultMaxPerRoute; + } + + + @Override + public final int getAsyncNumThreads() { + return asyncNumThreads; + } + + protected final void setAsyncNumThreads(int asyncNumThreads) { + this.asyncNumThreads = asyncNumThreads; + } + + @Override + public final long getContributingTo() { + return contributingTo; + } + + protected final void setContributingTo(long contributingTo) { + this.contributingTo = contributingTo; + } + + + @Override + public String getStreamBaseURL() { + return streamBaseURL; + } + + protected final void setStreamBaseURL(String streamBaseURL) { + this.streamBaseURL = streamBaseURL; + } + + + @Override + public String getDispatcherImpl() { + return dispatcherImpl; + } + + protected final void setDispatcherImpl(String dispatcherImpl) { + this.dispatcherImpl = dispatcherImpl; + } + + protected final void setUserAgent(String userAgent) { + this.userAgent= userAgent; + } + + @Override + public String getLoggerFactory() { + return loggerFactory; + } + + + protected final void setLoggerFactory(String loggerImpl) { + this.loggerFactory = loggerImpl; + } + + + public boolean isJSONStoreEnabled() { + return this.jsonStoreEnabled; + } + + protected final void setJSONStoreEnabled(boolean enabled) { + this.jsonStoreEnabled = enabled; + } + + @Override + public boolean isStallWarningsEnabled() { + return stallWarningsEnabled; + } + + protected final void setStallWarningsEnabled(boolean stallWarningsEnabled) { + this.stallWarningsEnabled = stallWarningsEnabled; + } + + static String fixURL(boolean useSSL, String url) { + if (null == url) { + return null; + } + int index = url.indexOf("://"); + if (-1 == index) { + throw new IllegalArgumentException("url should contain '://'"); + } + String hostAndLater = url.substring(index + 3); + if (useSSL) { + return "https://" + hostAndLater; + } else { + return "http://" + hostAndLater; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + WebConfigurationBase that = (WebConfigurationBase) o; + + if (asyncNumThreads != that.asyncNumThreads) return false; + if (contributingTo != that.contributingTo) return false; + if (debug != that.debug) return false; + if (defaultMaxPerRoute != that.defaultMaxPerRoute) return false; + if (gzipEnabled != that.gzipEnabled) return false; + if (httpConnectionTimeout != that.httpConnectionTimeout) return false; + if (httpProxyPort != that.httpProxyPort) return false; + if (httpReadTimeout != that.httpReadTimeout) return false; + if (httpRetryCount != that.httpRetryCount) return false; + if (httpRetryIntervalSeconds != that.httpRetryIntervalSeconds) return false; + if (httpStreamingReadTimeout != that.httpStreamingReadTimeout) return false; + if (includeEntitiesEnabled != that.includeEntitiesEnabled) return false; + if (jsonStoreEnabled != that.jsonStoreEnabled) return false; + if (maxTotalConnections != that.maxTotalConnections) return false; + if (prettyDebug != that.prettyDebug) return false; + if (stallWarningsEnabled != that.stallWarningsEnabled) return false; + if (useSSL != that.useSSL) return false; + if (dispatcherImpl != null ? !dispatcherImpl.equals(that.dispatcherImpl) : that.dispatcherImpl != null) + return false; + if (httpProxyHost != null ? !httpProxyHost.equals(that.httpProxyHost) : that.httpProxyHost != null) + return false; + if (httpProxyPassword != null ? !httpProxyPassword.equals(that.httpProxyPassword) : that.httpProxyPassword != null) + return false; + if (httpProxyUser != null ? !httpProxyUser.equals(that.httpProxyUser) : that.httpProxyUser != null) + return false; + if (loggerFactory != null ? !loggerFactory.equals(that.loggerFactory) : that.loggerFactory != null) + return false; + if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) + return false; + if (streamBaseURL != null ? !streamBaseURL.equals(that.streamBaseURL) : that.streamBaseURL != null) + return false; + return true; + } + + @Override + public int hashCode() { + int result = (debug ? 1 : 0); + result = 31 * result + (useSSL ? 1 : 0); + result = 31 * result + (prettyDebug ? 1 : 0); + result = 31 * result + (gzipEnabled ? 1 : 0); + result = 31 * result + (httpProxyHost != null ? httpProxyHost.hashCode() : 0); + result = 31 * result + (httpProxyUser != null ? httpProxyUser.hashCode() : 0); + result = 31 * result + (httpProxyPassword != null ? httpProxyPassword.hashCode() : 0); + result = 31 * result + httpProxyPort; + result = 31 * result + httpConnectionTimeout; + result = 31 * result + httpReadTimeout; + result = 31 * result + httpStreamingReadTimeout; + result = 31 * result + httpRetryCount; + result = 31 * result + httpRetryIntervalSeconds; + result = 31 * result + maxTotalConnections; + result = 31 * result + defaultMaxPerRoute; + result = 31 * result + (streamBaseURL != null ? streamBaseURL.hashCode() : 0); + result = 31 * result + (dispatcherImpl != null ? dispatcherImpl.hashCode() : 0); + result = 31 * result + (loggerFactory != null ? loggerFactory.hashCode() : 0); + result = 31 * result + asyncNumThreads; + result = 31 * result + (int) (contributingTo ^ (contributingTo >>> 32)); + result = 31 * result + (includeEntitiesEnabled ? 1 : 0); + result = 31 * result + (jsonStoreEnabled ? 1 : 0); + result = 31 * result + (stallWarningsEnabled ? 1 : 0); + result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "ConfigurationBase{" + + "debug=" + debug + + ", useSSL=" + useSSL + + ", prettyDebug=" + prettyDebug + + ", gzipEnabled=" + gzipEnabled + + ", httpProxyHost='" + httpProxyHost + '\'' + + ", httpProxyUser='" + httpProxyUser + '\'' + + ", httpProxyPassword='" + httpProxyPassword + '\'' + + ", httpProxyPort=" + httpProxyPort + + ", httpConnectionTimeout=" + httpConnectionTimeout + + ", httpReadTimeout=" + httpReadTimeout + + ", httpStreamingReadTimeout=" + httpStreamingReadTimeout + + ", httpRetryCount=" + httpRetryCount + + ", httpRetryIntervalSeconds=" + httpRetryIntervalSeconds + + ", maxTotalConnections=" + maxTotalConnections + + ", defaultMaxPerRoute=" + defaultMaxPerRoute + + ", streamBaseURL='" + streamBaseURL + '\'' + + ", dispatcherImpl='" + dispatcherImpl + '\'' + + ", loggerFactory='" + loggerFactory + '\'' + + ", asyncNumThreads=" + asyncNumThreads + + ", contributingTo=" + contributingTo + + ", includeEntitiesEnabled=" + includeEntitiesEnabled + + ", jsonStoreEnabled=" + jsonStoreEnabled + + ", stallWarningsEnabled=" + stallWarningsEnabled + + ", requestHeaders=" + requestHeaders + + '}'; + } + + private static final List instances = new ArrayList(); + + private static void cacheInstance(WebConfigurationBase conf) { + if (!instances.contains(conf)) { + instances.add(conf); + } + } + + protected void cacheInstance() { + cacheInstance(this); + } + + private static WebConfigurationBase getInstance(WebConfigurationBase configurationBase) { + int index; + if ((index = instances.indexOf(configurationBase)) == -1) { + instances.add(configurationBase); + return configurationBase; + } else { + return instances.get(index); + } + } + + // assures equality after deserializedation + protected Object readResolve() throws ObjectStreamException + { + return getInstance(this); + } +} diff --git a/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java b/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java new file mode 100644 index 00000000000..747fd9bb7b7 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebStatusStreamBase.java @@ -0,0 +1,176 @@ +//package druid.examples.twitter; +// +///* +// * Copyright 2007 Yusuke Yamamoto +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +// +// +//import org.mortbay.jetty.servlet.Dispatcher; +// +//import java.io.BufferedReader; +//import java.io.IOException; +//import java.io.InputStream; +//import java.io.InputStreamReader; +// +///** +// * @author Yusuke Yamamoto - yusuke at mac.com +// * @since Twitter4J 2.1.8 +// */ +//abstract class WebStatusStreamBase implements WebStatusStream +//{ +// protected static final twitter4j.internal.logging.Logger logger = twitter4j.internal +// .logging +// .Logger +// .getLogger(WebStatusStreamImpl.class); +// +// private boolean streamAlive = true; +// private BufferedReader br; +// private InputStream is; +// private HttpResponse response; +// protected final Dispatcher dispatcher; +// protected final Configuration CONF; +// protected z_InternalFactory factory; +// +// /*package*/ +// +// StatusStreamBase(Dispatcher dispatcher, InputStream stream, twitter4j.conf.Configuration conf) throws IOException +// { +// this.is = stream; +// this.br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); +// this.dispatcher = dispatcher; +// this.CONF = conf; +// this.factory = new z_T4JInternalJSONImplFactory(conf); +// } +// /*package*/ +// +// StatusStreamBase(Dispatcher dispatcher, HttpResponse response, twitter4j.conf.Configuration conf) throws IOException { +// this(dispatcher, response.asStream(), conf); +// this.response = response; +// } +// +// protected String parseLine(String line) { +// return line; +// } +// +// abstract class StreamEvent implements Runnable { +// String line; +// +// StreamEvent(String line) { +// this.line = line; +// } +// } +// +// protected void handleNextElement(final StreamListener[] listeners, +// final RawStreamListener[] rawStreamListeners) throws TwitterException +// { +// if (!streamAlive) { +// throw new IllegalStateException("Stream already closed."); +// } +// try { +// String line = br.readLine(); +// if (null == line) { +// //invalidate this status stream +// throw new IOException("the end of the stream has been reached"); +// } +// dispatcher.invokeLater(new StreamEvent(line) { +// public void run() { +// try { +// if (rawStreamListeners.length > 0) { +// onMessage(line, rawStreamListeners); +// } +// // SiteStreamsImpl will parse "forUser" attribute +// line = parseLine(line); +// if (line != null && line.length() > 0) { +// // parsing JSON is an expensive process and can be avoided when all listeners are instanceof RawStreamListener +// if (listeners.length > 0) { +// if (CONF.isJSONStoreEnabled()) { +// DataObjectFactoryUtil.clearThreadLocalMap(); +// } +// twitter4j.internal.org.json.JSONObject json = new twitter4j.internal.org.json.JSONObject(line); +// JSONObjectType.Type event = JSONObjectType.determine(json); +// if (logger.isDebugEnabled()) { +// logger.debug("Received:", CONF.isPrettyDebugEnabled() ? json.toString(1) : json.toString()); +// } +// switch (event) { +// case STATUS: +// onStatus(json, listeners); +// break; +// default: +// logger.warn("Received unknown event:", CONF.isPrettyDebugEnabled() ? json.toString(1) : json.toString()); +// } +// } +// } +// } catch (Exception ex) { +// onException(ex, listeners); +// } +// } +// }); +// +// } catch (IOException ioe) { +// try { +// is.close(); +// } catch (IOException ignore) { +// } +// boolean isUnexpectedException = streamAlive; +// streamAlive = false; +// if (isUnexpectedException) { +// throw new RuntimeException("Stream closed.", ioe); +// } +// } +// } +// +// +// +// protected void onStatus(twitter4j.internal.org.json.JSONObject json, StreamListener[] listeners) throws TwitterException { +// logger.warn("Unhandled event: onStatus"); +// } +// +// +// protected void onException(Exception e, StreamListener[] listeners) { +// logger.warn("Unhandled event: ", e.getMessage()); +// } +// +// public void close() throws IOException { +// streamAlive = false; +// is.close(); +// br.close(); +// if (response != null) { +// response.disconnect(); +// } +// } +// +// protected Status asStatus(twitter4j.internal.org.json.JSONObject json) throws TwitterException { +// Status status = factory.createStatus(json); +// if (CONF.isJSONStoreEnabled()) { +// DataObjectFactoryUtil.registerJSONObject(status, json); +// } +// return status; +// } +// +// +// public abstract void next(StatusListener listener) throws TwitterException; +// +// public abstract void next(StreamListener[] listeners, RawStreamListener[] rawStreamListeners) throws TwitterException; +// +// public void onException(Exception e, StreamListener[] listeners, RawStreamListener[] rawStreamListeners) { +// for (StreamListener listener : listeners) { +// listener.onException(e); +// } +// for (RawStreamListener listener : rawStreamListeners) { +// listener.onException(e); +// } +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java b/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java new file mode 100644 index 00000000000..53ef525f33b --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebStreamFactory.java @@ -0,0 +1,111 @@ +//package druid.examples.twitter; +// +///* +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +// +///** +// * An instance of this class is completely thread safe and can be re-used and used concurrently.
+// * Note that TwitterStream is NOT compatible with Google App Engine as GAE is not capable of handling requests longer than 30 seconds. +// * +// */ +//public final class WebStreamFactory implements java.io.Serializable { +// private static final long serialVersionUID = 8146074704915782233L; +// private final Configuration conf; +// private static final WebStream SINGLETON; +// +// static { +// SINGLETON = new TwitterStreamImpl(ConfigurationContext.getInstance(), TwitterFactory.DEFAULT_AUTHORIZATION); +// } +// +// /** +// * Creates a TwitterStreamFactory with the root configuration. +// */ +// public TwitterStreamFactory() { +// this(ConfigurationContext.getInstance()); +// } +// +// /** +// * Creates a TwitterStreamFactory with the given configuration. +// * +// * @param conf the configuration to use +// * @since Twitter4J 2.1.1 +// */ +// public WebStreamFactory(Configuration conf) { +// this.conf = conf; +// } +// +// /** +// * Creates a TwitterStreamFactory with a specified config tree. +// * +// * @param configTreePath the path +// */ +// public WebStreamFactory(String configTreePath) { +// this(ConfigurationContext.getInstance(configTreePath)); +// } +// +// // implementations for BasicSupportFactory +// +// /** +// * Returns a instance associated with the configuration bound to this factory. +// * +// * @return default instance +// */ +// public WebStream getInstance() { +// return getInstance(AuthorizationFactory.getInstance(conf)); +// } +// +// /** +// * Returns a OAuth Authenticated instance.
+// * consumer key and consumer Secret must be provided by twitter4j.properties, or system properties. +// * Unlike {@link TwitterStream#setOAuthAccessToken(twitter4j.auth.AccessToken)}, this factory method potentially returns a cached instance. +// * +// * @param accessToken access token +// * @return an instance +// */ +// public TwitterStream getInstance(AccessToken accessToken) { +// String consumerKey = conf.getOAuthConsumerKey(); +// String consumerSecret = conf.getOAuthConsumerSecret(); +// if (null == consumerKey && null == consumerSecret) { +// throw new IllegalStateException("Consumer key and Consumer secret not supplied."); +// } +// OAuthAuthorization oauth = new OAuthAuthorization(conf); +// oauth.setOAuthAccessToken(accessToken); +// return getInstance(conf, oauth); +// } +// +// /** +// * Returns a instance. +// * +// * @return an instance +// */ +// public TwitterStream getInstance(Authorization auth) { +// return getInstance(conf, auth); +// } +// +// private TwitterStream getInstance(twitter4j.conf.Configuration conf, Authorization auth) { +// return new TwitterStreamImpl(conf, auth); +// } +// +// /** +// * Returns default singleton TwitterStream instance. +// * +// * @return default singleton TwitterStream instance +// * @since Twitter4J 2.2.4 +// */ +// public static TwitterStream getSingleton() { +// return SINGLETON; +// } +//} diff --git a/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java b/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java new file mode 100644 index 00000000000..330ff6b5f95 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/WebStreamImpl.java @@ -0,0 +1,750 @@ +//package druid.examples.twitter; +// +///* +// * Copyright 2007 Yusuke Yamamoto +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//import java.io.IOException; +//import java.io.InputStream; +//import java.util.ArrayList; +//import java.util.HashMap; +//import java.util.List; +//import java.util.Map; +// +// +// +//import java.io.IOException; +//import java.io.InputStream; +//import java.util.ArrayList; +//import java.util.HashMap; +//import java.util.List; +//import java.util.Map; +// +///** +// * A java representation of the Streaming API: Methods
+// * Note that this class is NOT compatible with Google App Engine as GAE is not capable of handling requests longer than 30 seconds. +// * +// * @author Yusuke Yamamoto - yusuke at mac.com +// * @since Twitter4J 2.0.4 +// */ +//class WebStreamImpl extends TwitterBaseImpl implements WebStream +//{ +// private static final long serialVersionUID = 5529611191443189901L; +// private final HttpClientWrapper http; +// private static final Logger logger = Logger.getLogger(TwitterStreamImpl.class); +// +// private List lifeCycleListeners = new ArrayList(0); +// private TwitterStreamConsumer handler = null; +// +// private String stallWarningsGetParam; +// private HttpParameter stallWarningsParam; +// +// /*package*/ +// TwitterStreamImpl(twitter4j.conf.Configuration conf, Authorization auth) { +// super(conf, auth); +// http = new HttpClientWrapper(new StreamingReadTimeoutConfiguration(conf)); +// stallWarningsGetParam = "stall_warnings=" + (conf.isStallWarningsEnabled() ? "true" : "false"); +// stallWarningsParam = new HttpParameter("stall_warnings", conf.isStallWarningsEnabled()); +// } +// +// /* Streaming API */ +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void firehose(final int count) { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException +// { +// return getFirehoseStream(count); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getFirehoseStream(int count) throws TwitterException { +// ensureAuthorizationEnabled(); +// return getCountStream("statuses/firehose.json", count); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void links(final int count) { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getLinksStream(count); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getLinksStream(int count) throws TwitterException { +// ensureAuthorizationEnabled(); +// return getCountStream("statuses/links.json", count); +// } +// +// private StatusStream getCountStream(String relativeUrl, int count) throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() + relativeUrl +// , new HttpParameter[]{new HttpParameter("count", String.valueOf(count)) +// , stallWarningsParam}, auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void retweet() { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getRetweetStream(); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getRetweetStream() throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() + "statuses/retweet.json" +// , new HttpParameter[]{stallWarningsParam}, auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void sample() { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getSampleStream(); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getSampleStream() throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.get(conf.getStreamBaseURL() + "statuses/sample.json?" +// + stallWarningsGetParam, auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// public void user() { +// user(null); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void user(final String[] track) { +// ensureAuthorizationEnabled(); +// ensureUserStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getUserStream(track); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public UserStream getUserStream() throws TwitterException { +// return getUserStream(null); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public UserStream getUserStream(String[] track) throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// List params = new ArrayList(); +// params.add(stallWarningsParam); +// if (conf.isUserStreamRepliesAllEnabled()) { +// params.add(new HttpParameter("replies", "all")); +// } +// if (track != null) { +// params.add(new HttpParameter("track", z_T4JInternalStringUtil.join(track))); +// } +// return new UserStreamImpl(getDispatcher(), http.post(conf.getUserStreamBaseURL() + "user.json" +// , params.toArray(new HttpParameter[params.size()]) +// , auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StreamController site(final boolean withFollowings, final long[] follow) { +// ensureOAuthEnabled(); +// ensureSiteStreamsListenerIsSet(); +// final StreamController cs = new StreamController(http, auth); +// startHandler(new TwitterStreamConsumer(siteStreamsListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// try { +// return new SiteStreamsImpl(getDispatcher(), getSiteStream(withFollowings, follow), conf, cs); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// }); +// return cs; +// } +// +// private Dispatcher getDispatcher() { +// if (null == TwitterStreamImpl.dispatcher) { +// synchronized (TwitterStreamImpl.class) { +// if (null == TwitterStreamImpl.dispatcher) { +// // dispatcher is held statically, but it'll be instantiated with +// // the configuration instance associated with this TwitterStream +// // instance which invokes getDispatcher() on the first time. +// TwitterStreamImpl.dispatcher = new DispatcherFactory(conf).getInstance(); +// } +// } +// } +// return TwitterStreamImpl.dispatcher; +// } +// +// private static transient Dispatcher dispatcher; +// +// InputStream getSiteStream(boolean withFollowings, long[] follow) throws TwitterException { +// ensureOAuthEnabled(); +// return http.post(conf.getSiteStreamBaseURL() + "site.json", +// new HttpParameter[]{ +// new HttpParameter("with", withFollowings ? "followings" : "user") +// , new HttpParameter("follow", z_T4JInternalStringUtil.join(follow)) +// , stallWarningsParam}, auth).asStream(); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void filter(final FilterQuery query) { +// ensureAuthorizationEnabled(); +// ensureStatusStreamListenerIsSet(); +// startHandler(new TwitterStreamConsumer(statusListeners, rawStreamListeners) { +// @Override +// public StatusStream getStream() throws TwitterException { +// return getFilterStream(query); +// } +// }); +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public StatusStream getFilterStream(FilterQuery query) throws TwitterException { +// ensureAuthorizationEnabled(); +// try { +// return new StatusStreamImpl(getDispatcher(), http.post(conf.getStreamBaseURL() +// + "statuses/filter.json" +// , query.asHttpParameterArray(stallWarningsParam), auth), conf); +// } catch (IOException e) { +// throw new TwitterException(e); +// } +// } +// +// +// /** +// * check if any listener is set. Throws IllegalStateException if no listener is set. +// * +// * @throws IllegalStateException when no listener is set. +// */ +// +// private void ensureStatusStreamListenerIsSet() { +// if (statusListeners.size() == 0 && rawStreamListeners.size() == 0) { +// throw new IllegalStateException("StatusListener is not set."); +// } +// } +// +// private void ensureUserStreamListenerIsSet() { +// if (userStreamListeners.size() == 0 && rawStreamListeners.size() == 0) { +// throw new IllegalStateException("UserStreamListener is not set."); +// } +// } +// +// private void ensureSiteStreamsListenerIsSet() { +// if (siteStreamsListeners.size() == 0 && rawStreamListeners.size() == 0) { +// throw new IllegalStateException("SiteStreamsListener is not set."); +// } +// } +// +// private static int numberOfHandlers = 0; +// +// private synchronized void startHandler(TwitterStreamConsumer handler) { +// cleanUp(); +// this.handler = handler; +// this.handler.start(); +// numberOfHandlers++; +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public synchronized void cleanUp() { +// if (handler != null) { +// handler.close(); +// numberOfHandlers--; +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public synchronized void shutdown() { +// super.shutdown(); +// cleanUp(); +// synchronized (TwitterStreamImpl.class) { +// if (0 == numberOfHandlers) { +// if (dispatcher != null) { +// dispatcher.shutdown(); +// dispatcher = null; +// } +// } +// } +// } +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addConnectionLifeCycleListener(ConnectionLifeCycleListener listener) { +// this.lifeCycleListeners.add(listener); +// } +// +// private List userStreamListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addListener(UserStreamListener listener) { +// statusListeners.add(listener); +// userStreamListeners.add(listener); +// } +// +// private List statusListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addListener(StatusListener listener) { +// statusListeners.add(listener); +// } +// +// private List siteStreamsListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// @Override +// public void addListener(SiteStreamsListener listener) { +// siteStreamsListeners.add(listener); +// } +// +// private List rawStreamListeners = new ArrayList(0); +// +// /** +// * {@inheritDoc} +// */ +// public void addListener(RawStreamListener listener) { +// rawStreamListeners.add(listener); +// } +// +// /* +// https://dev.twitter.com/docs/streaming-api/concepts#connecting +// When a network error (TCP/IP level) is encountered, back off linearly. Perhaps start at 250 milliseconds, double, and cap at 16 seconds +// When a HTTP error (> 200) is returned, back off exponentially. +// Perhaps start with a 10 second wait, double on each subsequent failure, and finally cap the wait at 240 seconds. Consider sending an alert to a human operator after multiple HTTP errors, as there is probably a client configuration issue that is unlikely to be resolved without human intervention. There's not much point in polling any faster in the face of HTTP error codes and your client is may run afoul of a rate limit. +// */ +// private static final int TCP_ERROR_INITIAL_WAIT = 250; +// private static final int TCP_ERROR_WAIT_CAP = 16 * 1000; +// +// private static final int HTTP_ERROR_INITIAL_WAIT = 10 * 1000; +// private static final int HTTP_ERROR_WAIT_CAP = 240 * 1000; +// +// private static final int NO_WAIT = 0; +// +// static int count = 0; +// +// abstract class TwitterStreamConsumer extends Thread { +// private StatusStreamBase stream = null; +// private final String NAME = "Twitter Stream consumer-" + (++count); +// private volatile boolean closed = false; +// private final StreamListener[] streamListeners; +// private final RawStreamListener[] rawStreamListeners; +// +// TwitterStreamConsumer(List streamListeners, List rawStreamListeners) { +// super(); +// setName(NAME + "[initializing]"); +// this.streamListeners = streamListeners.toArray(new StreamListener[streamListeners.size()]); +// this.rawStreamListeners = rawStreamListeners.toArray(new RawStreamListener[rawStreamListeners.size()]); +// } +// +// @Override +// public void run() { +// int timeToSleep = NO_WAIT; +// boolean connected = false; +// while (!closed) { +// try { +// if (!closed && null == stream) { +// // try establishing connection +// logger.info("Establishing connection."); +// setStatus("[Establishing connection]"); +// stream = (StatusStreamBase) getStream(); +// connected = true; +// logger.info("Connection established."); +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onConnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// // connection established successfully +// timeToSleep = NO_WAIT; +// logger.info("Receiving status stream."); +// setStatus("[Receiving stream]"); +// while (!closed) { +// try { +// stream.next(this.streamListeners, this.rawStreamListeners); +// } catch (IllegalStateException ise) { +// logger.warn(ise.getMessage()); +// break; +// } catch (TwitterException e) { +// logger.info(e.getMessage()); +// stream.onException(e, this.streamListeners, this.rawStreamListeners); +// throw e; +// } catch (Exception e) { +// logger.info(e.getMessage()); +// stream.onException(e, this.streamListeners, this.rawStreamListeners); +// closed = true; +// break; +// } +// } +// } +// } catch (TwitterException te) { +// logger.info(te.getMessage()); +// if (!closed) { +// if (NO_WAIT == timeToSleep) { +// if (te.getStatusCode() == FORBIDDEN) { +// logger.warn("This account is not in required role. ", te.getMessage()); +// closed = true; +// for (StreamListener statusListener : streamListeners) { +// statusListener.onException(te); +// } +// break; +// } +// if (te.getStatusCode() == NOT_ACCEPTABLE) { +// logger.warn("Parameter not accepted with the role. ", te.getMessage()); +// closed = true; +// for (StreamListener statusListener : streamListeners) { +// statusListener.onException(te); +// } +// break; +// } +// connected = false; +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onDisconnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// if (te.getStatusCode() > 200) { +// timeToSleep = HTTP_ERROR_INITIAL_WAIT; +// } else if (0 == timeToSleep) { +// timeToSleep = TCP_ERROR_INITIAL_WAIT; +// } +// } +// if (te.getStatusCode() > 200 && timeToSleep < HTTP_ERROR_INITIAL_WAIT) { +// timeToSleep = HTTP_ERROR_INITIAL_WAIT; +// } +// if (connected) { +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onDisconnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// } +// for (StreamListener statusListener : streamListeners) { +// statusListener.onException(te); +// } +// // there was a problem establishing the connection, or the connection closed by peer +// if (!closed) { +// // wait for a moment not to overload Twitter API +// logger.info("Waiting for " + (timeToSleep) + " milliseconds"); +// setStatus("[Waiting for " + (timeToSleep) + " milliseconds]"); +// try { +// Thread.sleep(timeToSleep); +// } catch (InterruptedException ignore) { +// } +// timeToSleep = Math.min(timeToSleep * 2, (te.getStatusCode() > 200) ? HTTP_ERROR_WAIT_CAP : TCP_ERROR_WAIT_CAP); +// } +// stream = null; +// logger.debug(te.getMessage()); +// connected = false; +// } +// } +// } +// if (this.stream != null && connected) { +// try { +// this.stream.close(); +// } catch (IOException ignore) { +// } catch (Exception e) { +// e.printStackTrace(); +// logger.warn(e.getMessage()); +// } finally { +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onDisconnect(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// } +// } +// for (ConnectionLifeCycleListener listener : lifeCycleListeners) { +// try { +// listener.onCleanUp(); +// } catch (Exception e) { +// logger.warn(e.getMessage()); +// } +// } +// } +// +// public synchronized void close() { +// setStatus("[Disposing thread]"); +// try { +// if (stream != null) { +// try { +// stream.close(); +// } catch (IOException ignore) { +// } catch (Exception e) { +// e.printStackTrace(); +// logger.warn(e.getMessage()); +// } +// } +// } finally { +// closed = true; +// } +// } +// +// private void setStatus(String message) { +// String actualMessage = NAME + message; +// setName(actualMessage); +// logger.debug(actualMessage); +// } +// +// abstract StatusStream getStream() throws TwitterException; +// +// } +// +// @Override +// public boolean equals(Object o) { +// if (this == o) return true; +// if (o == null || getClass() != o.getClass()) return false; +// if (!super.equals(o)) return false; +// +// TwitterStreamImpl that = (TwitterStreamImpl) o; +// +// if (handler != null ? !handler.equals(that.handler) : that.handler != null) return false; +// if (http != null ? !http.equals(that.http) : that.http != null) return false; +// if (lifeCycleListeners != null ? !lifeCycleListeners.equals(that.lifeCycleListeners) : that.lifeCycleListeners != null) +// return false; +// if (rawStreamListeners != null ? !rawStreamListeners.equals(that.rawStreamListeners) : that.rawStreamListeners != null) +// return false; +// if (siteStreamsListeners != null ? !siteStreamsListeners.equals(that.siteStreamsListeners) : that.siteStreamsListeners != null) +// return false; +// if (stallWarningsGetParam != null ? !stallWarningsGetParam.equals(that.stallWarningsGetParam) : that.stallWarningsGetParam != null) +// return false; +// if (stallWarningsParam != null ? !stallWarningsParam.equals(that.stallWarningsParam) : that.stallWarningsParam != null) +// return false; +// if (statusListeners != null ? !statusListeners.equals(that.statusListeners) : that.statusListeners != null) +// return false; +// if (userStreamListeners != null ? !userStreamListeners.equals(that.userStreamListeners) : that.userStreamListeners != null) +// return false; +// +// return true; +// } +// +// @Override +// public int hashCode() { +// int result = super.hashCode(); +// result = 31 * result + (http != null ? http.hashCode() : 0); +// result = 31 * result + (lifeCycleListeners != null ? lifeCycleListeners.hashCode() : 0); +// result = 31 * result + (handler != null ? handler.hashCode() : 0); +// result = 31 * result + (stallWarningsGetParam != null ? stallWarningsGetParam.hashCode() : 0); +// result = 31 * result + (stallWarningsParam != null ? stallWarningsParam.hashCode() : 0); +// result = 31 * result + (userStreamListeners != null ? userStreamListeners.hashCode() : 0); +// result = 31 * result + (statusListeners != null ? statusListeners.hashCode() : 0); +// result = 31 * result + (siteStreamsListeners != null ? siteStreamsListeners.hashCode() : 0); +// result = 31 * result + (rawStreamListeners != null ? rawStreamListeners.hashCode() : 0); +// return result; +// } +// +// @Override +// public String toString() { +// return "TwitterStreamImpl{" + +// "http=" + http + +// ", lifeCycleListeners=" + lifeCycleListeners + +// ", handler=" + handler + +// ", stallWarningsGetParam='" + stallWarningsGetParam + '\'' + +// ", stallWarningsParam=" + stallWarningsParam + +// ", userStreamListeners=" + userStreamListeners + +// ", statusListeners=" + statusListeners + +// ", siteStreamsListeners=" + siteStreamsListeners + +// ", rawStreamListeners=" + rawStreamListeners + +// '}'; +// } +//} +// +//class StreamingReadTimeoutConfiguration implements HttpClientWrapperConfiguration +//{ +// twitter4j.conf.Configuration nestedConf; +// +// StreamingReadTimeoutConfiguration(twitter4j.conf.Configuration httpConf) { +// this.nestedConf = httpConf; +// } +// +// @Override +// public String getHttpProxyHost() { +// return nestedConf.getHttpProxyHost(); +// } +// +// @Override +// public int getHttpProxyPort() { +// return nestedConf.getHttpProxyPort(); +// } +// +// @Override +// public String getHttpProxyUser() { +// return nestedConf.getHttpProxyUser(); +// } +// +// @Override +// public String getHttpProxyPassword() { +// return nestedConf.getHttpProxyPassword(); +// } +// +// @Override +// public int getHttpConnectionTimeout() { +// return nestedConf.getHttpConnectionTimeout(); +// } +// +// @Override +// public int getHttpReadTimeout() { +// // this is the trick that overrides connection timeout +// return nestedConf.getHttpStreamingReadTimeout(); +// } +// +// @Override +// public int getHttpRetryCount() { +// return nestedConf.getHttpRetryCount(); +// } +// +// @Override +// public int getHttpRetryIntervalSeconds() { +// return nestedConf.getHttpRetryIntervalSeconds(); +// } +// +// @Override +// public int getHttpMaxTotalConnections() { +// return nestedConf.getHttpMaxTotalConnections(); +// } +// +// @Override +// public int getHttpDefaultMaxPerRoute() { +// return nestedConf.getHttpDefaultMaxPerRoute(); +// } +// +// @Override +// public Map getRequestHeaders() { +// // turning off keepalive connection explicitly because Streaming API doesn't need keepalive connection. +// // and this will reduce the shutdown latency of streaming api connection +// // see also - http://jira.twitter4j.org/browse/TFJ-556 +// Map headers = new HashMap(nestedConf.getRequestHeaders()); +// headers.put("Connection", "close"); +// return headers; +// } +// +// @Override +// public boolean isPrettyDebugEnabled() { +// return nestedConf.isPrettyDebugEnabled(); +// } +// +// @Override +// public boolean isGZIPEnabled() { +// return nestedConf.isGZIPEnabled(); +// } +//} +// diff --git a/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java b/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java new file mode 100644 index 00000000000..d214d755260 --- /dev/null +++ b/examples/src/main/java/druid/examples/twitter/z_InternalStringUtil.java @@ -0,0 +1,111 @@ +package druid.examples.twitter; + +/* + * Copyright 2007 Yusuke Yamamoto + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.List; + + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Yusuke Yamamoto - yusuke at mac.com + * @since Twitter4J 2.1.4 + */ +public class z_InternalStringUtil { + private z_InternalStringUtil() { + throw new AssertionError(); + } + + public static String maskString(String str) { + StringBuilder buf = new StringBuilder(str.length()); + for (int i = 0; i < str.length(); i++) { + buf.append("*"); + } + return buf.toString(); + } + + // for JDK1.4 compatibility + + public static String[] split(String str, String separator) { + String[] returnValue; + int index = str.indexOf(separator); + if (index == -1) { + returnValue = new String[]{str}; + } else { + List strList = new ArrayList(); + int oldIndex = 0; + while (index != -1) { + String subStr = str.substring(oldIndex, index); + strList.add(subStr); + oldIndex = index + separator.length(); + index = str.indexOf(separator, oldIndex); + } + if (oldIndex != str.length()) { + strList.add(str.substring(oldIndex)); + } + returnValue = strList.toArray(new String[strList.size()]); + } + + return returnValue; + } + + public static String join(int[] follows) { + StringBuilder buf = new StringBuilder(11 * follows.length); + for (int follow : follows) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(follow); + } + return buf.toString(); + } + + public static String join(long[] follows) { + StringBuilder buf = new StringBuilder(11 * follows.length); + for (long follow : follows) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(follow); + } + return buf.toString(); + } + + public static String join(String[] track) { + StringBuilder buf = new StringBuilder(11 * track.length); + for (String str : track) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(str); + } + return buf.toString(); + } + + public static String join(List strs) { + StringBuilder buf = new StringBuilder(11 * strs.size()); + for (String str : strs) { + if (0 != buf.length()) { + buf.append(","); + } + buf.append(str); + } + return buf.toString(); + } +} diff --git a/examples/twitter4j.properties b/examples/twitter4j.properties new file mode 100644 index 00000000000..002d832143b --- /dev/null +++ b/examples/twitter4j.properties @@ -0,0 +1,5 @@ +debug=true +oauth.consumerKey=HOQS4c7WveN7NCxzl2aNgg +oauth.consumerSecret=47mKCGRfzml268RWDmRYGkFF6pusVrzoYrhRZPfELY +oauth.accessToken=19520065-aBZl60q698VYksk78oayY1eRFucofWo3FLI1tvekA +oauth.accessTokenSecret=Ul0bSghDzlYGKzN6lH7W6qZxW4bFCANfVXMs3KPfM diff --git a/examples/twitter_realtime.spec b/examples/twitter_realtime.spec new file mode 100644 index 00000000000..f17aab07f17 --- /dev/null +++ b/examples/twitter_realtime.spec @@ -0,0 +1,44 @@ +[{ + "schema": { + "dataSource": "twitterstream", + "aggregators": [ + {"type": "count", "name": "tweets"}, + {"type": "doubleSum", "fieldName": "follower_count", "name": "total_follower_count"}, + {"type": "doubleSum", "fieldName": "retweet_count", "name": "total_retweet_count" }, + {"type": "doubleSum", "fieldName": "friends_count", "name": "total_friends_count" }, + {"type": "doubleSum", "fieldName": "statuses_count", "name": "total_statuses_count"}, + + {"type": "min", "fieldName": "follower_count", "name": "min_follower_count"}, + {"type": "max", "fieldName": "follower_count", "name": "max_follower_count"}, + + {"type": "min", "fieldName": "friends_count", "name": "min_friends_count"}, + {"type": "max", "fieldName": "friends_count", "name": "max_friends_count"}, + + {"type": "min", "fieldName": "statuses_count", "name": "min_statuses_count"}, + {"type": "max", "fieldName": "statuses_count", "name": "max_statuses_count"}, + + {"type": "min", "fieldName": "retweet_count", "name": "min_retweet_count"}, + {"type": "max", "fieldName": "retweet_count", "name": "max_retweet_count"} + ], + "indexGranularity": "minute", + "shardSpec": {"type": "none"} + }, + + "config": { + "maxRowsInMemory": 50000, + "intermediatePersistPeriod": "PT2m" + }, + + "firehose": { + "type": "twitzer", + "maxEventCount": 500000, + "maxRunMinutes": 120 + }, + + "plumber": { + "type": "realtime", + "windowPeriod": "PT3m", + "segmentGranularity": "hour", + "basePersistDirectory": "/tmp/example/twitter_realtime/basePersist" + } +}] diff --git a/indexing-common/.DS_Store b/indexing-common/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..feca8613504bcd7430524202412f605a970dd548 GIT binary patch literal 6148 zcmeH~JqiLr422VS3&Cbf%V|7-HyA`u-~~i21wpZ&qx41vI)z6#m+=T zbn`f`MFtU>!%bynVPcAXCwIBY>3lz3j@RpDteEA>YT$!Ro{xoGkN^pg011!)3H%HJ zJGWu;MJOW)kN^pc1nmD%;HEXTh5D}p!AAgSkF*=sK1)E0C7?C6g(3sfXoW_r`WRw) zZ-`o{NKSp&Hs}YZb^Uy{)~Y3 z+x>QfkIJ+4?ei>u$gHg!9O~r=FP8vp>?mHs-SE8F0$NjBC^9f!1RMhc34E2n6Mwc5 Aa{vGU literal 0 HcmV?d00001 diff --git a/indexing-common/src/.DS_Store b/indexing-common/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..9d6a56620545250bb9a34aa5ef87a586fdec28fa GIT binary patch literal 6148 zcmeH~J#ND=428+mgMlm_Giu2JdIP~oPS6W9ZI%FTJOtf!^gjNeK$Xr#5j+CqCsHB| zzriv^MCZ@*v&bYOBitx^3!N$QK|bUp7rCBq@5|e|E_r5?v@`HYy8YZHr~nn90#tws zP=SA>fZf}$aT{c$0#twsd@EqzhXOa&WDEMI1HnfCa6s7&Yo8^+Vg;}!TM!kPMk^St z>SKu2y&Wuht|nVB+C_8t(0sDm6a&*}7cEF&S{)2jfC_X9JVid){eOi2Hve}mOsN1B zcrXQYIJ_KYyj0$;cduvl*R0yQ!9hQc@Z%={iCx7zxEuD1Ex?*=K~!M;5pWq8sKB2p F@C6gX5~u(G literal 0 HcmV?d00001 diff --git a/indexing-common/src/main/.DS_Store b/indexing-common/src/main/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d38176301af6a9db8e9ffa5dc7916732489a254b GIT binary patch literal 6148 zcmeH~JqiLr422WjLa^D=avBfd4F=H@cme+^5iCUOIl3=D2(H#5@&d^>$xK-G6+0Ud z(dF&95a~r^0yoOs!oU=HCnwp-U3SOo;k>_G&Ut2(v@`Ht2K%{9Pys4H1*iZOpaMUn zfZf}$$u!7F1*iZOcq(AuhXOa&WDEMI1HnfCutV7mYo8^+Vg;}!TM!kPMk^St>SKu2 zy&Wuht|nVB+C_8t(0sDm6a&*}7cEF&S{)2jfC>y07)RdQ{lA8Pn*RqaOsN1B_%j8x zTWwZLyj0$;Z?9+dZB}jF;GiEzc=-uHVps7D?uPwh3$P|z5EU4I1Y8CND)3bW9?Vn{ Ar2qf` literal 0 HcmV?d00001 diff --git a/indexing-common/src/main/java/.DS_Store b/indexing-common/src/main/java/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2b615a767f6da08a02d4f0c4241f455170fff868 GIT binary patch literal 6148 zcmeH~JqiLr422W55Nx)zoW=uqgF*BJUO;q9Wec&Nqx41vI)z6#m+=T zboc1DBAtjV;ij^&Ffqk`E*BZ(*55DJ^Dtb;idn9#2Hwf!`B=yW36KB@kN^pgz|Rn{ za~rl?g)))=36Q`@!2S;fZdy}YsQ)?;d<1|lkaok`X9;Mr1hl5MP-I{lt-v z{c*p;N9Ec2_IZ{+W!BaW4)t<`mrDRPb`-DSZg^g70j;Sm6d4#V0*--!1inh(38@+p AY5)KL literal 0 HcmV?d00001 diff --git a/install/.DS_Store b/install/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..33eea665350de4626480e3f764234d3db87f9fef GIT binary patch literal 6148 zcmeHKyGjH>5UkbK950p*sT?z=0;xb>fo<u)Yv~NVO8`4siub?d6>T%G3hW$>&T*p?^CFJ5&P${lz9-`!Pzc zdXER<*ZLdQANehF!|JN4*Q=^tk*{K1(of&a7y(CkLQVc0&n)}wmmYhB9cNs3$CT?H z>kti<++xF3G(GNXkIQOUC=d#S0--=C@FN9yW~)u6j!}jJp+G3`setYeiLO{=Y#nXu zU|~-HqF$rT80+;|b5h14W9!Hsns_SFQzbt!#M9ZHGOx(kI(j-JKRzVS{PB1Zo1Nn) zqeBYEC_{lzpsT>ht1~_SFZjz0KJvRMQ40k^fq$lejF&gd1wSgEt=~RR&)P)4rK>5g p)quvha|vKY_mLCp^!}tVc}2$7QAg2sODD!fzyygR6!--N-T@|8LGJ(n literal 0 HcmV?d00001 diff --git a/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java b/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java new file mode 100644 index 00000000000..f57f5604f6b --- /dev/null +++ b/realtime/src/main/java/com/metamx/druid/realtime/firehose/WebFirehoseFactory.java @@ -0,0 +1,85 @@ +package com.metamx.druid.realtime.firehose; + +import com.metamx.druid.input.InputRow; +import com.metamx.druid.input.MapBasedInputRow; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +/** +* Created with IntelliJ IDEA. +* User: dhruvparthasarathy +* Date: 6/20/13 +* Time: 2:03 PM +* To change this template use File | Settings | File Templates. +*/ +public class WebFirehoseFactory implements FirehoseFactory +{ + + @Override + public Firehose connect() throws IOException + { + final UpdateListener updateListener; + final int QUEUE_SIZE=2000; + final BlockingQueue> queue= new ArrayBlockingQueue>(QUEUE_SIZE); + final LinkedList dimensions = new LinkedList(); + + dimensions.add("BITLY_HASH"); + dimensions.add("LATUTUDE-LONGITUDE"); + dimensions.add("COUNTRTY_CODE"); + dimensions.add("USER_AGENT"); + dimensions.add("CITY"); + dimensions.add("ENCODING_USER_LOGIN"); + dimensions.add("SHORT_URL_CNAME"); + dimensions.add("TIMESTAMP OF TIME HASH WAS CREATED"); + dimensions.add("ENCODING_USER_BITLY_HASH"); + dimensions.add("LONG_URL"); + dimensions.add("TIMEZONE"); + dimensions.add("TIMESTAMP OF TIME HASH WAS CREATED"); + dimensions.add("REFERRING_URL"); + dimensions.add("GEO_REGION"); + dimensions.add("KNOWN_USER"); + + UpdateStream updateStream = new UpdateStream(queue); + updateStream.start(); + + return new Firehose() { + private final Map theMap = new HashMap(); + + private final Runnable doNothingRunnable = new Runnable() { + public void run(){ + + } + }; + + @Override + public boolean hasMore(){ + if (queue.size()>0){ + return true; + } + return false; + } + + @Override + public InputRow nextRow() + { + if (Thread.currentThread().isInterrupted()) { + throw new RuntimeException("Interrupted, time to stop"); + } + Map update; + try{ + update=queue.take(); + } + catch (InterruptedException e) { + throw new RuntimeException("InterrutpedException", e); + } + + return new MapBasedInputRow(((Long) update.get("TIMESTAMP")).intValue(),dimensions,update); + } + }; + } +} diff --git a/server/.DS_Store b/server/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..8c136e4f0413485ac8547313f579d9bbd85218f3 GIT binary patch literal 6148 zcmeHK%Sr=55Ue&q1G((c;~c#h@DG*{1+V%8nyg4*WhExT<9?tY;NNI<_o(c;2Tvl> z4Kp>5>fV`xotXw8^IdfY6aXeHilU7H)8WCXgG?S1#bQ)A#S`wZ!G0jnUmVi47pS=6 z9oEFJ_3JKX9D6{0SvRX?-7Jj@To6y*%xp_Xctk_q49^4XmmYgWg*I4qW6D*Jb%-r) z$-AeT+aC9|$F)U76$*p`p+G1Q3j9z3-q~uCsbkckKqwFjd@7*lLt;@ZGq#R)bg;4% zfT+)CGqyFCP@j~s%-A||h9>SxbXUnKhPXS&Q|6T!TSs??=D$M8m25C${{!mFx+#}+vm{@|x?nkZ%SI14z#|&+W_TW8zx3E6=3S%g=9H@* zYad(Ol6TK}Y7d{yOug zOotSPQHKJdKv#j0HK%(1pYxX)eB^gSq7@2+0{=__885CEb1ury)^E$xvo^8Zu&61o pRfk5ua|vKW_mSf~XyZwJ@(Rb+P_t-%4kyM%APW*zDDVpkyaPdALL2}9 literal 0 HcmV?d00001