From 69f1fb5749e23aa9961afbc00c392faded31bc69 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Wed, 11 Jan 2012 01:47:20 -0500 Subject: [PATCH] Now using a cron job to enqueue 'store tweet' tasks every 5min since the GAE task queue actually removes tasks once they have completed successfully. This meant that the blobstores were only populated *once*. --- .../config/SpringServletConfig.java | 18 ++-- .../controller/EnqueueStoresController.java | 92 +++++++++++++++++++ .../src/main/platform/cron.xml | 28 ++++++ .../src/main/webapp/WEB-INF/queue.xml | 4 +- .../src/main/webapp/WEB-INF/web.xml | 17 +++- .../EnqueueStoresControllerTest.java | 65 +++++++++++++ 6 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresController.java create mode 100644 demos/tweetstore/gae-tweetstore-spring/src/main/platform/cron.xml create mode 100644 demos/tweetstore/gae-tweetstore-spring/src/test/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresControllerTest.java diff --git a/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/config/SpringServletConfig.java b/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/config/SpringServletConfig.java index 4d9e5d5606..53189d47b3 100644 --- a/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/config/SpringServletConfig.java +++ b/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/config/SpringServletConfig.java @@ -18,7 +18,6 @@ */ package org.jclouds.demo.tweetstore.config; -import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Predicates.in; @@ -47,6 +46,7 @@ import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.demo.tweetstore.config.util.CredentialsCollector; import org.jclouds.demo.tweetstore.controller.AddTweetsController; +import org.jclouds.demo.tweetstore.controller.EnqueueStoresController; import org.jclouds.demo.tweetstore.controller.StoreTweetsController; import org.jclouds.demo.tweetstore.functions.ServiceToStoredTweetStatuses; import org.jclouds.gae.config.GoogleAppEngineConfigurationModule; @@ -65,7 +65,6 @@ import twitter4j.conf.ConfigurationBuilder; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -121,10 +120,7 @@ public class SpringServletConfig extends LoggingConfig implements ServletConfigA // get a queue for submitting store tweet requests queue = QueueFactory.getQueue("twitter"); - // submit a job to store tweets for each configured blobstore - for (String name : providerTypeToBlobStoreMap.keySet()) { - queue.add(withUrl("/store/do").header("context", name).method(Method.GET)); - } + logger.trace("Members initialized. Twitter: '%s', container: '%s', provider types: '%s'", twitterClient, container, providerTypeToBlobStoreMap.keySet()); } @@ -169,6 +165,11 @@ public class SpringServletConfig extends LoggingConfig implements ServletConfigA return controller; } + @Bean + public EnqueueStoresController enqueueStoresController() { + return new EnqueueStoresController(providerTypeToBlobStoreMap, queue); + } + private void injectServletConfig(Servlet servlet) { logger.trace("About to inject servlet config '%s'", servletConfig); try { @@ -190,10 +191,11 @@ public class SpringServletConfig extends LoggingConfig implements ServletConfigA Map urlMap = Maps.newHashMapWithExpectedSize(2); urlMap.put("/store/*", storeTweetsController()); urlMap.put("/tweets/*", addTweetsController()); + urlMap.put("/stores/*", enqueueStoresController()); mapping.setUrlMap(urlMap); /* - * "/store" and "/tweets" are part of the servlet mapping and thus stripped by the mapping if - * using default settings. + * "/store", "/tweets" and "/stores" are part of the servlet mapping and thus + * stripped by the mapping if using default settings. */ mapping.setAlwaysUseFullPath(true); return mapping; diff --git a/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresController.java b/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresController.java new file mode 100644 index 0000000000..94568e4dec --- /dev/null +++ b/demos/tweetstore/gae-tweetstore-spring/src/main/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresController.java @@ -0,0 +1,92 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.demo.tweetstore.controller; + +import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; +import static com.google.appengine.repackaged.com.google.common.base.Strings.nullToEmpty; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; + +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.logging.Logger; + +import com.google.appengine.api.taskqueue.Queue; +import com.google.appengine.api.taskqueue.TaskOptions.Method; +import com.google.common.annotations.VisibleForTesting; + +/** + * Adds tasks to retrieve and store tweets in all registered contexts to an async + * task queue. + * + * @author Andrew Phillips + * @see StoreTweetsController + */ +@Singleton +public class EnqueueStoresController extends HttpServlet { + /** The serialVersionUID */ + private static final long serialVersionUID = 7215420527854203714L; + + private final Set contextNames; + private final Queue taskQueue; + + @Resource + protected Logger logger = Logger.NULL; + + @Inject + public EnqueueStoresController(Map contexts, + Queue taskQueue) { + contextNames = contexts.keySet(); + this.taskQueue = taskQueue; + } + + @VisibleForTesting + void enqueueStoreTweetTasks() { + for (String contextName : contextNames) { + logger.debug("enqueuing task to store tweets in blobstore '%s'", contextName); + taskQueue.add(withUrl("/store/do").header("context", contextName).method(Method.GET)); + } + } + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + if (!nullToEmpty(request.getHeader("X-AppEngine-Cron")).equals("true")) { + response.sendError(401); + } + + try { + enqueueStoreTweetTasks(); + response.setContentType(MediaType.TEXT_PLAIN); + response.getWriter().println("Done!"); + } catch (Exception e) { + logger.error(e, "Error storing tweets"); + throw new ServletException(e); + } + } +} \ No newline at end of file diff --git a/demos/tweetstore/gae-tweetstore-spring/src/main/platform/cron.xml b/demos/tweetstore/gae-tweetstore-spring/src/main/platform/cron.xml new file mode 100644 index 0000000000..193a5402f6 --- /dev/null +++ b/demos/tweetstore/gae-tweetstore-spring/src/main/platform/cron.xml @@ -0,0 +1,28 @@ + + + + + /stores/do + Enqueue 'store tweet' tasks for all contexts + every 10 minutes + + \ No newline at end of file diff --git a/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/queue.xml b/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/queue.xml index e8c42d708a..1bc53f398d 100644 --- a/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/queue.xml +++ b/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/queue.xml @@ -22,6 +22,8 @@ twitter - 1/m + + 2/m + 1 diff --git a/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/web.xml b/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/web.xml index c6dead1056..971ada29cd 100644 --- a/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/web.xml +++ b/demos/tweetstore/gae-tweetstore-spring/src/main/webapp/WEB-INF/web.xml @@ -41,9 +41,22 @@ dispatcher /tweets/* - + + dispatcher + /stores/* + + + + + + /stores/* + + + admin + + + index.jsp - \ No newline at end of file diff --git a/demos/tweetstore/gae-tweetstore-spring/src/test/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresControllerTest.java b/demos/tweetstore/gae-tweetstore-spring/src/test/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresControllerTest.java new file mode 100644 index 0000000000..9a6ca29b32 --- /dev/null +++ b/demos/tweetstore/gae-tweetstore-spring/src/test/java/org/jclouds/demo/tweetstore/controller/EnqueueStoresControllerTest.java @@ -0,0 +1,65 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.demo.tweetstore.controller; + +import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; +import static org.easymock.EasyMock.expect; +import static org.easymock.classextension.EasyMock.createMock; +import static org.easymock.classextension.EasyMock.replay; +import static org.easymock.classextension.EasyMock.verify; + +import java.util.Map; + +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.BlobStoreContextFactory; +import org.testng.annotations.Test; + +import com.google.appengine.api.taskqueue.Queue; +import com.google.appengine.api.taskqueue.TaskOptions.Method; +import com.google.common.collect.ImmutableMap; + +/** + * Tests behavior of {@code StoreTweetsController} + * + * @author Adrian Cole + */ +@Test(groups = "unit") +public class EnqueueStoresControllerTest { + + Map createBlobStores() { + Map contexts = ImmutableMap.of( + "test1", new BlobStoreContextFactory().createContext("transient", "dummy", "dummy"), + "test2", new BlobStoreContextFactory().createContext("transient", "dummy", "dummy")); + return contexts; + } + + public void testEnqueueStores() { + Map stores = createBlobStores(); + Queue taskQueue = createMock(Queue.class); + EnqueueStoresController function = new EnqueueStoresController(stores, taskQueue); + + expect(taskQueue.add(withUrl("/store/do").header("context", "test1").method(Method.GET))).andReturn(null); + expect(taskQueue.add(withUrl("/store/do").header("context", "test2").method(Method.GET))).andReturn(null); + replay(taskQueue); + + function.enqueueStoreTweetTasks(); + + verify(taskQueue); + } +}