JCLOUDS-137: Retry on HTTP 500 AtmosError 1040

apis/atmos/src/main/java/org/jclouds/atmos/config/AtmosRestClientModule.java

@@ -25,6 +25,7 @@ import org.jclouds.Constants;
 import org.jclouds.atmos.AtmosAsyncClient;
 import org.jclouds.atmos.AtmosClient;
 import org.jclouds.atmos.handlers.AtmosClientErrorRetryHandler;
+import org.jclouds.atmos.handlers.AtmosServerErrorRetryHandler;
 import org.jclouds.atmos.handlers.ParseAtmosErrorFromXmlContent;
 import org.jclouds.date.DateService;
 import org.jclouds.date.TimeStamp;
@@ -92,6 +93,7 @@ public class AtmosRestClientModule extends RestClientModule<AtmosClient, AtmosAs
    @Override
    protected void bindRetryHandlers() {
       bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(AtmosClientErrorRetryHandler.class);
+      bind(HttpRetryHandler.class).annotatedWith(ServerError.class).to(AtmosServerErrorRetryHandler.class);
    }
 
 }

apis/atmos/src/main/java/org/jclouds/atmos/handlers/AtmosServerErrorRetryHandler.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.atmos.handlers;
+
+import javax.annotation.Resource;
+import javax.inject.Named;
+
+import org.jclouds.Constants;
+import org.jclouds.atmos.domain.AtmosError;
+import org.jclouds.atmos.util.AtmosUtils;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.HttpRetryHandler;
+import org.jclouds.http.HttpUtils;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
+import org.jclouds.logging.Logger;
+
+import com.google.inject.Inject;
+
+/**
+ * Handles Retryable responses with error codes in the 5xx range
+ *
+ * @see Error codes section at <a href="https://www.synaptic.att.com/assets/us/en/home/Atmos_Programmers_Guide_1.3.4A.pdf" />
+ * @author Andrew Gaul
+ */
+public class AtmosServerErrorRetryHandler implements HttpRetryHandler {
+   private final AtmosUtils utils;
+   private final BackoffLimitedRetryHandler backoffHandler;
+
+   @Inject
+   public AtmosServerErrorRetryHandler(BackoffLimitedRetryHandler backoffHandler,
+            AtmosUtils utils) {
+      this.backoffHandler = backoffHandler;
+      this.utils = utils;
+   }
+
+   @Inject(optional = true)
+   @Named(Constants.PROPERTY_MAX_RETRIES)
+   private int retryCountLimit = 5;
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
+      if (command.getFailureCount() > retryCountLimit) {
+         return false;
+      }
+      if (response.getStatusCode() == 500) {
+         byte[] content = HttpUtils.closeClientButKeepContentStream(response);
+         // Content can be null in the case of HEAD requests
+         if (content != null) {
+            try {
+               AtmosError error = utils.parseAtmosErrorFromContent(command, response,
+                        new String(content));
+               if (error.getCode() == 1040) {  // The server is busy. Please try again.
+                  return backoffHandler.shouldRetryRequest(command, response);
+               }
+               // don't increment count before here, since backoff handler does already
+               command.incrementFailureCount();
+            } catch (HttpException e) {
+               logger.warn(e, "error parsing response: %s", new String(content));
+            }
+         } else {
+            command.incrementFailureCount();
+         }
+         return false;
+      }
+      return false;
+   }
+
+}

apis/atmos/src/test/java/org/jclouds/atmos/handlers/AtmosServerErrorRetryHandlerTest.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.atmos.binders;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.jclouds.atmos.domain.AtmosError;
+import org.jclouds.atmos.handlers.AtmosServerErrorRetryHandler;
+import org.jclouds.atmos.util.AtmosUtils;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpResponse;
+import org.testng.annotations.Test;
+
+/**
+ * Tests behavior of {@code AtmosServerErrorRetryHandler}
+ *
+ * @author Andrew Gaul
+ */
+@Test(groups = "unit")
+public class AtmosServerErrorRetryHandlerTest {
+   private static final String HTTP_MESSAGE_FORMAT =
+         "<?xml version='1.0' encoding='UTF-8'?>\n" +
+         "<Error>\n" +
+         "<Code>%d</Code>\n" +
+         "<Message>%s</Message>\n" +
+         "</Error>\n";
+
+   @Test
+   public void testGet500WithoutError() {
+      AtmosUtils utils = createMock(AtmosUtils.class);
+      BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class);
+      HttpCommand command = createMock(HttpCommand.class);
+
+      expect(command.getFailureCount()).andReturn(0).once();
+      expect(command.incrementFailureCount()).andReturn(1).once();
+
+      replay(utils, backoffLimitedRetryHandler, command);
+
+      AtmosServerErrorRetryHandler retry = new AtmosServerErrorRetryHandler(backoffLimitedRetryHandler, utils);
+
+      assertFalse(retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(500).build()));
+
+      verify(utils, backoffLimitedRetryHandler, command);
+   }
+
+   @Test
+   public void testGet500WithError1040() {
+      AtmosUtils utils = createMock(AtmosUtils.class);
+      BackoffLimitedRetryHandler backoffLimitedRetryHandler = createMock(BackoffLimitedRetryHandler.class);
+      HttpCommand command = createMock(HttpCommand.class);
+      String content = String.format(HTTP_MESSAGE_FORMAT, 1040, "The server is busy. Please try again");
+      HttpResponse response = HttpResponse.builder().statusCode(500).payload(content).build();
+
+      expect(command.getFailureCount()).andReturn(0).once();
+      expect(utils.parseAtmosErrorFromContent(command, response, content)).andReturn(new AtmosError(1040, "The server is busy. Please try again")).once();
+      expect(backoffLimitedRetryHandler.shouldRetryRequest(command, response)).andReturn(true).once();
+
+      replay(utils, backoffLimitedRetryHandler, command);
+
+      AtmosServerErrorRetryHandler retry = new AtmosServerErrorRetryHandler(backoffLimitedRetryHandler, utils);
+
+      assertTrue(retry.shouldRetryRequest(command, response));
+
+      verify(utils, backoffLimitedRetryHandler, command);
+   }
+}