New tests for session overlap and token retry

ricellis · ricellis · commit 98512955e040 · 2019-09-26T16:11:44.000+01:00
Added a test for multiple threads expriencing session expiry
and trying to renew, asserting that only one renewal attempt
is made.

Added IAM token 429 retry tests for success and failure cases.
diff --git a/cloudant-client/src/test/java/com/cloudant/tests/HttpIamTest.java b/cloudant-client/src/test/java/com/cloudant/tests/HttpIamTest.java
@@ -38,6 +38,7 @@
 import com.cloudant.client.api.CloudantClient;
 import com.cloudant.client.org.lightcouch.CouchDbException;
 import com.cloudant.http.Http;
+import com.cloudant.http.interceptors.Replay429Interceptor;
 import com.cloudant.tests.extensions.MockWebServerExtension;
 import com.cloudant.tests.util.IamSystemPropertyMock;
 import com.cloudant.tests.util.MockWebServerResources;
@@ -322,7 +323,6 @@ public void iamRenewalFailureOnIamToken() throws Exception {
                 assertThrows(CouchDbException.class,
                         () -> c.executeRequest(Http.GET(c.getBaseUri())).responseAsString(),
                         "Failure to get a token should throw a CouchDbException.");
-        re.printStackTrace();
         assertTrue(re.getMessage().startsWith("HTTP response error getting session"), "The " +
                 "exception should have been for a HTTP response error.");
 
@@ -458,5 +458,166 @@ public void iamRenewalFailureOnSessionCookie() throws Exception {
                         "/identity/token");
     }
 
+    @Test
+    public void iamTokenServer429RetryAndSucceed() throws Exception {
+        // Mock request sequence
+        mockIamServer.enqueue(new MockResponse().setResponseCode(200).setBody(IAM_TOKEN));
+        mockWebServer.enqueue(OK_IAM_COOKIE);
+        // First get request succeeds
+        mockWebServer.enqueue(new MockResponse().setResponseCode(200)
+                .setBody(hello));
+
+        // Second get request has a 401 cookie expired
+        mockWebServer.enqueue(new MockResponse().setResponseCode(401).
+                setBody("{\"error\":\"credentials_expired\"}"));
+        // IAM server 429 on token request
+        mockIamServer.enqueue(new MockResponse().setStatus("HTTP/1.1 429 Too many requests"));
+        // Success on retry
+        mockIamServer.enqueue(new MockResponse().setResponseCode(200).setBody(IAM_TOKEN_2));
+        mockWebServer.enqueue(OK_IAM_COOKIE_2);
+        // Second get request suceeds after renewal
+        mockWebServer.enqueue(new MockResponse().setResponseCode(200)
+                .setBody(hello));
+
+        // Request sequence
+        CloudantClient c = CloudantClientHelper.newMockWebServerClientBuilder(mockWebServer)
+                .iamApiKey(IAM_API_KEY)
+                .interceptors(Replay429Interceptor.WITH_DEFAULTS)
+                .build();
+
+        String response = c.executeRequest(Http.GET(c.getBaseUri())).responseAsString();
+        assertEquals(hello, response, "The expected response should be received");
+
+        String response2 = c.executeRequest(Http.GET(c.getBaseUri())).responseAsString();
+        assertEquals(hello, response2, "The expected response should be received");
+
+        // iam mock server
+
+        // assert that there were 3 calls
+        RecordedRequest[] recordedIamRequests = takeN(mockIamServer, 3);
+        // first time, automatically fetch because cookie jar is empty
+        assertEquals(iamTokenEndpoint,
+                recordedIamRequests[0].getPath(), "The request should have been for " +
+                        "/identity/token");
+        assertThat("The request body should contain the IAM API key",
+                recordedIamRequests[0].getBody().readString(Charset.forName("UTF-8")),
+                containsString("apikey=" + IAM_API_KEY));
+        // second time, 429 response
+        assertEquals(iamTokenEndpoint,
+                recordedIamRequests[1].getPath(), "The request should have been for " +
+                        "/identity/token");
+        // third time, refresh because the cloudant session cookie has expired
+        assertEquals(iamTokenEndpoint,
+                recordedIamRequests[1].getPath(), "The request should have been for " +
+                        "/identity/token");
+
+        // cloudant mock server
+
+        // assert that there were 5 calls
+        RecordedRequest[] recordedRequests = takeN(mockWebServer, 5);
+
+        assertEquals("/_iam_session",
+                recordedRequests[0].getPath(), "The request should have been for /_iam_session");
+        assertThat("The request body should contain the IAM token",
+                recordedRequests[0].getBody().readString(Charset.forName("UTF-8")),
+                containsString(IAM_TOKEN));
+        // first request
+        assertEquals("/",
+                recordedRequests[1].getPath(), "The request should have been for /");
+        // The cookie may or may not have the session id quoted, so check both
+        assertThat("The Cookie header should contain the expected session value",
+                recordedRequests[1].getHeader("Cookie"),
+                anyOf(containsString(iamSession(EXPECTED_OK_COOKIE)),
+                        containsString(iamSessionUnquoted(EXPECTED_OK_COOKIE))));
+        // second request (rejected for cookie expiry)
+        assertEquals("/",
+                recordedRequests[2].getPath(), "The request should have been for /");
+        // with new valid token get a new session
+        assertEquals("/_iam_session",
+                recordedRequests[3].getPath(), "The request should have been for /_iam_session");
+        assertThat("The request body should contain the IAM token",
+                recordedRequests[3].getBody().readString(Charset.forName("UTF-8")),
+                containsString(IAM_TOKEN_2));
+        assertEquals("/",
+                recordedRequests[2].getPath(), "The request should have been for /");
+    }
+
+    @Test
+    public void iamTokenServer429RetryAndFail() throws Exception {
+        // Mock request sequence
+        mockIamServer.enqueue(new MockResponse().setResponseCode(200).setBody(IAM_TOKEN));
+        mockWebServer.enqueue(OK_IAM_COOKIE);
+        // First get request succeeds
+        mockWebServer.enqueue(new MockResponse().setResponseCode(200)
+                .setBody(hello));
+
+        // Second get request has a 401 cookie expired
+        mockWebServer.enqueue(new MockResponse().setResponseCode(401).
+                setBody("{\"error\":\"credentials_expired\"}"));
+        // IAM server 429 on subsequent token requests
+        mockIamServer.enqueue(new MockResponse().setStatus("HTTP/1.1 429 Too many requests"));
+        mockIamServer.enqueue(new MockResponse().setStatus("HTTP/1.1 429 Too many requests"));
+        mockIamServer.enqueue(new MockResponse().setStatus("HTTP/1.1 429 Too many requests"));
+        mockIamServer.enqueue(new MockResponse().setStatus("HTTP/1.1 429 Too many requests"));
+        // Request will fail
+
+        // Request sequence
+        CloudantClient c = CloudantClientHelper.newMockWebServerClientBuilder(mockWebServer)
+                .iamApiKey(IAM_API_KEY)
+                .interceptors(Replay429Interceptor.WITH_DEFAULTS)
+                .build();
 
+        String response = c.executeRequest(Http.GET(c.getBaseUri())).responseAsString();
+        assertEquals(hello, response, "The expected response should be received");
+
+        CouchDbException re =
+                assertThrows(CouchDbException.class,
+                        () -> c.executeRequest(Http.GET(c.getBaseUri())).responseAsString(),
+                        "Failure to get a token should throw a CouchDbException.");
+        assertTrue(re.getMessage().startsWith("HTTP response error getting session"), "The " +
+                "exception should have been for a HTTP response error.");
+        assertTrue(re.getMessage().contains("response code 429"), "The exception should report a " +
+                "429 response code");
+
+        // iam mock server
+
+        // assert that there were 5 calls
+        RecordedRequest[] recordedIamRequests = takeN(mockIamServer, 5);
+        // first time, automatically fetch because cookie jar is empty
+        assertEquals(iamTokenEndpoint,
+                recordedIamRequests[0].getPath(), "The request should have been for " +
+                        "/identity/token");
+        assertThat("The request body should contain the IAM API key",
+                recordedIamRequests[0].getBody().readString(Charset.forName("UTF-8")),
+                containsString("apikey=" + IAM_API_KEY));
+        // 4 more times all 429 responses
+        for (int i = 1; i <= 4; i++) {
+            assertEquals(iamTokenEndpoint,
+                    recordedIamRequests[i].getPath(), "The request[" + i + "] should have been " +
+                            "for " +
+                            "/identity/token");
+        }
+
+        // cloudant mock server
+
+        // assert that there were 3 calls
+        RecordedRequest[] recordedRequests = takeN(mockWebServer, 3);
+
+        assertEquals("/_iam_session",
+                recordedRequests[0].getPath(), "The request should have been for /_iam_session");
+        assertThat("The request body should contain the IAM token",
+                recordedRequests[0].getBody().readString(Charset.forName("UTF-8")),
+                containsString(IAM_TOKEN));
+        // first request
+        assertEquals("/",
+                recordedRequests[1].getPath(), "The request should have been for /");
+        // The cookie may or may not have the session id quoted, so check both
+        assertThat("The Cookie header should contain the expected session value",
+                recordedRequests[1].getHeader("Cookie"),
+                anyOf(containsString(iamSession(EXPECTED_OK_COOKIE)),
+                        containsString(iamSessionUnquoted(EXPECTED_OK_COOKIE))));
+        // Second request gives a 401 that starts token renewal
+        // Retry of that request never reaches the server
+        assertEquals("/", recordedRequests[1].getPath(), "The request should have been for /");
+    }
 }
diff --git a/cloudant-client/src/test/java/com/cloudant/tests/HttpTest.java b/cloudant-client/src/test/java/com/cloudant/tests/HttpTest.java
@@ -86,10 +86,16 @@
 import java.net.URLClassLoader;
 import java.net.URLDecoder;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
@@ -1103,6 +1109,100 @@ public void noErrorStream401() throws Exception {
         assertEquals("TEST", response, "The expected response body should be received");
     }
 
+    /**
+     * This test checks that only a single session renewal request is made on expiry.
+     * Flow:
+     * - First request to _all_dbs
+     *   - sends a _session request and gets OK_COOKIE
+     *   - _all_dbs returns ["a"]
+     * - Multi-threaded requests to root endpoint
+     *   - Any that occur before session renewal get a 401 unauthorized and try to renew the session
+     *   - a _session request will return OK_COOKIE_2 but can only be invoked once for test purposes
+     *   - any requests after session renewal will get an OK response
+     *
+     * @throws Exception
+     */
+    @TestTemplate
+    public void singleSessionRequestOnExpiry() throws Exception {
+        final AtomicInteger sessionCounter = new AtomicInteger();
+        mockWebServer.setDispatcher(new Dispatcher() {
+
+            // Use 444 response for error cases as we know this will get an exception without retries
+            private final MockResponse FAIL = new MockResponse().setStatus("HTTP/1.1 444 session locking fail");
+
+            @Override
+            public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
+                if (request.getPath().endsWith("_session")) {
+                    int session = sessionCounter.incrementAndGet();
+                    switch (session) {
+                        case 1:
+                            return OK_COOKIE;
+                        case 2:
+                            return OK_COOKIE_2;
+                        default:
+                            return FAIL;
+                    }
+                } else if (request.getPath().endsWith("_all_dbs")) {
+                    return new MockResponse().setBody("[\"a\"]");
+                } else {
+                    String cookie = request.getHeader("COOKIE");
+                    if (cookie.contains(EXPECTED_OK_COOKIE)) {
+                        // Request in first session
+                        return new MockResponse().setResponseCode(401);
+                    } else if (cookie.contains(EXPECTED_OK_COOKIE_2)) {
+                        // Request in second session, return OK
+                        return new MockResponse();
+                    } else {
+                        return FAIL;
+                    }
+                }
+            }
+        });
+
+        CloudantClient c = CloudantClientHelper.newMockWebServerClientBuilder(mockWebServer)
+                .username("a")
+                .password("b")
+                .build();
+
+        // Do a single request to start the first session
+        c.getAllDbs();
+
+        // Now run lots of requests simultaneously
+        int threads = 25;
+        int requests = 1250;
+
+        ExecutorService executorService = Executors.newFixedThreadPool(threads);
+
+        List<ServerInfoCallable> tasks = new ArrayList<ServerInfoCallable>(requests);
+        for (int i = 0; i < requests; i++) {
+            tasks.add(new ServerInfoCallable(c));
+        }
+        List<Future<Throwable>> results = executorService.invokeAll(tasks);
+        for (Future<Throwable> result : results) {
+            assertNull(result.get(), "There should be no exceptions.");
+        }
+        assertEquals(2, sessionCounter.get(), "There should only be 2 session requests");
+    }
+
+    private final class ServerInfoCallable implements Callable<Throwable> {
+
+        private final CloudantClient c;
+
+        ServerInfoCallable(CloudantClient c) {
+            this.c = c;
+        }
+
+        @Override
+        public Throwable call() {
+            try {
+                c.metaInformation();
+            } catch (Throwable t) {
+                return t;
+            }
+            return null;
+        }
+    }
+
     /**
      * Tests that loading the OkHelper will not try to use any classes from outside the
      * cloudant-http built classes. Specifically it won't cause any okhttp classes to try to be
diff --git a/cloudant-client/src/test/java/com/cloudant/tests/util/IamSystemPropertyMock.java b/cloudant-client/src/test/java/com/cloudant/tests/util/IamSystemPropertyMock.java
