diff --git a/.gitignore b/.gitignore index e61e4381..cdd78d64 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,4 @@ build *.iws dependency-reduced-pom.xml .gradle -# Local workspace files (not part of the repo) -CLIENT_DEMO.md -*.pptx -src/test/java/com/emc/object/demo/ -video_tools/ test.properties -release-notes-*.md diff --git a/build.gradle b/build.gradle index 36e30d0b..59d8eaf3 100644 --- a/build.gradle +++ b/build.gradle @@ -320,13 +320,6 @@ clean { delete aggregatedDocsDir } -task runDemo(type: JavaExec, dependsOn: testClasses) { - group = 'Demo' - description = 'Runs EcsClientDemo against a live ECS cluster (configure via test.properties)' - classpath = sourceSets.test.runtimeClasspath - mainClass = 'com.emc.object.demo.EcsClientDemo' -} - // allow typing in credentials // note: this only works when run without the Gradle daemon (--no-daemon) gradle.taskGraph.whenReady { taskGraph -> diff --git a/src/main/java/com/emc/object/AbstractJerseyClient.java b/src/main/java/com/emc/object/AbstractJerseyClient.java index 808f027d..6699ec86 100644 --- a/src/main/java/com/emc/object/AbstractJerseyClient.java +++ b/src/main/java/com/emc/object/AbstractJerseyClient.java @@ -80,10 +80,6 @@ protected Response executeRequest(Client client, ObjectRequest request) { InputStream is = (InputStream) entity; InputStream buffered = is.markSupported() ? is : new BufferedInputStream(is, bufSize); buffered.mark(bufSize); - try { - ((EntityRequest) request).setEntity(buffered); - } catch (UnsupportedOperationException ignored) { - } entityStream = buffered; } } diff --git a/src/main/java/com/emc/object/EntityRequest.java b/src/main/java/com/emc/object/EntityRequest.java index 314ae859..a5e85468 100644 --- a/src/main/java/com/emc/object/EntityRequest.java +++ b/src/main/java/com/emc/object/EntityRequest.java @@ -34,8 +34,4 @@ public interface EntityRequest { Long getContentLength(); boolean isChunkable(); - - default void setEntity(Object entity) { - throw new UnsupportedOperationException("setEntity not supported on this request type"); - } } diff --git a/src/main/java/com/emc/object/s3/S3SignerV4.java b/src/main/java/com/emc/object/s3/S3SignerV4.java index 91b58be9..ae1c5020 100644 --- a/src/main/java/com/emc/object/s3/S3SignerV4.java +++ b/src/main/java/com/emc/object/s3/S3SignerV4.java @@ -182,11 +182,8 @@ protected SortedMap getCanonicalizedHeaders(Map metaBackup = new HashMap<>(userMeta); context.setProperty("com.emc.object.codecFilter.metaBackup", metaBackup); - // we need pre-stream metadata from the encoder, but we don't have the entity output stream, so we'll use - // a "dangling" output stream and connect it in the interceptor - DanglingOutputStream danglingStream = new DanglingOutputStream(); - OutputStream encodeStream = encodeChain.getEncodeStream(danglingStream, userMeta); - // NOTE: do NOT add encode metadata to context.getHeaders() here. // With HttpUrlConnectorProvider, Jersey's CommittingOutputStream defers the // setOutboundHeaders() callback until the first byte is written. If we add @@ -132,7 +130,8 @@ private void stripContentLength() { } } }; - danglingStream.setOutputStream(safeOut); + + OutputStream encodeStream = encodeChain.getEncodeStream(safeOut, userMeta); context.setOutputStream(encodeStream); try { @@ -146,10 +145,9 @@ private void stripContentLength() { // make sure we clear the content-length override for this thread if we set it SizeOverrideWriter.setEntitySize(null); } - return; + } else { + context.proceed(); } - - context.proceed(); } @SuppressWarnings("unchecked") @@ -222,30 +220,4 @@ public CodecFilter withCodecProperties(Map codecProperties) { return this; } - private static class DanglingOutputStream extends FilterOutputStream { - private static final OutputStream BOGUS_STREAM = new OutputStream() { - @Override - public void write(int b) throws IOException { - throw new RuntimeException("you didn't connect a dangling output stream!"); - } - }; - - DanglingOutputStream() { - super(BOGUS_STREAM); - } - - void setOutputStream(OutputStream out) { - this.out = out; - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - } - - @Override - public void write(int b) throws IOException { - throw new UnsupportedOperationException("single-byte write called!"); - } - } } diff --git a/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java b/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java index c89cb8c1..3b1707d4 100644 --- a/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java +++ b/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java @@ -30,8 +30,6 @@ import java.net.URL; import java.util.Map; -import org.glassfish.jersey.client.spi.ConnectorProvider; - import com.emc.codec.CodecChain; import com.emc.codec.encryption.DoesNotNeedRekeyException; import com.emc.codec.encryption.EncryptionCodec; @@ -123,11 +121,7 @@ public class S3EncryptionClient extends S3JerseyClient { private EncryptionConfig encryptionConfig; public S3EncryptionClient(S3Config s3Config, EncryptionConfig encryptionConfig) { - this(s3Config, null, encryptionConfig); - } - - public S3EncryptionClient(S3Config s3Config, ConnectorProvider connectorProvider, EncryptionConfig encryptionConfig) { - super(s3Config, connectorProvider); + super(s3Config); this.encryptionConfig = encryptionConfig; // create an encode chain based on parameters diff --git a/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java b/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java index 52439661..6f0b0c8b 100644 --- a/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java +++ b/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java @@ -37,8 +37,6 @@ import com.emc.rest.smart.jersey.SmartClientFactory; import com.emc.rest.smart.jersey.SmartFilter; import org.glassfish.jersey.client.ClientProperties; -import org.glassfish.jersey.client.HttpUrlConnectorProvider; -import org.glassfish.jersey.client.spi.ConnectorProvider; import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; @@ -117,13 +115,7 @@ * 
  *     System.setProperty(ReaderWriter.BUFFER_SIZE_SYSTEM_PROPERTY, "" + 128 * 1024); // 128k
  *
- * You can also try using Jersey's HttpUrlConnectorProvider, but be aware that this connector does not support - * Expect: 100-Continue behavior if that is important to you. You should also increase - * http.maxConnections to match your thread count. - * 
- *     System.setProperty("http.maxConnections", "" + 32); // if you have 32 threads
- *     S3Client s3Client = new S3JerseyClient(configX, new HttpUrlConnectorProvider());
- *
+ * This client always uses the Apache HTTP connector. */ public class S3JerseyClient extends AbstractJerseyClient implements S3Client { @@ -136,26 +128,7 @@ public class S3JerseyClient extends AbstractJerseyClient implements S3Client { protected RetryFilter retryFilter; public S3JerseyClient(S3Config s3Config) { - this(s3Config, null); - } - - /** - * Provide a specific Jersey {@link ConnectorProvider} implementation (default is - * {@link org.glassfish.jersey.apache.connector.ApacheConnectorProvider}). If you experience - * performance problems, you might try using - * {@link org.glassfish.jersey.client.HttpUrlConnectorProvider}, but note that it will not support the - * Expect: 100-Continue header and upload size is limited to 2GB. Also note that when using that - * provider, you should set the "http.maxConnections" system property to match your thread count - * (default is only 5). - */ - public S3JerseyClient(S3Config s3Config, ConnectorProvider connectorProvider) { super(new S3Config(s3Config)); // deep-copy config so that two clients don't share the same host lists (SDK-122) - // HttpURLConnection restricts certain headers (Host, Content-Length, etc.) by default. - // V4 signing requires the Host header to pass through unchanged, so we must allow - // restricted headers when using HttpUrlConnectorProvider. - if (connectorProvider instanceof HttpUrlConnectorProvider) { - System.setProperty("sun.net.http.allowRestrictedHeaders", "true"); - } this.s3Config = (S3Config) super.getObjectConfig(); if (this.s3Config.isUseV2Signer()) this.signer = new S3SignerV2(this.s3Config); @@ -169,11 +142,7 @@ public S3JerseyClient(S3Config s3Config, ConnectorProvider connectorProvider) { smartConfig.setProperty(ClientProperties.CHUNKED_ENCODING_SIZE, this.s3Config.getChunkedEncodingSize()); // creates a standard (non-load-balancing) jersey client - if (connectorProvider == null) { - client = SmartClientFactory.createStandardClient(smartConfig); - } else { - client = SmartClientFactory.createStandardClient(smartConfig, connectorProvider); - } + client = SmartClientFactory.createStandardClient(smartConfig); if (this.s3Config.isSmartClient()) { // SMART CLIENT SETUP @@ -208,11 +177,7 @@ public S3JerseyClient(S3Config s3Config, ConnectorProvider connectorProvider) { // S.C. - CLIENT CREATION // create a load-balancing jersey client - if (connectorProvider == null) { - client = SmartClientFactory.createSmartClient(smartConfig); - } else { - client = SmartClientFactory.createSmartClient(smartConfig, connectorProvider); - } + client = SmartClientFactory.createSmartClient(smartConfig); } // In Jersey 2.x, filters are registered on the client (order matters for request filters: diff --git a/src/main/java/com/emc/object/s3/request/PutObjectRequest.java b/src/main/java/com/emc/object/s3/request/PutObjectRequest.java index ba03af8b..1577bdd2 100644 --- a/src/main/java/com/emc/object/s3/request/PutObjectRequest.java +++ b/src/main/java/com/emc/object/s3/request/PutObjectRequest.java @@ -120,15 +120,6 @@ public Object getObject() { return object; } - public void setObject(Object object) { - this.object = object; - } - - @Override - public void setEntity(Object entity) { - setObject(entity); - } - public Range getRange() { return range; } diff --git a/src/main/java/com/emc/object/s3/request/UploadPartRequest.java b/src/main/java/com/emc/object/s3/request/UploadPartRequest.java index b9e75b77..2d352a6a 100644 --- a/src/main/java/com/emc/object/s3/request/UploadPartRequest.java +++ b/src/main/java/com/emc/object/s3/request/UploadPartRequest.java @@ -99,11 +99,6 @@ public void setObject(Object object) { this.object = object; } - @Override - public void setEntity(Object entity) { - setObject(entity); - } - public void setContentLength(Long contentLength) { this.contentLength = contentLength; } diff --git a/src/test/java/com/emc/object/s3/ExtendedConfigTest.java b/src/test/java/com/emc/object/s3/ExtendedConfigTest.java index 7f9dac30..1affa4ea 100644 --- a/src/test/java/com/emc/object/s3/ExtendedConfigTest.java +++ b/src/test/java/com/emc/object/s3/ExtendedConfigTest.java @@ -6,6 +6,7 @@ import javax.ws.rs.client.Client; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.junit.Assert; import org.junit.Test; @@ -46,9 +47,6 @@ private S3Config loadTestConfig() throws IOException { return s3Config; } - // NOTE: In Jersey 2.x with Apache connector, connection pool settings are configured - // through SmartClientFactory which sets up the Apache HttpClient 5.x connection manager. - // This test verifies that a custom connection limit can be set via SmartConfig properties. @Test public void testApacheConnectionLimit() throws IOException { S3Config s3Config = loadTestConfig(); @@ -56,15 +54,18 @@ public void testApacheConnectionLimit() throws IOException { int connectionLimitPerHost = 4; // non-default number int connectionLimitTotal = 39; // non-default number - // In Jersey 2.x, connection limits are set via SmartConfig properties s3Config.setProperty(SmartClientFactory.MAX_CONNECTIONS_PER_HOST, connectionLimitPerHost); s3Config.setProperty(SmartClientFactory.MAX_CONNECTIONS, connectionLimitTotal); TestS3JerseyClient s3Client = new TestS3JerseyClient(s3Config); - // verify the client was created successfully with custom config + // verify actual Apache connection pool settings were applied Client jerseyClient = s3Client.getClient(); - Assert.assertNotNull(jerseyClient); + PoolingHttpClientConnectionManager cm = (PoolingHttpClientConnectionManager) + jerseyClient.getConfiguration().getProperty(SmartClientFactory.CONNECTION_MANAGER_PROPERTY_KEY); + Assert.assertNotNull("Apache connection manager not found in client config", cm); + Assert.assertEquals(connectionLimitPerHost, cm.getDefaultMaxPerRoute()); + Assert.assertEquals(connectionLimitTotal, cm.getMaxTotal()); } static class TestS3JerseyClient extends S3JerseyClient { diff --git a/src/test/java/com/emc/object/s3/GeoPinningTest.java b/src/test/java/com/emc/object/s3/GeoPinningTest.java index 34d8d539..e6075330 100644 --- a/src/test/java/com/emc/object/s3/GeoPinningTest.java +++ b/src/test/java/com/emc/object/s3/GeoPinningTest.java @@ -148,28 +148,42 @@ public void testVdcDistribution() { @Test public void testReadRetryFailoverInFilter() throws Exception { - S3Config s3ConfigF = new S3Config(createS3Config()); + // create a separate client with geo-read-retry-failover enabled + S3Config s3ConfigF = createS3Config(); s3ConfigF.setGeoReadRetryFailover(true); + S3JerseyClient failoverClient = new S3JerseyClient(s3ConfigF); + Thread.sleep(500); // wait for polling daemon - String bucket = "foo"; - String key = "my/object/key"; - int geoIndex = 0xbb8619 % vdcs.size(); + try { + String key = "my/object/key"; + int geoIndex = 0xbb8619 % vdcs.size(); - // In Jersey 2.x, we test geo-pinning index calculation directly - // since we can't easily construct mock ClientRequestContext - int geoPinIndex = GeoPinningUtil.getGeoPinIndex(GeoPinningUtil.getGeoId(bucket, key), vdcs.size()); - Assert.assertEquals(geoIndex, geoPinIndex); + // write the test object + failoverClient.putObject(getTestBucket(), key, "Hello GeoPinning!", "text/plain"); - // test retry failover indices - int retryIndex1 = (geoIndex + 1) % vdcs.size(); - Assert.assertNotEquals(geoIndex, retryIndex1); + LoadBalancer loadBalancer = failoverClient.getLoadBalancer(); + loadBalancer.resetStats(); - int retryIndex2 = (geoIndex + 2) % vdcs.size(); - Assert.assertNotEquals(geoIndex, retryIndex2); + // read the object 10 times — should all route to the geo-pinned VDC + for (int i = 0; i < 10; i++) { + failoverClient.readObject(getTestBucket(), key, String.class); + } - // test 3rd retry (we have 3 VDCs, so this should go back to the primary) - int retryIndex3 = (geoIndex + 3) % vdcs.size(); - Assert.assertEquals(geoIndex, retryIndex3); + // verify no errors and total count + Assert.assertEquals(0, loadBalancer.getTotalErrors()); + Assert.assertEquals(10, loadBalancer.getTotalConnections()); + + // verify reads are routed to the correct VDC + for (HostStats stats : loadBalancer.getHostStats()) { + if (vdcs.get(geoIndex).equals(((VdcHost) stats).getVdc())) { + Assert.assertTrue(stats.getTotalConnections() > 0); + } else { + Assert.assertEquals(0, stats.getTotalConnections()); + } + } + } finally { + failoverClient.destroy(); + } } protected void testKeyDistribution(String key, int vdcIndex) { diff --git a/src/test/java/com/emc/object/s3/S3EncryptionUrlConnectionTest.java b/src/test/java/com/emc/object/s3/S3EncryptionUrlConnectionTest.java deleted file mode 100644 index a6f007f6..00000000 --- a/src/test/java/com/emc/object/s3/S3EncryptionUrlConnectionTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2015, EMC Corporation. - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * + Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * + The name of EMC Corporation may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package com.emc.object.s3; - -import java.net.URI; - -import org.glassfish.jersey.client.HttpUrlConnectorProvider; - -import com.emc.object.EncryptionConfig; -import com.emc.object.ObjectConfig; -import com.emc.object.s3.jersey.S3EncryptionClient; -import com.emc.object.s3.jersey.S3JerseyClient; - -public class S3EncryptionUrlConnectionTest extends S3EncryptionClientBasicTest { - @Override - protected String getTestBucketPrefix() { - return "s3-encryption-url-connection-test"; - } - - @Override - public S3Client createS3Client() throws Exception { - System.setProperty("http.maxConnections", "100"); - S3Config config = createS3Config(); - String proxy = config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); - if (proxy != null) { - URI proxyUri = new URI(proxy); - System.setProperty("http.proxyHost", proxyUri.getHost()); - System.setProperty("http.proxyPort", "" + proxyUri.getPort()); - } - HttpUrlConnectorProvider connectorProvider = new HttpUrlConnectorProvider(); - rclient = new S3JerseyClient(config, connectorProvider); - EncryptionConfig eConfig = createEncryptionConfig(); - eclient = new S3EncryptionClient(config, connectorProvider, eConfig); - encodeSpec = eConfig.getEncryptionSpec(); - if (eConfig.isCompressionEnabled()) encodeSpec = eConfig.getCompressionSpec() + "," + encodeSpec; - return eclient; - } -} diff --git a/src/test/java/com/emc/object/s3/S3EncryptionUrlConnectionV4Test.java b/src/test/java/com/emc/object/s3/S3EncryptionUrlConnectionV4Test.java deleted file mode 100644 index 8aa188d7..00000000 --- a/src/test/java/com/emc/object/s3/S3EncryptionUrlConnectionV4Test.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.emc.object.s3; - -import java.net.URI; - -import org.glassfish.jersey.client.HttpUrlConnectorProvider; -import org.junit.Assert; -import org.junit.Test; - -import com.emc.codec.CodecChain; -import com.emc.object.EncryptionConfig; -import com.emc.object.ObjectConfig; -import com.emc.object.s3.jersey.S3EncryptionClient; -import com.emc.object.s3.jersey.S3JerseyClient; - -public class S3EncryptionUrlConnectionV4Test extends S3EncryptionClientBasicTest { - @Override - protected String getTestBucketPrefix() { - return "s3-encryption-url-connection-v4-test"; - } - - @Override - public S3Client createS3Client() throws Exception { - System.setProperty("http.maxConnections", "100"); - S3Config config = createS3Config().withUseV2Signer(false); - String proxy = config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); - if (proxy != null) { - URI proxyUri = new URI(proxy); - System.setProperty("http.proxyHost", proxyUri.getHost()); - System.setProperty("http.proxyPort", "" + proxyUri.getPort()); - } - HttpUrlConnectorProvider connectorProvider = new HttpUrlConnectorProvider(); - rclient = new S3JerseyClient(config, connectorProvider); - EncryptionConfig eConfig = createEncryptionConfig(); - eclient = new S3EncryptionClient(config, connectorProvider, eConfig); - encodeSpec = eConfig.getEncryptionSpec(); - if (eConfig.isCompressionEnabled()) encodeSpec = eConfig.getCompressionSpec() + "," + encodeSpec; - return eclient; - } - - @Override - @Test - public void testRetries() throws Exception { - byte[] data = "Testing retries!!".getBytes(); - String key = "retry-test"; - - S3Config _config = createS3Config().withUseV2Signer(false); - _config.setFaultInjectionRate(0.4f); - _config.setRetryLimit(6); - S3Client _client = new S3EncryptionClient(_config, createEncryptionConfig()); - - // make sure we hit at least one error - for (int i = 0; i < 6; i++) { - _client.putObject(getTestBucket(), key, data, null); - S3ObjectMetadata metadata = rclient.getObjectMetadata(getTestBucket(), key); - Assert.assertEquals(encodeSpec, metadata.getUserMetadata(CodecChain.META_TRANSFORM_MODE)); - } - } -} diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index 6ad0ad6c..9c2bcd9b 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -2860,9 +2860,10 @@ public void run() { @Test public void testListMarkerWithSpecialChars() { String marker = "foo/bar/blah%blah&blah"; - ListObjectsResult result = client.listObjects(new ListObjectsRequest(getTestBucket()).withMarker(marker)); - + ListObjectsResult result = client.listObjects(new ListObjectsRequest(getTestBucket()).withMarker(marker) + .withEncodingType(EncodingType.url)); Assert.assertEquals(marker, result.getMarker()); + Assert.assertEquals(EncodingType.url, result.getEncodingType()); } @Test diff --git a/src/test/java/com/emc/object/s3/S3JerseyUrlConnectionTest.java b/src/test/java/com/emc/object/s3/S3JerseyUrlConnectionTest.java deleted file mode 100644 index 1c79dc72..00000000 --- a/src/test/java/com/emc/object/s3/S3JerseyUrlConnectionTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2015, EMC Corporation. - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * + Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * + The name of EMC Corporation may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package com.emc.object.s3; - -import java.io.InputStream; -import java.net.URI; - -import org.glassfish.jersey.client.HttpUrlConnectorProvider; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import com.emc.object.ObjectConfig; -import com.emc.object.s3.jersey.S3JerseyClient; -import com.emc.object.s3.request.PutObjectRequest; -import com.emc.util.RandomInputStream; - -public class S3JerseyUrlConnectionTest extends S3JerseyClientTest { - @Override - protected String getTestBucketPrefix() { - return "s3-url-connection-test"; - } - - @Override - public S3Client createS3Client() throws Exception { - System.setProperty("http.maxConnections", "100"); - S3Config config = createS3Config(); - String proxy = config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); - if (proxy != null) { - URI proxyUri = new URI(proxy); - System.setProperty("http.proxyHost", proxyUri.getHost()); - System.setProperty("http.proxyPort", "" + proxyUri.getPort()); - } - return new S3JerseyClient(config, new HttpUrlConnectorProvider()); - } - - @Ignore // only run this test against a co-located ECS! - @Test - public void testVeryLargeWrite() throws Exception { - String key = "very-large-object"; - long size = (long) Integer.MAX_VALUE + 102400; - InputStream content = new RandomInputStream(size); - S3ObjectMetadata metadata = new S3ObjectMetadata().withContentLength(size); - PutObjectRequest request = new PutObjectRequest(getTestBucket(), key, content).withObjectMetadata(metadata); - client.putObject(request); - - Assert.assertEquals(size, client.getObjectMetadata(getTestBucket(), key).getContentLength().longValue()); - } - - @Ignore // only run this test against a co-located ECS! - @Test - public void testVeryLargeChunkedWrite() throws Exception { - String key = "very-large-chunked-object"; - long size = (long) Integer.MAX_VALUE + 102400; - InputStream content = new RandomInputStream(size); - client.putObject(getTestBucket(), key, content, null); - - Assert.assertEquals(size, client.getObjectMetadata(getTestBucket(), key).getContentLength().longValue()); - } -} diff --git a/src/test/java/com/emc/object/s3/S3JerseyUrlConnectionV4Test.java b/src/test/java/com/emc/object/s3/S3JerseyUrlConnectionV4Test.java deleted file mode 100644 index a8c2b3f6..00000000 --- a/src/test/java/com/emc/object/s3/S3JerseyUrlConnectionV4Test.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.emc.object.s3; - -import java.io.InputStream; -import java.net.URI; - -import org.glassfish.jersey.client.HttpUrlConnectorProvider; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import com.emc.object.ObjectConfig; -import com.emc.object.s3.jersey.S3JerseyClient; -import com.emc.object.s3.request.PutObjectRequest; -import com.emc.util.RandomInputStream; - -public class S3JerseyUrlConnectionV4Test extends S3JerseyClientV4Test { - @Override - protected String getTestBucketPrefix() { - return "s3-url-connection-v4-test"; - } - - @Override - public S3Client createS3Client() throws Exception { - System.setProperty("http.maxConnections", "100"); - S3Config config = createS3Config().withUseV2Signer(false); - String proxy = config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); - if (proxy != null) { - URI proxyUri = new URI(proxy); - System.setProperty("http.proxyHost", proxyUri.getHost()); - System.setProperty("http.proxyPort", "" + proxyUri.getPort()); - } - return new S3JerseyClient(config, new HttpUrlConnectorProvider()); - } - - @Ignore // only run this test against a co-located ECS! - @Test - public void testVeryLargeWrite() throws Exception { - String key = "very-large-object"; - long size = (long) Integer.MAX_VALUE + 102400; - InputStream content = new RandomInputStream(size); - S3ObjectMetadata metadata = new S3ObjectMetadata().withContentLength(size); - PutObjectRequest request = new PutObjectRequest(getTestBucket(), key, content).withObjectMetadata(metadata); - client.putObject(request); - - Assert.assertEquals(size, client.getObjectMetadata(getTestBucket(), key).getContentLength().longValue()); - } - - @Ignore // only run this test against a co-located ECS! - @Test - public void testVeryLargeChunkedWrite() throws Exception { - String key = "very-large-chunked-object"; - long size = (long) Integer.MAX_VALUE + 102400; - InputStream content = new RandomInputStream(size); - client.putObject(getTestBucket(), key, content, null); - - Assert.assertEquals(size, client.getObjectMetadata(getTestBucket(), key).getContentLength().longValue()); - } -} diff --git a/src/test/java/com/emc/object/s3/WriteTruncationTest.java b/src/test/java/com/emc/object/s3/WriteTruncationTest.java index a0dd2e2a..adf1750a 100644 --- a/src/test/java/com/emc/object/s3/WriteTruncationTest.java +++ b/src/test/java/com/emc/object/s3/WriteTruncationTest.java @@ -9,7 +9,6 @@ import javax.xml.bind.DatatypeConverter; import org.apache.commons.codec.digest.DigestUtils; -import org.junit.After; import org.junit.Assert; import org.junit.Assume; import org.junit.Ignore; @@ -28,13 +27,10 @@ public class WriteTruncationTest extends AbstractS3ClientTest { static final int OBJECT_RETENTION_PERIOD = 15; // 15 seconds static final int MOCK_OBJ_SIZE = 5 * 1024 * 1024; // 5MB - S3Client jvmClient; final Random random = new Random(); @Override protected S3Client createS3Client() throws Exception { - S3Config s3Config = createS3Config().withRetryEnabled(false); - this.jvmClient = new S3JerseyClient(s3Config); return new S3JerseyClient(createS3Config().withRetryEnabled(false)); } @@ -66,39 +62,14 @@ protected void cleanUpBucket(String bucketName) { super.cleanUpBucket(bucketName); } - @After - public void shutdownJvmClient() { - if (jvmClient != null) jvmClient.destroy(); - } - - @Test - public void testIOExceptionDuringReadApache() { - testTruncatedWrite(true, true, ExceptionType.IOException, 0, false); - } - - @Test - public void testIOExceptionDuringReadJvm() { - testTruncatedWrite(false, true, ExceptionType.IOException, 0, false); - } - - @Test - public void testRuntimeExceptionDuringReadApache() { - testTruncatedWrite(true, true, ExceptionType.RuntimeException, 0, false); - } - @Test - public void testRuntimeExceptionDuringReadJvm() { - testTruncatedWrite(false, true, ExceptionType.RuntimeException, 0, false); + public void testIOExceptionDuringRead() { + testTruncatedWrite(true, ExceptionType.IOException, 0, false); } - // This will fail because ECS does not know how much data is the correct amount (as would be the case if - // Content-Length was used) - // TODO: the client still seems to send a 0-byte chunk terminator, which it probably shouldn't - // - see if we can stop the write such that ECS does not commit it - @Ignore @Test - public void testIOExceptionChunkedEncodingApache() { - testTruncatedWrite(true, false, ExceptionType.IOException, 0, false); + public void testRuntimeExceptionDuringRead() { + testTruncatedWrite(true, ExceptionType.RuntimeException, 0, false); } // This will fail because ECS does not know how much data is the correct amount (as would be the case if @@ -107,46 +78,32 @@ public void testIOExceptionChunkedEncodingApache() { // - see if we can stop the write such that ECS does not commit it @Ignore @Test - public void testIOExceptionChunkedEncodingJvm() { - testTruncatedWrite(false, false, ExceptionType.IOException, 0, false); - } - - // (see above) - // However, when a Content-MD5 header is sent on the PUT, ECS will reject it if the data doesn't match - this is our workaround - @Test - public void testIOExceptionChunkedEncodingMd5Apache() { - testTruncatedWrite(true, false, ExceptionType.IOException, 0, true); + public void testIOExceptionChunkedEncoding() { + testTruncatedWrite(false, ExceptionType.IOException, 0, false); } // (see above) // However, when a Content-MD5 header is sent on the PUT, ECS will reject it if the data doesn't match - this is our workaround @Test - public void testIOExceptionChunkedEncodingMd5Jvm() { - testTruncatedWrite(false, false, ExceptionType.IOException, 0, true); - } - - @Test - public void testDelayedIOExceptionApache() { - testTruncatedWrite(true, true, ExceptionType.IOException, 61, false); + public void testIOExceptionChunkedEncodingMd5() { + testTruncatedWrite(false, ExceptionType.IOException, 0, true); } @Test - public void testDelayedIOExceptionJvm() { - testTruncatedWrite(false, true, ExceptionType.IOException, 61, false); + public void testDelayedIOException() { + testTruncatedWrite(true, ExceptionType.IOException, 61, false); } - void testTruncatedWrite(boolean useApacheClient, - boolean setContentLength, + void testTruncatedWrite(boolean setContentLength, ExceptionType exceptionType, int delayBeforeException, boolean sendContentMd5) { - S3Client s3Client = useApacheClient ? this.client : this.jvmClient; + S3Client s3Client = this.client; - String key = String.format("read-%s%s-%s%stest", + String key = String.format("read-%s%s-%stest", delayBeforeException > 0 ? "delayed-" : "", exceptionType, - setContentLength ? "chunked-" : "", - useApacheClient ? "apache-" : "jvm-"); + setContentLength ? "chunked-" : ""); S3ObjectMetadata metadata = new S3ObjectMetadata(); if (setContentLength) metadata.withContentLength(MOCK_OBJ_SIZE); metadata.withRetentionPeriod((long) OBJECT_RETENTION_PERIOD); diff --git a/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java b/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java index f03aa1e1..6cfce616 100644 --- a/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java +++ b/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java @@ -1,14 +1,11 @@ package com.emc.object.s3; import com.emc.object.s3.jersey.S3JerseyClient; -// URLConnectionClientHandler removed in Jersey 2.x migration public class WriteTruncationV4Test extends WriteTruncationTest { @Override protected S3Client createS3Client() throws Exception { - S3Config s3Config = createS3Config().withRetryEnabled(false).withUseV2Signer(false); - this.jvmClient = new S3JerseyClient(s3Config); - return new S3JerseyClient(createS3Config().withRetryEnabled(false)); + return new S3JerseyClient(createS3Config().withRetryEnabled(false).withUseV2Signer(false)); } @Override diff --git a/src/test/resources/log4j2.xml b/src/test/resources/log4j2.xml index a7fc26e8..6d10d2ec 100644 --- a/src/test/resources/log4j2.xml +++ b/src/test/resources/log4j2.xml @@ -11,8 +11,10 @@ +