From 919ff0015864dd2ce39056bd374bc8989ab2cec8 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Wed, 7 May 2014 13:21:53 +0200 Subject: [PATCH 01/47] JCBC-457: Force CCCP config fetching on node reconnect. Motivation ---------- If a node needs to be reconnected, there is a strong indication that the socket has been closed and this could be because of a topology change. Modification ------------ If a reconnect is scheduled, make sure it forces a config update. Also, the method is added for memcache buckets to keep the behavior consistent. Result ------ Quicker detection of topology changes, eventually getting quicker to a valid config state. Change-Id: I5244dfc6d6f19288977ef98745d47efe25773093 Reviewed-on: http://review.couchbase.org/36779 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- .../couchbase/client/CouchbaseConnection.java | 25 +++++++------ .../client/CouchbaseMemcachedConnection.java | 35 +++++++++++++++---- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index a6d4a8e6..e6ef8fe8 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -24,10 +24,18 @@ import com.couchbase.client.internal.AdaptiveThrottler; import com.couchbase.client.internal.ThrottleManager; -import com.couchbase.client.vbucket.ConfigurationProvider; import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.VBucketNodeLocator; import com.couchbase.client.vbucket.config.Bucket; +import net.spy.memcached.ConnectionObserver; +import net.spy.memcached.FailureMode; +import net.spy.memcached.MemcachedConnection; +import net.spy.memcached.MemcachedNode; +import net.spy.memcached.OperationFactory; +import net.spy.memcached.ops.KeyedOperation; +import net.spy.memcached.ops.Operation; +import net.spy.memcached.ops.ReplicaGetOperation; +import net.spy.memcached.ops.VBucketAware; import java.io.IOException; import java.net.InetSocketAddress; @@ -43,16 +51,6 @@ import java.util.List; import java.util.Map; -import net.spy.memcached.ConnectionObserver; -import net.spy.memcached.FailureMode; -import net.spy.memcached.MemcachedConnection; -import net.spy.memcached.MemcachedNode; -import net.spy.memcached.OperationFactory; -import net.spy.memcached.ops.KeyedOperation; -import net.spy.memcached.ops.Operation; -import net.spy.memcached.ops.ReplicaGetOperation; -import net.spy.memcached.ops.VBucketAware; - /** * Maintains connections to each node in a cluster of Couchbase Nodes. * @@ -326,10 +324,15 @@ protected void handleRetryInformation(byte[] retryMessage) { /** * Only queue for reconnect if the given node is still part of the cluster. * + * Since a node is queued to reconnect, it indicates a close socket and + * therefore an outdated configuration. With some providers, it is important + * to force a config reload which is also issued immediately. + * * @param node the node to check. */ @Override protected void queueReconnect(final MemcachedNode node) { + cf.getConfigurationProvider().reloadConfig(); if (isShutDown() || !locator.getAll().contains(node)) { getLogger().debug("Preventing reconnect for node " + node + " because it" + "is either not part of the cluster anymore or the connection is " diff --git a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java index 371737e2..78f06872 100644 --- a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java @@ -25,6 +25,12 @@ import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.VBucketNodeLocator; import com.couchbase.client.vbucket.config.Bucket; +import net.spy.memcached.ConnectionObserver; +import net.spy.memcached.FailureMode; +import net.spy.memcached.MemcachedConnection; +import net.spy.memcached.MemcachedNode; +import net.spy.memcached.OperationFactory; +import net.spy.memcached.ops.Operation; import java.io.IOException; import java.net.InetSocketAddress; @@ -38,13 +44,6 @@ import java.util.Iterator; import java.util.List; -import net.spy.memcached.ConnectionObserver; -import net.spy.memcached.FailureMode; -import net.spy.memcached.MemcachedConnection; -import net.spy.memcached.MemcachedNode; -import net.spy.memcached.OperationFactory; -import net.spy.memcached.ops.Operation; - /** * Couchbase implementation of CouchbaseConnection. * @@ -227,6 +226,28 @@ public void run() { getLogger().info("Shut down Couchbase client"); } + /** + * Only queue for reconnect if the given node is still part of the cluster. + * + * Since a node is queued to reconnect, it indicates a close socket and + * therefore an outdated configuration. With some providers, it is important + * to force a config reload which is also issued immediately. + * + * @param node the node to check. + */ + @Override + protected void queueReconnect(final MemcachedNode node) { + cf.getConfigurationProvider().reloadConfig(); + if (isShutDown() || !locator.getAll().contains(node)) { + getLogger().debug("Preventing reconnect for node " + node + " because it" + + "is either not part of the cluster anymore or the connection is " + + "shutting down."); + return; + } + + super.queueReconnect(node); + } + private void logRunException(Exception e) { if (shutDown) { // There are a couple types of errors that occur during the From 31c7ad01f02b06ace73ab04e7cca3e684bf01fbd Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Thu, 24 Apr 2014 10:55:21 +0200 Subject: [PATCH 02/47] JCBC-453: Also check for fresh configs with failure modes != redistribute Motivation ---------- When a different failure mode is used, it can be the case that a new configuration is not picked up (ie with cancel). Modification ------------ This changeset also makes sure we update the config reload threshold if other failure modes are used. Result ------ New configurations eventually get picked up even if other failure modes are used. Change-Id: I302b2192f2fc34ea7df36d3b83ce63197513d7be Reviewed-on: http://review.couchbase.org/36828 Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../couchbase/client/CouchbaseConnection.java | 22 +++++++++++++------ .../client/CouchbaseMemcachedConnection.java | 16 ++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index e6ef8fe8..d41a1d98 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -196,10 +196,13 @@ public void addOperation(final String key, final Operation o) { return; } + boolean needsRecheckConfigUpdate = false; if (primary.isActive() || failureMode == FailureMode.Retry) { placeIn = primary; + needsRecheckConfigUpdate = !primary.isActive(); } else if (failureMode == FailureMode.Cancel) { o.cancel(); + needsRecheckConfigUpdate = true; } else { // Look for another node in sequence that is ready. for (Iterator i = locator.getSequence(key); placeIn == null @@ -209,18 +212,23 @@ public void addOperation(final String key, final Operation o) { placeIn = n; } } - // If we didn't find an active node, queue it in the primary node - // and wait for it to come back online. + if (placeIn == null) { placeIn = primary; - getLogger().warn( - "Node expected to receive data is inactive. This could be due to " - + "a failure within the cluster. Will check for updated " - + "configuration. Key without a configured node is: %s.", key); - cf.checkConfigUpdate(); + needsRecheckConfigUpdate = true; } } + // If we didn't find an active node, queue it in the primary node + // and wait for it to come back online. + if (needsRecheckConfigUpdate) { + getLogger().warn( + "Node expected to receive data is inactive. This could be due to " + + "a failure within the cluster. Will check for updated " + + "configuration. Key without a configured node is: %s.", key); + cf.checkConfigUpdate(); + } + assert o.isCancelled() || placeIn != null : "No node found for key " + key; if (placeIn != null) { // add the vbucketIndex to the operation diff --git a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java index 78f06872..b5221b88 100644 --- a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java @@ -164,14 +164,13 @@ protected void addOperation(final String key, final Operation o) { return; } - if(!primary.isActive()) { - cf.checkConfigUpdate(); - } - + boolean needsRecheckConfigUpdate = false; if (primary.isActive() || failureMode == FailureMode.Retry) { placeIn = primary; + needsRecheckConfigUpdate = !primary.isActive(); } else if (failureMode == FailureMode.Cancel) { o.cancel(); + needsRecheckConfigUpdate = true; } else { // Look for another node in sequence that is ready. for (Iterator i = locator.getSequence(key); placeIn == null @@ -186,12 +185,21 @@ protected void addOperation(final String key, final Operation o) { // and wait for it to come back online. if (placeIn == null) { placeIn = primary; + needsRecheckConfigUpdate = true; this.getLogger().warn( "Could not redistribute " + "to another node, retrying primary node for %s.", key); } } + if (needsRecheckConfigUpdate) { + getLogger().warn( + "Node expected to receive data is inactive. This could be due to " + + "a failure within the cluster. Will check for updated " + + "configuration. Key without a configured node is: %s.", key); + cf.checkConfigUpdate(); + } + assert o.isCancelled() || placeIn != null : "No node found for key " + key; if (placeIn != null) { addOperation(placeIn, o); From 407b2352a44b58ef398f2ff36208eb980d22e78d Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Thu, 8 May 2014 10:09:11 +0200 Subject: [PATCH 03/47] Upgrade spymemcached to 2.11.2 Change-Id: I2209edf896c448b4b14e71572866fd5057a1de30 Reviewed-on: http://review.couchbase.org/36829 Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index c8497d09..09c8f515 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.0 -spymemcached-test.version=2.11.0 +spymemcached.version=2.11.2 +spymemcached-test.version=2.11.2 From a5ab1d36ef0cc3b947fc184b6602bc16dd84a517 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 12 May 2014 14:07:43 +0200 Subject: [PATCH 04/47] JCBC-460: Make sure replica vbucket changes count as significant. Motivation ---------- When a new configuration arise, there is a simple difference heuristic in place which checks if a new configuration is worth applying or if it only contains insignificant changes that are not worth the effort of applying a new configuration. One of those measurements is if vbuckets have changed (the values in the vbuckets array). One case that was not thought of before is if only replica vbuckets change and the master buckets stay the same. This has been recently found to be an issue when configurations are not applied because only the replica vbucket information has changed. Modifications ------------- The algorithm now also checks the content of the replica vbuckets to see if a significant modification has been made, leading to an update of the map to the according parties. Result ------ New configuration is now also applied if only the replica vbuckets change. A test case has been added to verivy the functionality (and fails without the change). Change-Id: I50d81f0053360b1c073fa90c217bde18abafd78d Reviewed-on: http://review.couchbase.org/36985 Reviewed-by: Matt Ingenthron Reviewed-by: Jeffry Morris Tested-by: Michael Nitschinger --- .../vbucket/config/CouchbaseConfig.java | 9 +++-- .../vbucket/config/CouchbaseConfigTest.java | 37 +++++++++++++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/couchbase/client/vbucket/config/CouchbaseConfig.java b/src/main/java/com/couchbase/client/vbucket/config/CouchbaseConfig.java index f6063b60..b2357fe5 100644 --- a/src/main/java/com/couchbase/client/vbucket/config/CouchbaseConfig.java +++ b/src/main/java/com/couchbase/client/vbucket/config/CouchbaseConfig.java @@ -22,15 +22,15 @@ package com.couchbase.client.vbucket.config; +import net.spy.memcached.HashAlgorithm; +import net.spy.memcached.compat.SpyObject; + import java.net.InetSocketAddress; import java.net.URL; import java.util.HashSet; import java.util.List; import java.util.Set; -import net.spy.memcached.HashAlgorithm; -import net.spy.memcached.compat.SpyObject; - /** * A {@link Config} implementation that represents a "couchbase" bucket config. * @@ -197,6 +197,9 @@ public ConfigDifference compareTo(Config config) { int vbucketsChanges = 0; for (int i = 0; i < this.vbucketsCount; i++) { vbucketsChanges += (this.getMaster(i) == config.getMaster(i)) ? 0 : 1; + for (int r = 0; r < this.replicasCount; r++) { + vbucketsChanges += this.getReplica(i, r) == config.getReplica(i, r) ? 0 : 1; + } } difference.setVbucketsChanges(vbucketsChanges); } else { diff --git a/src/test/java/com/couchbase/client/vbucket/config/CouchbaseConfigTest.java b/src/test/java/com/couchbase/client/vbucket/config/CouchbaseConfigTest.java index 3cc20e0a..68600a5b 100644 --- a/src/test/java/com/couchbase/client/vbucket/config/CouchbaseConfigTest.java +++ b/src/test/java/com/couchbase/client/vbucket/config/CouchbaseConfigTest.java @@ -22,15 +22,15 @@ package com.couchbase.client.vbucket.config; +import net.spy.memcached.DefaultHashAlgorithm; +import net.spy.memcached.HashAlgorithm; +import org.junit.Test; + import java.net.InetSocketAddress; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import net.spy.memcached.DefaultHashAlgorithm; -import org.junit.Test; - -import net.spy.memcached.HashAlgorithm; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -70,4 +70,33 @@ public void computesCachedServersWithVBuckets() throws Exception { assertFalse(config.nodeHasActiveVBuckets(new InetSocketAddress("node3", 8092))); } + @Test + public void shouldDetectChangeOnOnlyReplicaVBucket() throws Exception { + List servers = Arrays.asList("node1", "node2"); + List couchServers = Arrays.asList(new URL("http://node1:8092/"), + new URL("http://node2:8092/")); + List endpoints = Arrays.asList("http://node1:8091/pools", + "http://node2:8091/pools"); + + final int numVBuckets = 32; + List vbucketsOld = new ArrayList(); + for (int i = 0; i < numVBuckets; i++) { + vbucketsOld.add(new VBucket((short)(i % 2))); + } + List vbucketsNew = new ArrayList(); + for (int i = 0; i < numVBuckets; i++) { + vbucketsNew.add(new VBucket((short)(i % 2), (short) 1)); + } + + CouchbaseConfig oldConfig = new CouchbaseConfig( + hashAlgorithm, 2, 1, numVBuckets, servers, vbucketsOld, couchServers, + endpoints, false); + CouchbaseConfig newConfig = new CouchbaseConfig( + hashAlgorithm, 2, 1, numVBuckets, servers, vbucketsNew, couchServers, + endpoints, false); + + ConfigDifference difference = oldConfig.compareTo(newConfig); + assertTrue(difference.getVbucketsChanges() > 0); + } + } From a059bf9431dd67b68930e50418274339db4293c9 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 12 May 2014 11:33:39 +0200 Subject: [PATCH 05/47] JCBC-459: Ignore Configurations with old (or same) revID. Motivation ---------- If carrier config is used, new configuration arrives in the payload of the NMVB responses. Since configurations trickle through the server cluster not 100% at the same time, it can happen that we get a outdated configuration from one of the nodes, where the client already is in the posession of a newer one. The server emits a revid with a monotonically increasing number, so if we have a higher revid we can skip/discard incoming older ones and keep going. By not ignoring them, the client may run into a situation where it updates a older config in place for a newer one. Modifications ------------- The revnr is parsed out of the configuration, but if no id is set (for backwards compatibility reasons), -1 is used. In the provider itself, if the number is > 0 it is checked against the current known revnr and if higher the new config is used and the subscribers are notified. Result ------ Older configurations will now not be used anymore if a newer version is already in the posession of the SDK. Change-Id: I0eb5af448e8149017962030de6fe054358bd67ea Reviewed-on: http://review.couchbase.org/36981 Reviewed-by: Matt Ingenthron Reviewed-by: Jeffry Morris Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../client/vbucket/config/Bucket.java | 8 +- .../config/ConfigurationParserJSON.java | 14 ++-- .../provider/BucketConfigurationProvider.java | 10 ++- .../ConfigurationProviderMemcacheMock.java | 5 +- .../config/ConfigurationParserMock.java | 9 ++- .../BucketConfigurationProviderTest.java | 79 ++++++++++++++++++- 6 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/couchbase/client/vbucket/config/Bucket.java b/src/main/java/com/couchbase/client/vbucket/config/Bucket.java index 3706ed7a..b7ce4775 100644 --- a/src/main/java/com/couchbase/client/vbucket/config/Bucket.java +++ b/src/main/java/com/couchbase/client/vbucket/config/Bucket.java @@ -42,14 +42,16 @@ public class Bucket { // nodes list private final List nodes; + private final long revision; public Bucket(String name, Config configuration, URI streamingURI, - List nodes) { + List nodes, long revision) { this.name = name; this.configuration = configuration; this.streamingURI = streamingURI; this.nodes = nodes; this.isNotUpdating = false; + this.revision = revision; } public String getName() { @@ -64,6 +66,10 @@ public URI getStreamingURI() { return streamingURI; } + public long getRevision() { + return revision; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/com/couchbase/client/vbucket/config/ConfigurationParserJSON.java b/src/main/java/com/couchbase/client/vbucket/config/ConfigurationParserJSON.java index b223264f..a66e0dbc 100644 --- a/src/main/java/com/couchbase/client/vbucket/config/ConfigurationParserJSON.java +++ b/src/main/java/com/couchbase/client/vbucket/config/ConfigurationParserJSON.java @@ -23,6 +23,11 @@ package com.couchbase.client.vbucket.config; import com.couchbase.client.vbucket.ConnectionException; +import net.spy.memcached.compat.SpyObject; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + import java.net.URI; import java.net.URISyntaxException; import java.text.ParseException; @@ -31,12 +36,6 @@ import java.util.List; import java.util.Map; -import net.spy.memcached.compat.SpyObject; - -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - /** * This {@link ConfigurationParser} takes JSON-based configuration information * and transforms it into a {@link Bucket}. @@ -184,6 +183,7 @@ private Bucket parseBucketFromJSON(JSONObject bucketJson, Bucket current) try { String bucketName = bucketJson.getString("name"); URI streamingUri = new URI(bucketJson.getString("streamingUri")); + long revision = bucketJson.has("rev") ? bucketJson.getLong("rev") : -1; Config currentConfig = null; if (current != null) { currentConfig = current.getConfig(); @@ -202,7 +202,7 @@ private Bucket parseBucketFromJSON(JSONObject bucketJson, Bucket current) currentNode.getJSONObject("ports")); nodes.add(new Node(status, hostname, ports)); } - return new Bucket(bucketName, config, streamingUri, nodes); + return new Bucket(bucketName, config, streamingUri, nodes, revision); } catch (JSONException e) { throw new ParseException(e.getMessage(), 0); } catch (URISyntaxException e) { diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index 9cdf8eee..2fd70e2d 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -50,7 +50,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -84,6 +83,7 @@ public class BucketConfigurationProvider extends SpyObject private final boolean disableCarrierBootstrap; private final boolean disableHttpBootstrap; private volatile boolean isBinary; + private volatile long lastRevision; public BucketConfigurationProvider(final List seedNodes, final String bucket, final String password, @@ -356,6 +356,14 @@ public void setConfig(final Bucket config) { signalOutdated(); return; } + long configRevision = config.getRevision(); + if (configRevision > 0) { + if (configRevision > lastRevision) { + lastRevision = configRevision; + } else { + return; + } + } getLogger().debug("Applying new bucket config for bucket \"" + bucket + "\" (carrier publication: " + isBinary + "): " + config); diff --git a/src/test/java/com/couchbase/client/vbucket/ConfigurationProviderMemcacheMock.java b/src/test/java/com/couchbase/client/vbucket/ConfigurationProviderMemcacheMock.java index 996d516a..ebfdb6c7 100644 --- a/src/test/java/com/couchbase/client/vbucket/ConfigurationProviderMemcacheMock.java +++ b/src/test/java/com/couchbase/client/vbucket/ConfigurationProviderMemcacheMock.java @@ -25,11 +25,12 @@ import com.couchbase.client.vbucket.config.Bucket; import com.couchbase.client.vbucket.config.MemcacheConfig; import com.couchbase.client.vbucket.config.Node; +import net.spy.memcached.TestConfig; + import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import net.spy.memcached.TestConfig; /** * Implements a stub configuration provider for testing memcache buckets. @@ -66,7 +67,7 @@ public Bucket getConfig() { List nodes = new ArrayList(); - return new Bucket(bucket, config, streamingURI, nodes); + return new Bucket(bucket, config, streamingURI, nodes, -1); } @Override diff --git a/src/test/java/com/couchbase/client/vbucket/config/ConfigurationParserMock.java b/src/test/java/com/couchbase/client/vbucket/config/ConfigurationParserMock.java index 0b7e6984..55eee5de 100644 --- a/src/test/java/com/couchbase/client/vbucket/config/ConfigurationParserMock.java +++ b/src/test/java/com/couchbase/client/vbucket/config/ConfigurationParserMock.java @@ -23,6 +23,8 @@ package com.couchbase.client.vbucket.config; +import net.spy.memcached.DefaultHashAlgorithm; + import java.net.URI; import java.net.URISyntaxException; import java.text.ParseException; @@ -32,8 +34,6 @@ import java.util.List; import java.util.Map; -import net.spy.memcached.DefaultHashAlgorithm; - /** * A ConfigurationParserMock. */ @@ -73,7 +73,8 @@ public Map parseBuckets(String buckets) try { parseBucketsCalled = true; Bucket bucket = - new Bucket(bucketName, vbuckets, new URI(bucketStreamingUri), nodes); + new Bucket(bucketName, vbuckets, new URI(bucketStreamingUri), nodes, + -1); result.put(bucketName, bucket); } catch (URISyntaxException e) { throw new ParseException(e.getMessage(), 0); @@ -86,7 +87,7 @@ public Bucket parseBucket(String sBucket) throws ParseException { try { parseBucketsCalled = true; return new Bucket(bucketName, vbuckets, new URI(bucketStreamingUri), - nodes); + nodes, -1); } catch (URISyntaxException e) { throw new ParseException(e.getMessage(), 0); } diff --git a/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java b/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java index 4cd15f1b..506440c6 100644 --- a/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java +++ b/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java @@ -25,8 +25,8 @@ import com.couchbase.client.CbTestConfig; import com.couchbase.client.CouchbaseClient; import com.couchbase.client.CouchbaseConnectionFactory; -import com.couchbase.client.CouchbaseProperties; import com.couchbase.client.vbucket.ConfigurationException; +import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.config.Bucket; import net.spy.memcached.TestConfig; import net.spy.memcached.compat.log.Logger; @@ -34,15 +34,19 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.internal.util.MockUtil; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class BucketConfigurationProviderTest { @@ -235,6 +239,79 @@ public void shouldSkipHttpOnManualDisable() throws Exception { assertFalse(provider.bootstrapHttp()); } + @Test + public void shouldIgnoreOutdatedOrCurrentConfig() throws Exception { + BucketConfigurationProvider provider = new BucketConfigurationProvider( + seedNodes, + bucket, + password, + new CouchbaseConnectionFactory(seedNodes, bucket, password) + ); + + final AtomicInteger configArrived = new AtomicInteger(); + provider.subscribe(new Reconfigurable() { + @Override + public void reconfigure(Bucket bucket) { + if (new MockUtil().isMock(bucket)) { + configArrived.incrementAndGet(); + } + } + }); + + Bucket storedBucket = provider.getConfig(); + assertTrue(storedBucket.getRevision() > 0); + + + Bucket oldBucket = mock(Bucket.class); + when(oldBucket.getName()).thenReturn(bucket); + when(oldBucket.getRevision()).thenReturn(storedBucket.getRevision() - 1); + Bucket currentBucket = mock(Bucket.class); + when(currentBucket.getName()).thenReturn(bucket); + when(currentBucket.getRevision()).thenReturn(storedBucket.getRevision()); + Bucket newBucket = mock(Bucket.class); + when(newBucket.getName()).thenReturn(bucket); + when(newBucket.getRevision()).thenReturn(storedBucket.getRevision() + 1); + when(newBucket.getConfig()).thenReturn(storedBucket.getConfig()); + + provider.setConfig(oldBucket); + provider.setConfig(currentBucket); + provider.setConfig(newBucket); + + assertEquals(1, configArrived.get()); + } + + @Test + public void shouldUseConfigIfRevNotSet() throws Exception { + BucketConfigurationProvider provider = new BucketConfigurationProvider( + seedNodes, + bucket, + password, + new CouchbaseConnectionFactory(seedNodes, bucket, password) + ); + + final AtomicInteger configArrived = new AtomicInteger(); + provider.subscribe(new Reconfigurable() { + @Override + public void reconfigure(Bucket bucket) { + if (new MockUtil().isMock(bucket)) { + configArrived.incrementAndGet(); + } + } + }); + + Bucket storedBucket = provider.getConfig(); + assertTrue(storedBucket.getRevision() > 0); + + Bucket newBucket = mock(Bucket.class); + when(newBucket.getName()).thenReturn(bucket); + when(newBucket.getRevision()).thenReturn(-1L); + when(newBucket.getConfig()).thenReturn(storedBucket.getConfig()); + + provider.setConfig(newBucket); + + assertEquals(1, configArrived.get()); + } + /** * A provider that can fail either one or both of the bootstrap mechanisms. */ From 35856598be74a8386d498e12fbaebe69e7741afe Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 2 Jun 2014 09:25:56 +0200 Subject: [PATCH 06/47] JCBC-413: Fix race condition in ClusterManager. Motivation ---------- When the ClusterManager response returns with a successful response, it could be that the code waiting on the result is notified before the corresponding results are stored. Modifications ------------- The latch is now counted down after the results have been gathered, avoidin the race condition. Result ------ The code afterwards waiting on the latch is now only counted down before the results have been gathered. Change-Id: I97f0f0eb72e74a6400fecd5d03e8d35e537b4339 Reviewed-on: http://review.couchbase.org/37719 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- src/main/java/com/couchbase/client/ClusterManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/couchbase/client/ClusterManager.java b/src/main/java/com/couchbase/client/ClusterManager.java index 21820a32..be6540bf 100644 --- a/src/main/java/com/couchbase/client/ClusterManager.java +++ b/src/main/java/com/couchbase/client/ClusterManager.java @@ -471,9 +471,9 @@ private HttpResult sendRequest(final HttpRequest request) { new FutureCallback() { @Override public void completed(final HttpResponse result) { - latch.countDown(); success.set(true); response.set(result); + latch.countDown(); } @Override From d6dde6ee101efa829d016e62a2ad5f7f84e93822 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 2 Jun 2014 07:02:53 +0200 Subject: [PATCH 07/47] JCBC-359, JCBC-408, JCBC-409, : Multiple Javadoc enhancements. Motivation ---------- Several javadoc enhancements have been reported where API parts need clarifications and more information. Modifications ------------- ComplexKey, Builder and replica get docs have been enhanced, hopefully clarifying their usage and behavior. Result ------ Better docs! Change-Id: I7008bfc9b8d5722447a58a4cd415a4caaafb5906 Reviewed-on: http://review.couchbase.org/37718 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- .../couchbase/client/CouchbaseClientIF.java | 262 +++++++---- .../CouchbaseConnectionFactoryBuilder.java | 431 ++++++++++++++---- .../client/protocol/views/ComplexKey.java | 13 +- 3 files changed, 507 insertions(+), 199 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseClientIF.java b/src/main/java/com/couchbase/client/CouchbaseClientIF.java index 9c33dd96..f23656fd 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClientIF.java +++ b/src/main/java/com/couchbase/client/CouchbaseClientIF.java @@ -31,10 +31,6 @@ import com.couchbase.client.protocol.views.SpatialView; import com.couchbase.client.protocol.views.View; import com.couchbase.client.protocol.views.ViewResponse; -import java.io.UnsupportedEncodingException; -import java.util.Map; -import java.util.concurrent.Future; - import net.spy.memcached.CASResponse; import net.spy.memcached.CASValue; import net.spy.memcached.MemcachedClientIF; @@ -45,6 +41,10 @@ import net.spy.memcached.internal.OperationFuture; import net.spy.memcached.transcoders.Transcoder; +import java.io.UnsupportedEncodingException; +import java.util.Map; +import java.util.concurrent.Future; + /** * This interface is provided as a helper for testing clients of the * CouchbaseClient. @@ -62,7 +62,7 @@ public interface CouchbaseClientIF extends MemcachedClientIF { * @throws IllegalStateException in the rare circumstance where queue is too * full to accept any more requests */ - Future> asyncGetAndLock(final String key, int exp); + Future> asyncGetAndLock(String key, int exp); /** * Gets and locks the given key asynchronously. By default the maximum allowed @@ -76,8 +76,8 @@ public interface CouchbaseClientIF extends MemcachedClientIF { * @throws IllegalStateException in the rare circumstance where queue is too * full to accept any more requests */ - Future> asyncGetAndLock(final String key, int exp, - final Transcoder tc); + Future> asyncGetAndLock(String key, int exp, + Transcoder tc); /** * Getl with a single key. By default the maximum allowed timeout is 30 @@ -120,8 +120,8 @@ Future> asyncGetAndLock(final String key, int exp, * @throws IllegalStateException in the rare circumstance where queue is too * full to accept any more requests */ - OperationFuture asyncUnlock(final String key, - long casId, final Transcoder tc); + OperationFuture asyncUnlock(String key, + long casId, Transcoder tc); /** * Unlock the given key asynchronously from the cache with the default @@ -133,7 +133,7 @@ OperationFuture asyncUnlock(final String key, * @throws IllegalStateException in the rare circumstance where queue is too * full to accept any more requests */ - OperationFuture asyncUnlock(final String key, + OperationFuture asyncUnlock(String key, long casId); /** @@ -147,8 +147,8 @@ OperationFuture asyncUnlock(final String key, * full to accept any more requests * @throws java.util.concurrent.CancellationException if operation was canceled */ - Boolean unlock(final String key, - long casId, final Transcoder tc); + Boolean unlock(String key, + long casId, Transcoder tc); /** * Unlock the given key synchronously from the cache with the default @@ -160,7 +160,7 @@ Boolean unlock(final String key, * @throws IllegalStateException in the rare circumstance where queue is too * full to accept any more requests */ - Boolean unlock(final String key, long casId); + Boolean unlock(String key, long casId); /** * Observe a key with a associated CAS. @@ -176,7 +176,7 @@ Boolean unlock(final String key, * @throws IllegalStateException in the rare circumstance where queue is too * full to accept any more requests. */ - Map observe(final String key, long cas); + Map observe(String key, long cas); /** @@ -1115,7 +1115,7 @@ OperationFuture delete(String key, PersistTo req, * @param doc the design document to store. * @return a future containing the result of the creation operation. */ - HttpFuture asyncCreateDesignDoc(final DesignDocument doc) + HttpFuture asyncCreateDesignDoc(DesignDocument doc) throws UnsupportedEncodingException; /** @@ -1134,7 +1134,7 @@ HttpFuture asyncCreateDesignDoc(String name, String value) * @param name the design document to delete. * @return a future containing the result of the deletion operation. */ - HttpFuture asyncDeleteDesignDoc(final String name) + HttpFuture asyncDeleteDesignDoc(String name) throws UnsupportedEncodingException; /** @@ -1169,7 +1169,7 @@ HttpFuture asyncDeleteDesignDoc(final String name) * @return the result of the creation operation. * @throws java.util.concurrent.CancellationException if operation was canceled. */ - Boolean createDesignDoc(final DesignDocument doc); + Boolean createDesignDoc(DesignDocument doc); /** * Delete a design document in the cluster. @@ -1178,7 +1178,7 @@ HttpFuture asyncDeleteDesignDoc(final String name) * @return the result of the deletion operation. * @throws java.util.concurrent.CancellationException if operation was canceled. */ - Boolean deleteDesignDoc(final String name); + Boolean deleteDesignDoc(String name); /** * Returns a representation of a design document stored in the cluster. @@ -1191,7 +1191,7 @@ HttpFuture asyncDeleteDesignDoc(final String name) * @throws java.util.concurrent.CancellationException if operation was canceled. */ @Deprecated - DesignDocument getDesignDocument(final String designDocumentName); + DesignDocument getDesignDocument(String designDocumentName); /** * Returns a representation of a design document stored in the cluster. @@ -1201,133 +1201,197 @@ HttpFuture asyncDeleteDesignDoc(final String name) * @throws com.couchbase.client.protocol.views.InvalidViewException if no design document or view was found. * @throws java.util.concurrent.CancellationException if operation was canceled. */ - DesignDocument getDesignDoc(final String designDocumentName); + DesignDocument getDesignDoc(String designDocumentName); /** - * Get a document from a replica node. + * Get a document from the replica (or the active) nodes. + * + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

* - * This method allows you to explicitly load a document from a replica - * instead of the master node. + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * This command only works on couchbase type buckets. + *

Replicas are only supported on couchbase buckets.

* - * @param key the key to fetch. - * @return the fetched document or null when no document available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + * @param key the unique key of the document. + * @return the fetched document or null when no document found. */ Object getFromReplica(String key); /** - * Get a document from a replica node including its CAS value. + * Get a document and its CAS from the replica (or the active) nodes. * - * This method allows you to explicitly load a document from a replica - * instead of the master node including its CAS value. + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

* - * This command only works on couchbase type buckets. + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * @param key the key to fetch. - * @return the fetched document or null when no document available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + *

Replicas are only supported on couchbase buckets.

+ * + * @param key the unique key of the document. + * @return the fetched document or null when no document found. */ CASValue getsFromReplica(String key); /** - * Get a document from a replica node. + * Get a document from the replica (or the active) nodes and use a custom + * transcoder. * - * This method allows you to explicitly load a document from a replica - * instead from the master node. + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

* - * This command only works on couchbase type buckets. + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * @param key the key to fetch. - * @param tc a custom document transcoder. - * @return the fetched document or null when no document available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + *

Replicas are only supported on couchbase buckets.

+ * + * @param key the unique key of the document. + * @param tc a custom transcoder. + * @return the fetched document or null when no document found. */ T getFromReplica(String key, Transcoder tc); /** - * Get a document from a replica node including its CAS value. + * Get a document and its CAS from the replica (or the active) nodes and use + * a custom transcoder. + * + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

* - * This method allows you to explicitly load a document from a replica - * instead of the master node including its CAS value. + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * This command only works on couchbase type buckets. + *

Replicas are only supported on couchbase buckets.

* - * @param key the key to fetch. - * @param tc a custom document transcoder. - * @return the fetched document or null when no document available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + * @param key the unique key of the document. + * @param tc a custom transcoder. + * @return the fetched document or null when no document found. */ CASValue getsFromReplica(String key, Transcoder tc); /** - * Get a document from a replica node asynchronously. + * Asynchronously get a document from the replica (or the active) nodes. + * + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

* - * This method allows you to explicitly load a document from a replica - * instead from the master node. This command only works on couchbase - * type buckets. + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * @param key the key to fetch. + *

Replicas are only supported on couchbase buckets.

+ * + * @param key the unique key of the document. * @return a future containing the fetched document or null when no document - * available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + * found. */ - ReplicaGetFuture asyncGetFromReplica(final String key); + ReplicaGetFuture asyncGetFromReplica(String key); /** - * Get a document from a replica node asynchronously and load the CAS. + * Asynchronously get a document and its CAS from the replica (or the active) + * nodes. + * + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

* - * This method allows you to explicitly load a document from a replica - * instead from the master node. This command only works on couchbase - * type buckets. + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * @param key the key to fetch. + *

Replicas are only supported on couchbase buckets.

+ * + * @param key the unique key of the document. * @return a future containing the fetched document or null when no document - * available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + * found. */ - ReplicaGetFuture> asyncGetsFromReplica(final String key); + ReplicaGetFuture> asyncGetsFromReplica(String key); /** - * Get a document from a replica node asynchronously. + * Asynchronously get a document from the replica (or the active) nodes with + * a custom transcoder. + * + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

+ * + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * This method allows you to explicitly load a document from a replica - * instead from the master node. This command only works on couchbase - * type buckets. + *

Replicas are only supported on couchbase buckets.

* - * @param key the key to fetch. - * @param tc a custom document transcoder. + * @param key the unique key of the document. + * @param tc a custom transcoder. * @return a future containing the fetched document or null when no document - * available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + * found. */ - ReplicaGetFuture asyncGetFromReplica(final String key, - final Transcoder tc); + ReplicaGetFuture asyncGetFromReplica(String key, Transcoder tc); /** - * Get a document from a replica node asynchronously and load the CAS. + * Asynchronously get a document and its CAS from the replica (or the active) + * nodes with a custom transcoder. + * + *

A replica read command "fans out" to all configured replicas and also + * the active node and asks them for the document. The response which arrives + * first at the client is used and the others are discarded. This means that + * the method should only be used as a fallback on failure, since it sends + * out potentially more than one request, leading to increased traffic if + * used regularly.

+ * + *

Important: since a response from the replica can arrive first, the code + * calling this method must always be prepared to get outdated results, since + * the data takes some time to be propagated to the replica. Use it only if + * availability is explicitly traded for consistency.

* - * This method allows you to explicitly load a document from a replica - * instead from the master node. This command only works on couchbase - * type buckets. + *

Replicas are only supported on couchbase buckets.

* - * @param key the key to fetch. - * @param tc a custom document transcoder. + * @param key the unique key of the document. + * @param tc a custom transcoder. * @return a future containing the fetched document or null when no document - * available. - * @throws RuntimeException when less replicas available then in the index - * argument defined. + * found. */ - ReplicaGetFuture> asyncGetsFromReplica(final String key, - final Transcoder tc); + ReplicaGetFuture> asyncGetsFromReplica(String key, + Transcoder tc); /** * Gets access to a view contained in a design document from the cluster. @@ -1353,7 +1417,7 @@ ReplicaGetFuture> asyncGetsFromReplica(final String key, * flight * @throws java.util.concurrent.ExecutionException if an error occurs during execution */ - HttpFuture asyncGetView(String designDocumentName, final String viewName); + HttpFuture asyncGetView(String designDocumentName, String viewName); /** * Gets access to a spatial view contained in a design document from the @@ -1373,7 +1437,7 @@ ReplicaGetFuture> asyncGetsFromReplica(final String key, * @throws java.util.concurrent.ExecutionException if an error occurs during execution */ HttpFuture asyncGetSpatialView(String designDocumentName, - final String viewName); + String viewName); HttpFuture asyncQuery(AbstractView view, Query query); @@ -1423,7 +1487,7 @@ HttpFuture asyncGetSpatialView(String designDocumentName, * @throws com.couchbase.client.protocol.views.InvalidViewException if no design document or view was found. * @throws java.util.concurrent.CancellationException if operation was canceled. */ - View getView(final String designDocumentName, final String viewName); + View getView(String designDocumentName, String viewName); /** @@ -1441,8 +1505,8 @@ HttpFuture asyncGetSpatialView(String designDocumentName, * @throws com.couchbase.client.protocol.views.InvalidViewException if no design document or view was found. * @throws java.util.concurrent.CancellationException if operation was canceled. */ - SpatialView getSpatialView(final String designDocumentName, - final String viewName); + SpatialView getSpatialView(String designDocumentName, + String viewName); OperationFuture> getKeyStats(String key); diff --git a/src/main/java/com/couchbase/client/CouchbaseConnectionFactoryBuilder.java b/src/main/java/com/couchbase/client/CouchbaseConnectionFactoryBuilder.java index 2c101063..08a8a0bc 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnectionFactoryBuilder.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnectionFactoryBuilder.java @@ -24,20 +24,8 @@ import com.couchbase.client.vbucket.CouchbaseNodeOrder; import com.couchbase.client.vbucket.config.Config; - -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - import net.spy.memcached.ConnectionFactoryBuilder; import net.spy.memcached.ConnectionObserver; -import net.spy.memcached.DefaultConnectionFactory; import net.spy.memcached.FailureMode; import net.spy.memcached.HashAlgorithm; import net.spy.memcached.OperationFactory; @@ -47,12 +35,50 @@ import net.spy.memcached.ops.Operation; import net.spy.memcached.transcoders.Transcoder; +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + /** - * CouchbaseConnectionFactoryBuilder. + * The {@link CouchbaseConnectionFactoryBuilder} enables the customization of + * {@link CouchbaseConnectionFactory} settings. + * + *

Some of the provides setter methods are for internal usage only, see the + * individual documentation for setter methods on this class for their impact, + * usage and defaults.

* + *

The builder should be used like this:

+ * + * 
{@code
+ * // Create the builder
+ * CouchbaseConnectionFactoryBuilder builder =
+ * new CouchbaseConnectionFactoryBuilder();
+ *
+ * // Change a setting
+ * builder.setOpTimeout(3000);
+ *
+ * // Build the factory and use it
+ * List nodes = Arrays.asList(URI.create("..."));
+ * CouchbaseClient client = new CouchbaseClient(
+ *   builder.buildCouchbaseConnection(nodes, "bucket", "password")
+ * );
+ * }
*/ +public class CouchbaseConnectionFactoryBuilder + extends ConnectionFactoryBuilder { -public class CouchbaseConnectionFactoryBuilder extends ConnectionFactoryBuilder { + /** + * The logger used for the builder. + */ + private static final Logger LOGGER = Logger.getLogger( + CouchbaseConnectionFactoryBuilder.class.getName() + ); private Config vBucketConfig; private long reconnThresholdTimeMsecs = @@ -69,65 +95,128 @@ public class CouchbaseConnectionFactoryBuilder extends ConnectionFactoryBuilder private CouchbaseNodeOrder nodeOrder = CouchbaseConnectionFactory.DEFAULT_STREAMING_NODE_ORDER; - private static final Logger LOGGER = - Logger.getLogger(CouchbaseConnectionFactoryBuilder.class.getName()); - protected MetricType metricType = null; - protected MetricCollector collector = null; - protected ExecutorService executorService = null; - protected long authWaitTime = -1; - public Config getVBucketConfig() { - return vBucketConfig; - } + protected MetricType metricType; + protected MetricCollector collector; + protected ExecutorService executorService; + protected long authWaitTime = -1; - public void setVBucketConfig(Config config) { - this.vBucketConfig = config; + /** + * Sets a custom {@link Config} to use. + * + *

Note that this method is only used for internal purposes and should not + * be used by regular applications. Use with care!

+ * + * @param config the configuration to use. + * @return the builder instance for dsl-like chaining functionality. + */ + public CouchbaseConnectionFactoryBuilder setVBucketConfig(final Config config) { + vBucketConfig = config; + return this; } - public void setReconnectThresholdTime(long time, TimeUnit unit) { + /** + * Sets a custom reconnect threshold. + * + *

If operations are failing for a period of time, the attached streaming + * connection for configuration updates is considered invalid and the client + * tries to establish a new connection by reconnecting. This threshold sets + * the time between those reconnect attempts to not overwhelm the server if + * it is already in a potentially unstable state.

+ * + *

Defaults to: + * {@link CouchbaseConnectionFactory#DEFAULT_MIN_RECONNECT_INTERVAL}

+ * + * @param time the time for the threshold. + * @param unit the unit for the time. + * @return the builder instance for dsl-like chaining functionality. + */ + public CouchbaseConnectionFactoryBuilder setReconnectThresholdTime( + final long time, final TimeUnit unit) { reconnThresholdTimeMsecs = TimeUnit.MILLISECONDS.convert(time, unit); + return this; } /** - * Set the interval between observe polls in milliseconds. + * Sets a custom interval between observe poll cycles. + * + *

Every time a mutation operation with constraints (like + * {@link CouchbaseClient#set(String, Object, net.spy.memcached.PersistTo, + * net.spy.memcached.ReplicateTo)}) is used, internally a poller observes the + * server state until the required constraints can be met. This configuration + * setting tunes this poll interval. If you expect faster replication or + * persistence timings on the server than provided by the default interval, + * set it to a lower value. If this value gets adjusted, it might make sense + * to also adjust {@link #setObsTimeout(long)}.

+ * + *

Defaults to: + * {@link CouchbaseConnectionFactory#DEFAULT_OBS_POLL_INTERVAL}

* * @param interval the interval in milliseconds. - * @return the builder for proper chaining. + * @return the builder instance for dsl-like chaining functionality. */ - public CouchbaseConnectionFactoryBuilder setObsPollInterval(long interval) { + public CouchbaseConnectionFactoryBuilder setObsPollInterval( + final long interval) { obsPollInterval = interval; return this; } /** - * Set the timeout for observe-based operations in milliseconds. + * Sets the absolute per-operation timeout for observe calls. * - * This timeout is always used when PersistTo or ReplicateTo overloaded - * methods are used, instead of the default operation timeout. + *

This is the total timeout that the observe poller spends polling for + * the desired constraint specified through a call like + * {@link CouchbaseClient#set(String, Object, net.spy.memcached.PersistTo, + * net.spy.memcached.ReplicateTo)}. Together with + * {@link #setObsPollInterval(long)}, both settings provide the tuning knobs + * for observe-related behavior.

+ * + *

Defaults to: + * {@link CouchbaseConnectionFactory#DEFAULT_OBS_TIMEOUT}.

* * @param timeout the timeout in milliseconds. - * @return the builder for proper chaining. + * @return the builder instance for dsl-like chaining functionality. */ - public CouchbaseConnectionFactoryBuilder setObsTimeout(long timeout) { + public CouchbaseConnectionFactoryBuilder setObsTimeout(final long timeout) { obsTimeout = timeout; return this; } /** - * Sets the maximum number of observe polls. + * Sets the maximum number of observe poll cycles for observe calls. * - * Do not use this method directly, but instead use a combination of - * {@link #setObsPollInterval(long)} and {@link #setObsTimeout(long)}. + *

This method has been deprecated. Do not use this method directly, but + * instead use a combination of {@link #setObsPollInterval(long)} and + * {@link #setObsTimeout(long)} to achieve the same effect. This setting + * is ignored and has no impact on client behavior.

* * @param maxPoll the maximum number of polls to run before giving up. - * @return the builder for proper chaining. + * @return the builder instance for dsl-like chaining functionality. */ @Deprecated - public CouchbaseConnectionFactoryBuilder setObsPollMax(int maxPoll) { + public CouchbaseConnectionFactoryBuilder setObsPollMax(final int maxPoll) { obsPollMax = maxPoll; return this; } + /** + * Sets a custom timeout for view operations. + * + *

A custom timeout in miliseconds can be specified to overrule the default + * timeout. If tuning this value significantly lower, it should be taken into + * consideration that view results take eventually longer to return than their + * key-based counterparts.

+ * + *

If a timeout lower than 500ms is applied, it will be capped to 500ms as + * a precaution and a warning will be issued. If it is lower than 2.5 seconds, + * a warning is also issued to notify a potential configuration issue.

+ * + *

Defaults to: + * {@link CouchbaseConnectionFactory#DEFAULT_VIEW_TIMEOUT}.

+ * + * @param timeout the timeout in miliseconds. + * @return the builder instance for dsl-like chaining functionality. + */ public CouchbaseConnectionFactoryBuilder setViewTimeout(int timeout) { if(timeout < 500) { timeout = 500; @@ -141,6 +230,21 @@ public CouchbaseConnectionFactoryBuilder setViewTimeout(int timeout) { return this; } + /** + * Sets a custom worker count for view operations. + * + *

View requests are dispatched to asynchronous workers. This setting + * normally only needs to be tuned if there is a very high view workload + * and there is a suspicion that the default worker size is the bottleneck + * and cannot handle the needed traffic. If increased, also consider + * increasing the maximum connections per node.

+ * + *

Defaults to + * {@link CouchbaseConnectionFactory#DEFAULT_VIEW_WORKER_SIZE}.

+ * + * @param workers the number of workers. + * @return the builder instance for dsl-like chaining functionality. + */ public CouchbaseConnectionFactoryBuilder setViewWorkerSize(int workers) { if (workers < 1) { throw new IllegalArgumentException("The View worker size needs to be " @@ -151,6 +255,20 @@ public CouchbaseConnectionFactoryBuilder setViewWorkerSize(int workers) { return this; } + /** + * Changes the maximum parallel open connections per node for view operations. + * + *

View connections are openend on demand to each node, so with increasing + * traffic more and more parallel connections are opened to satisfy the + * workload needs. This setting allows to change the upper limit of open + * connections per node.

+ * + *

Defaults to + * {@link CouchbaseConnectionFactory#DEFAULT_VIEW_CONNS_PER_NODE}.

+ * + * @param conns maximum parallel open connections per node. + * @return the builder instance for dsl-like chaining functionality. + */ public CouchbaseConnectionFactoryBuilder setViewConnsPerNode(int conns) { if (conns < 1) { throw new IllegalArgumentException("The View connections per node need " @@ -161,111 +279,144 @@ public CouchbaseConnectionFactoryBuilder setViewConnsPerNode(int conns) { } /** - * Set the streaming connection node ordering. + * Changes the node ordering for streaming connections. + * + *

To prevent the case where all streaming connections are opened on the + * first node in the bootstrap list (and potentially overwhelm the server), + * the default ordering mixes the nodes before attaching the connection. This + * provides better distribution of the streaming connection. If old behavior + * should be forced or there is a specific reason to fallback to the same + * ordering, the setting can be changed.

+ * + *

Defaults to + * {@link CouchbaseConnectionFactory#DEFAULT_STREAMING_NODE_ORDER}.

* * @param order the ordering to use. - * @return the builder for chaining. + * @return the builder instance for dsl-like chaining functionality. */ - public CouchbaseConnectionFactoryBuilder setStreamingNodeOrder(CouchbaseNodeOrder order) { + public CouchbaseConnectionFactoryBuilder setStreamingNodeOrder( + final CouchbaseNodeOrder order) { nodeOrder = order; return this; } /** - * Enable or disable metric collection. + * Enable the collection of runtime metrics. + * + *

Runtime metrics collection can be enabled to provide better insight + * to what is happening inside the client library. Since the collection of + * metrics comes with a performance penalty, it is disable by default and can + * be enabled to a variety of different metric collection types. The debug + * mode subsumes the performance mode and provides the deepest insight.

+ * + *

Defaults to {@link CouchbaseConnectionFactory#DEFAULT_METRIC_TYPE}.

* * @param type the metric type to use (or disable). + * @return the builder instance for dsl-like chaining functionality. */ @Override - public ConnectionFactoryBuilder setEnableMetrics(MetricType type) { + public ConnectionFactoryBuilder setEnableMetrics(final MetricType type) { metricType = type; return this; } /** - * Set a custom {@link MetricCollector}. + * Allows to provide a custom {@link MetricCollector}. + * + *

If metrics should be collected in a different way, a custom collector + * can be passed in which will then receive the metrics exposed by the client + * library. This is considered advanced functionality and should normally + * not be overriden.

* * @param collector the metric collector to use. + * @return the builder instance for dsl-like chaining functionality. */ @Override - public ConnectionFactoryBuilder setMetricCollector(MetricCollector collector) { + public ConnectionFactoryBuilder setMetricCollector( + final MetricCollector collector) { this.collector = collector; return this; } /** - * Set a custom {@link ExecutorService} to execute the listener callbacks. + * Set a custom {@link ExecutorService} to execute the listener callbacks and + * other callback-type operations. + * + *

If the executor is created inside the client library, it is also shut + * down automatically. If a custom one gets passed in, it is the + * responsibility of the caller to shut it down since the client library does + * not know who else is depending on the executor.

+ * + *

Passing in a custom executor service is handy if it should be shared + * across many {@link CouchbaseClient} instances or if the application already + * uses a thread pool for long running tasks and it can be shared with the + * client library.

+ * + *

Defaults to a thread pool executor which can scale up to the number of + * configured CPUs as exposed by the {@link Runtime#availableProcessors()} + * setting.

* * @param executorService the ExecutorService to use. + * @return the builder instance for dsl-like chaining functionality. */ @Override - public ConnectionFactoryBuilder setListenerExecutorService(ExecutorService executorService) { + public ConnectionFactoryBuilder setListenerExecutorService( + final ExecutorService executorService) { this.executorService = executorService; return this; } + /** + * Sets a custom waiting time until authentication completes. + * + *

This setting does not need to be changed normally, but if the client + * logs indicate that authentication to server nodes takes longer than + * expected, it might pay off to increase the wait time. That said, in general + * this indicates some odd behavior and should be looked into separately.

+ * + *

Defaults to + * {@link CouchbaseConnectionFactory#DEFAULT_AUTH_WAIT_TIME}.

+ * + * @param authWaitTime the wait time in miliseconds. + * @return the builder instance for dsl-like chaining functionality. + */ @Override - public ConnectionFactoryBuilder setAuthWaitTime(long authWaitTime) { + public ConnectionFactoryBuilder setAuthWaitTime(final long authWaitTime) { this.authWaitTime = authWaitTime; return this; } /** - * Get the CouchbaseConnectionFactory set up with the provided parameters. - * Note that a CouchbaseConnectionFactory requires the failure mode is set - * to retry, and the locator type is discovered dynamically based on the - * cluster you are connecting to. As a result, these values will be - * overridden upon calling this function. - * - * @param baseList a list of URI's that will be used to connect to the cluster - * @param bucketName the name of the bucket to connect to, also used for - * username - * @param pwd the password for the bucket - * @return a CouchbaseConnectionFactory object - * @throws IOException + * Build the {@link CouchbaseConnectionFactory} set up with the provided + * settings. + * + * @param baseList list of seed nodes. + * @param bucketName the name of the bucket. + * @param pwd the password of the bucket. + * @return a {@link CouchbaseConnectionFactory}. + * @throws IOException if the connection could not be established. */ public CouchbaseConnectionFactory buildCouchbaseConnection( - final List baseList, final String bucketName, final String pwd) + final List baseList, final String bucketName, final String pwd) throws IOException { - return this.buildCouchbaseConnection(baseList, bucketName, bucketName, pwd); - } - - - /** - * Get the CouchbaseConnectionFactory set up with the provided parameters. - * Note that a CouchbaseConnectionFactory requires the failure mode is set - * to retry, and the locator type is discovered dynamically based on the - * cluster you are connecting to. As a result, these values will be - * overridden upon calling this function. - * - * @param baseList a list of URI's that will be used to connect to the cluster - * @param bucketName the name of the bucket to connect to - * @param usr the username for the bucket - * @param pwd the password for the bucket - * @return a CouchbaseConnectionFactory object - * @throws IOException - */ - public CouchbaseConnectionFactory buildCouchbaseConnection( - final List baseList, final String bucketName, final String usr, - final String pwd) throws IOException { return new CouchbaseConnectionFactory(baseList, bucketName, pwd) { @Override public BlockingQueue createOperationQueue() { return opQueueFactory == null ? super.createOperationQueue() - : opQueueFactory.create(); + : opQueueFactory.create(); } @Override public BlockingQueue createReadOperationQueue() { return readQueueFactory == null ? super.createReadOperationQueue() - : readQueueFactory.create(); + : readQueueFactory.create(); } @Override public BlockingQueue createWriteOperationQueue() { return writeQueueFactory == null ? super.createReadOperationQueue() - : writeQueueFactory.create(); + : writeQueueFactory.create(); } @Override @@ -331,7 +482,7 @@ public long getMaxReconnectDelay() { @Override public long getOpQueueMaxBlockTime() { return opQueueMaxBlockTime > -1 ? opQueueMaxBlockTime - : super.getOpQueueMaxBlockTime(); + : super.getOpQueueMaxBlockTime(); } @Override @@ -339,6 +490,7 @@ public int getTimeoutExceptionThreshold() { return timeoutExceptionThreshold; } + @Override public long getMinReconnectInterval() { return reconnThresholdTimeMsecs; } @@ -368,6 +520,11 @@ public int getViewConnsPerNode() { return viewConns; } + @Override + public CouchbaseNodeOrder getStreamingNodeOrder() { + return nodeOrder; + } + @Override public MetricType enableMetrics() { return metricType == null ? super.enableMetrics() : metricType; @@ -392,15 +549,48 @@ public boolean isDefaultExecutorService() { public AuthDescriptor getAuthDescriptor() { return authDescriptor == null ? super.getAuthDescriptor() : authDescriptor; } - }; + }; } + + + /** + * Build the {@link CouchbaseConnectionFactory} set up with the provided + * settings. + * + *

This method is deprecated since the username does not need to be + * provided - only the name of the bucket. Use the + * {@link #buildCouchbaseConnection(java.util.List, String, String)} method + * instead.

+ * + * @param baseList list of seed nodes. + * @param bucketName the name of the bucket. + * @param usr the username. + * @param pwd the password of the bucket. + * @return a {@link CouchbaseConnectionFactory}. + * @throws IOException if the connection could not be established. + */ + @Deprecated + public CouchbaseConnectionFactory buildCouchbaseConnection( + final List baseList, final String bucketName, final String usr, + final String pwd) throws IOException { + return buildCouchbaseConnection(baseList, bucketName, pwd); } /** - * Get the CouchbaseConnectionFactory set up with parameters provided by - * system properties. + * Build the {@link CouchbaseConnectionFactory} set up with the provided + * settings. + * + *

If this method is used, the seed nodes, bucket name and password are + * picked from system properties instead of the actual method params. They + * are:

+ * + *

The following properties need to be set in order to bootstrap: + * - cbclient.nodes: ;-separated list of URIs + * - cbclient.bucket: name of the bucket + * - cbclient.password: password of the bucket + *

* - * @return the created factory. - * @throws IOException + * @return a {@link CouchbaseConnectionFactory}. + * @throws IOException if the connection could not be established. */ public CouchbaseConnectionFactory buildCouchbaseConnection() throws IOException { return new CouchbaseConnectionFactory() { @@ -494,10 +684,16 @@ public int getTimeoutExceptionThreshold() { return timeoutExceptionThreshold; } + @Override public long getMinReconnectInterval() { return reconnThresholdTimeMsecs; } + @Override + public CouchbaseNodeOrder getStreamingNodeOrder() { + return nodeOrder; + } + @Override public long getObsPollInterval() { return obsPollInterval; @@ -551,42 +747,85 @@ public AuthDescriptor getAuthDescriptor() { } /** - * @return the obsPollInterval + * Returns the currently set observe poll interval. + * + * @return the observe poll interval. */ public long getObsPollInterval() { return obsPollInterval; } /** - * @return the obsPollMax + * Returns the currently set maximum observe poll cycles. + * + * @return the poll cycles. */ + @Deprecated public int getObsPollMax() { return obsPollMax; } /** - * @return the observe timeout + * Returns the currently set observe timeout. + * + * @return the timeout setting. */ public long getObsTimeout() { return obsTimeout; } /** - * @return the viewTimeout + * Returns the currently set view timeout. + * + * @return the view timeout. */ public int getViewTimeout() { return viewTimeout; } + /** + * The currently configured number of view workers. + * + * @return view worker count. + */ public int getViewWorkerSize() { return viewWorkers; } + /** + * The currently configured number of maximum open view connections per node. + * + * @return the number of view connections. + */ public int getViewConnsPerNode() { return viewConns; } + /** + * Returns the currently configured authentication wait time. + * + * @return the auth wait time. + */ public long getAuthWaitTime() { return authWaitTime; } + + /** + * Returns a potentially set configuration. + * + *

For internal use only.

+ * @return the vbucket config. + */ + public Config getVBucketConfig() { + return vBucketConfig; + } + + /** + * Returns the currently configured streaming node order. + * + * @return the streaming node order. + */ + public CouchbaseNodeOrder getStreamingNodeOrder() { + return nodeOrder; + } } diff --git a/src/main/java/com/couchbase/client/protocol/views/ComplexKey.java b/src/main/java/com/couchbase/client/protocol/views/ComplexKey.java index 9477d43c..c9c41ffb 100644 --- a/src/main/java/com/couchbase/client/protocol/views/ComplexKey.java +++ b/src/main/java/com/couchbase/client/protocol/views/ComplexKey.java @@ -22,24 +22,28 @@ package com.couchbase.client.protocol.views; -import java.util.Arrays; -import java.util.List; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; +import java.util.Arrays; +import java.util.List; + /** * Allows simple definition of complex JSON keys for query inputs. * + *

* If you use the ComplexKex class, the stored objects ultimately get converted * into a JSON string. As a result, make sure your custom objects implement the * "toString" method accordingly (unless you work with trivial types like * Strings or numbers). - * + *

* Instead of using a constructor, use the static "of" method to generate your * objects. You can also use a special empty object or array. - * + *

* Here are some simple examples: + *

* + * 
{@code
  * // generated JSON: [2012,9,7]
  * ComplexKey.of(2012, 9, 7);
  *
@@ -51,6 +55,7 @@
  *
  * // generated JSON: []
  * ComplexKey.of(ComplexKey.emptyArray());
+ * }
* * This was inspired by the Ektorp project, which queries Apache CouchDB. */ From ed3fec9f9cff518d62deab8a119c0a37aa3f5f06 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 2 Jun 2014 10:23:12 +0200 Subject: [PATCH 08/47] JCBC-461: Limit the netty worker count for streaming conns. Motivation ---------- Under normal circumstances, netty should create worker threads on the fly and since the client only uses one channel for the bucket streaming connection, there should every only one be used. Under failure conditions, it could be the case that many more threads are created falsely. Modifications ------------- The worker count is held to one thread only, since only one channel needs ever to be accessed. Result ------ Providing a hard upper limit for worker threads prevents thread exhaustion. Change-Id: Idb285bc794c190b876e63383d1eff690ba57d9ba Reviewed-on: http://review.couchbase.org/37722 Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../client/vbucket/BucketMonitor.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/couchbase/client/vbucket/BucketMonitor.java b/src/main/java/com/couchbase/client/vbucket/BucketMonitor.java index 3aebc31e..08e2ddeb 100644 --- a/src/main/java/com/couchbase/client/vbucket/BucketMonitor.java +++ b/src/main/java/com/couchbase/client/vbucket/BucketMonitor.java @@ -25,17 +25,6 @@ import com.couchbase.client.http.HttpUtil; import com.couchbase.client.vbucket.config.Bucket; import com.couchbase.client.vbucket.config.ConfigurationParser; - -import java.io.UnsupportedEncodingException; -import java.lang.reflect.Method; -import java.net.InetSocketAddress; -import java.net.URI; -import java.text.ParseException; -import java.util.Observable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - import net.spy.memcached.compat.log.Logger; import net.spy.memcached.compat.log.LoggerFactory; import org.jboss.netty.bootstrap.ClientBootstrap; @@ -50,12 +39,31 @@ import org.jboss.netty.handler.codec.http.HttpRequest; import org.jboss.netty.handler.codec.http.HttpVersion; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Method; +import java.net.InetSocketAddress; +import java.net.URI; +import java.text.ParseException; +import java.util.Observable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + /** * The BucketMonitor will open an HTTP comet stream to monitor for changes to * the list of nodes. If the list of nodes changes, it will notify observers. */ public class BucketMonitor extends Observable { + /** + * Number of netty worker threads to start. + * + *

Since there is only one streaming connection open per + * {@link BucketMonitor}, setting it to a low value prohibits ressource usage + * in case of a failure.

+ */ + private static final int NETTY_WORKER_COUNT = 1; + private final URI cometStreamURI; private final String httpUser; private final String httpPass; @@ -98,7 +106,7 @@ public BucketMonitor(URI cometStreamURI, String username, String password, this.host = cometStreamURI.getHost(); this.port = cometStreamURI.getPort() == -1 ? 80 : cometStreamURI.getPort(); factory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(), - Executors.newCachedThreadPool()); + Executors.newCachedThreadPool(), NETTY_WORKER_COUNT); this.headers = new HttpMessageHeaders(); this.provider = provider; } From 02bea4ba7e03c521b0e864cece3c13a5c55b46c1 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 2 Jun 2014 12:52:19 +0200 Subject: [PATCH 09/47] JCBC-463: Harden shutdown procedure in CouchbaseClient Motivation ---------- Lingering bucket streaming threads are reported, this could be the case if something is preventing them from being shut down. Modifications ------------- Split up the spymemcached IO and view IO shutdown in two different try/catch blocks, to make sure the other part always runs, even if something goes wrong in the other. This change goes hand-in-hand with related shutdown hardening changes in spymemcached directly. Result ------ Shutting down of the config threads now also happens if an exception is raised during memcached IO shutdown. Change-Id: I65bbe9ab1b5c27333012268cd3aafa5d8aa528b4 Reviewed-on: http://review.couchbase.org/37726 Reviewed-by: Matt Ingenthron Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../com/couchbase/client/CouchbaseClient.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseClient.java b/src/main/java/com/couchbase/client/CouchbaseClient.java index 817f28e5..befaf582 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClient.java +++ b/src/main/java/com/couchbase/client/CouchbaseClient.java @@ -91,7 +91,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; -import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -123,6 +122,9 @@ public class CouchbaseClient extends MemcachedClient implements CouchbaseClientIF, Reconfigurable { + private static final Logger LOGGER = Logger.getLogger( + CouchbaseClient.class.getName()); + private static final String MODE_PRODUCTION = "production"; private static final String MODE_DEVELOPMENT = "development"; private static final String DEV_PREFIX = "dev_"; @@ -1715,20 +1717,24 @@ public int getNumVBuckets() { } @Override - public boolean shutdown(long timeout, TimeUnit unit) { + public boolean shutdown(final long timeout, final TimeUnit unit) { boolean shutdownResult = false; + try { shutdownResult = super.shutdown(timeout, unit); + } catch(Exception ex) { + LOGGER.log(Level.SEVERE, "Unexpected Exception in shutdown", ex); + } + + try { CouchbaseConnectionFactory cf = (CouchbaseConnectionFactory) connFactory; cf.getConfigurationProvider().shutdown(); if(vconn != null) { vconn.shutdown(); } } catch (IOException ex) { - Logger.getLogger( - CouchbaseClient.class.getName()).log(Level.SEVERE, - "Unexpected IOException in shutdown", ex); - throw new RuntimeException(null, ex); + LOGGER.log(Level.SEVERE, "Unexpected IOException in shutdown", ex); + shutdownResult = false; } return shutdownResult; } From 5269259739bf0de01191bb6bed693741074f7b47 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 2 Jun 2014 14:42:41 +0200 Subject: [PATCH 10/47] JCBC-424: Broadcast NOOP on idle state. Motivation ---------- This changeset has two main motivations: - Detect rebalancing out nodes & broken sockets even when there is no load. - Help firewalls that close idle sockets. Modifications ------------- In addition to a change in spymemcached which calls a method if the selector is woken up without actually selecting something (so the delay hits), this change correlates the last write with the woken up time. If the timespan is longer than a given maximum, a NOOP broadcast is sent, which works like any other operation but immediately triggers broken sockets and therefore a new configuration will be fetched. Note that the NOOP broadcast is not sent under load, since the write timestamp will always be fresh (and the more load the more seldom the method itself will be called from spy). Result ------ Rebalancing away nodes and broken sockets are much quicker detected and also the socket is prevented from being closed becasue it is idling too much in case of no load. Change-Id: I056b76ea009a689f4ba700730fb281d78d57484d Reviewed-on: http://review.couchbase.org/37729 Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../couchbase/client/CouchbaseConnection.java | 63 ++++++++++++++++++- .../client/CouchbaseMemcachedConnection.java | 58 +++++++++++++++++ 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index d41a1d98..55989fc3 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -27,6 +27,7 @@ import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.VBucketNodeLocator; import com.couchbase.client.vbucket.config.Bucket; +import net.spy.memcached.BroadcastOpFactory; import net.spy.memcached.ConnectionObserver; import net.spy.memcached.FailureMode; import net.spy.memcached.MemcachedConnection; @@ -34,6 +35,8 @@ import net.spy.memcached.OperationFactory; import net.spy.memcached.ops.KeyedOperation; import net.spy.memcached.ops.Operation; +import net.spy.memcached.ops.OperationCallback; +import net.spy.memcached.ops.OperationStatus; import net.spy.memcached.ops.ReplicaGetOperation; import net.spy.memcached.ops.VBucketAware; @@ -50,6 +53,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * Maintains connections to each node in a cluster of Couchbase Nodes. @@ -58,11 +63,19 @@ public class CouchbaseConnection extends MemcachedConnection implements Reconfigurable { + /** + * The amount in seconds after which a op broadcast is forced to detect + * dead connections. + */ + private static final int ALLOWED_IDLE_TIME = 5; + protected volatile boolean reconfiguring = false; private final CouchbaseConnectionFactory cf; private final ThrottleManager throttleManager; private final boolean enableThrottling; + private volatile long lastWrite; + public CouchbaseConnection(int bufSize, CouchbaseConnectionFactory f, List a, Collection obs, FailureMode fm, OperationFactory opfactory) throws IOException { @@ -77,6 +90,7 @@ public CouchbaseConnection(int bufSize, CouchbaseConnectionFactory f, } else { this.throttleManager = null; } + updateLastWrite(); } public void reconfigure(Bucket bucket) { @@ -256,6 +270,7 @@ public void addOperation(final String key, final Operation o) { throttleManager.getThrottler( (InetSocketAddress)placeIn.getSocketAddress()).throttle(); } + updateLastWrite(); addOperation(placeIn, o); } else { assert o.isCancelled() : "No node found for " + key @@ -290,6 +305,7 @@ public void addOperations(final Map ops) { node.addOp(o); addedQueue.offer(node); } + updateLastWrite(); Selector s = selector.wakeup(); assert s == selector : "Wakeup returned the wrong selector."; } @@ -342,9 +358,9 @@ protected void handleRetryInformation(byte[] retryMessage) { protected void queueReconnect(final MemcachedNode node) { cf.getConfigurationProvider().reloadConfig(); if (isShutDown() || !locator.getAll().contains(node)) { - getLogger().debug("Preventing reconnect for node " + node + " because it" - + "is either not part of the cluster anymore or the connection is " - + "shutting down."); + getLogger().debug("Preventing reconnect for node " + node + + " because it" + "is either not part of the cluster anymore or " + + "the connection is " + "shutting down."); return; } @@ -387,4 +403,45 @@ private void logRunException(Exception e) { } } + /** + * Helper method to centralize updating the last write timestamp. + */ + private void updateLastWrite() { + long now = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()); + if (lastWrite != now) { + lastWrite = now; + } + } + + /** + * Make sure that if the selector is woken up manually for an extended period + * of time that the sockets are still alive. + * + *

This is done by broadcasting a operation so that disconnected sockets + * are discovered even when no load is applied.

+ */ + @Override + protected void handleWokenUpSelector() { + long now = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()); + long diff = now - lastWrite; + if (lastWrite > 0 && diff >= ALLOWED_IDLE_TIME) { + updateLastWrite(); + getLogger().debug("Wakeup counter triggered, broadcasting noops."); + final OperationFactory fact = cf.getOperationFactory(); + broadcastOperation(new BroadcastOpFactory() { + @Override + public Operation newOp(MemcachedNode n, final CountDownLatch latch) { + return fact.noop(new OperationCallback() { + @Override + public void receivedStatus(OperationStatus status) { } + + @Override + public void complete() { + latch.countDown(); + } + }); + } + }); + } + } } diff --git a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java index b5221b88..ae2c2623 100644 --- a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java @@ -25,12 +25,15 @@ import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.VBucketNodeLocator; import com.couchbase.client.vbucket.config.Bucket; +import net.spy.memcached.BroadcastOpFactory; import net.spy.memcached.ConnectionObserver; import net.spy.memcached.FailureMode; import net.spy.memcached.MemcachedConnection; import net.spy.memcached.MemcachedNode; import net.spy.memcached.OperationFactory; import net.spy.memcached.ops.Operation; +import net.spy.memcached.ops.OperationCallback; +import net.spy.memcached.ops.OperationStatus; import java.io.IOException; import java.net.InetSocketAddress; @@ -43,6 +46,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * Couchbase implementation of CouchbaseConnection. @@ -58,14 +63,23 @@ public class CouchbaseMemcachedConnection extends MemcachedConnection implements Reconfigurable { + /** + * The amount in seconds after which a op broadcast is forced to detect + * dead connections. + */ + private static final int ALLOWED_IDLE_TIME = 5; + protected volatile boolean reconfiguring = false; private final CouchbaseConnectionFactory cf; + private volatile long lastWrite; + public CouchbaseMemcachedConnection(int bufSize, CouchbaseConnectionFactory f, List a, Collection obs, FailureMode fm, OperationFactory opfactory) throws IOException { super(bufSize, f, a, obs, fm, opfactory); this.cf = f; + updateLastWrite(); } @@ -202,6 +216,7 @@ protected void addOperation(final String key, final Operation o) { assert o.isCancelled() || placeIn != null : "No node found for key " + key; if (placeIn != null) { + updateLastWrite(); addOperation(placeIn, o); } else { assert o.isCancelled() : "No node found for " + key @@ -266,4 +281,47 @@ private void logRunException(Exception e) { } } + /** + * Helper method to centralize updating the last write timestamp. + */ + private void updateLastWrite() { + long now = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()); + if (lastWrite != now) { + lastWrite = now; + } + } + + + /** + * Make sure that if the selector is woken up manually for an extended period + * of time that the sockets are still alive. + * + *

This is done by broadcasting a operation so that disconnected sockets + * are discovered even when no load is applied.

+ */ + @Override + protected void handleWokenUpSelector() { + long now = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()); + long diff = now - lastWrite; + if (lastWrite > 0 && diff >= ALLOWED_IDLE_TIME) { + updateLastWrite(); + getLogger().debug("Wakeup counter triggered, broadcasting noops."); + final OperationFactory fact = cf.getOperationFactory(); + broadcastOperation(new BroadcastOpFactory() { + @Override + public Operation newOp(MemcachedNode n, final CountDownLatch latch) { + return fact.noop(new OperationCallback() { + @Override + public void receivedStatus(OperationStatus status) { } + + @Override + public void complete() { + latch.countDown(); + } + }); + } + }); + } + } + } From b301f290ff7b0479b8e0920ea4373860af3791a2 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Thu, 5 Jun 2014 15:27:55 +0200 Subject: [PATCH 11/47] Upgrading Spymemcached to 2.11.3 Change-Id: I7b51952195a55c69d70ef151e493cd5d36c447a8 Reviewed-on: http://review.couchbase.org/37903 Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 09c8f515..880f647d 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.2 -spymemcached-test.version=2.11.2 +spymemcached.version=2.11.3 +spymemcached-test.version=2.11.3 From 04fa4338c8ee62a50dca4a9f0562c38b479390a7 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 23 Jun 2014 17:20:22 +0200 Subject: [PATCH 12/47] JCBC-477: Pass down the timeout to the getBulk for views Motivation ---------- If no timeout is passed down to the BulkGetFuture, Long.MAX_VALUE is used as a timeout which, if something goes wrong, basically translates to block the thread forever. Also, this breaks the contract since a custom time can always be passed in. Modifications ------------- The timeout is now passed down to the bulk get so that a timeout is properly respected. Result ------ Correct behavior from the client side. Change-Id: I56696a21ed14e9c0669eb5d05941f67a987eeb18 Reviewed-on: http://review.couchbase.org/38678 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- .../java/com/couchbase/client/internal/ViewFuture.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/couchbase/client/internal/ViewFuture.java b/src/main/java/com/couchbase/client/internal/ViewFuture.java index 0bc09770..06583f00 100644 --- a/src/main/java/com/couchbase/client/internal/ViewFuture.java +++ b/src/main/java/com/couchbase/client/internal/ViewFuture.java @@ -29,6 +29,10 @@ import com.couchbase.client.protocol.views.ViewResponseWithDocs; import com.couchbase.client.protocol.views.ViewRow; import com.couchbase.client.protocol.views.ViewRowWithDocs; +import net.spy.memcached.internal.BulkFuture; +import net.spy.memcached.internal.GenericCompletionListener; +import net.spy.memcached.ops.OperationStatus; + import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; @@ -39,9 +43,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; -import net.spy.memcached.internal.BulkFuture; -import net.spy.memcached.internal.GenericCompletionListener; -import net.spy.memcached.ops.OperationStatus; /** * A ViewFuture. @@ -68,7 +69,7 @@ public ViewResponse get(long duration, TimeUnit units) return null; } - Map docMap = multigetRef.get().get(); + Map docMap = multigetRef.get().get(duration, units); final ViewResponseWithDocs viewResp = (ViewResponseWithDocs) objRef.get(); Collection rows = new LinkedList(); Iterator itr = viewResp.iterator(); From 0a740e124c8ded89ed42bf8489a8e3b7191cb89c Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 23 Jun 2014 17:00:09 +0200 Subject: [PATCH 13/47] JCBC-476: Rework misleading log message. Motivation ---------- The message in the changeset is misleading because the ops are now queued up and therefore more resilient. Also, WARN is too high since it should be just an INFO message. If auth or anything else fails, different messages will be shown in the logs anyway. Modifications ------------- Change it to INFO level and also rework the text to be more clear what is happening with the given key. Result ------ No misleading information anymore which looks falsly suspicious. Change-Id: I9b0052dd82bad4edbc54df4d06c98b6b18f31dce Reviewed-on: http://review.couchbase.org/38668 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- src/main/java/com/couchbase/client/CouchbaseConnection.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index 55989fc3..3e776e32 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -236,10 +236,8 @@ public void addOperation(final String key, final Operation o) { // If we didn't find an active node, queue it in the primary node // and wait for it to come back online. if (needsRecheckConfigUpdate) { - getLogger().warn( - "Node expected to receive data is inactive. This could be due to " - + "a failure within the cluster. Will check for updated " - + "configuration. Key without a configured node is: %s.", key); + getLogger().info("Node for key \"%s\" is not active (yet). Queueing " + + "up for retry and checking for stale configuration.", key); cf.checkConfigUpdate(); } From 1d0fdce12b8028a2d41c73ad8a0d3800f2170dd8 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Tue, 17 Jun 2014 13:46:50 +0200 Subject: [PATCH 14/47] JCBC-475: Harden configuration provider on shutdown and restarting of nodes. Motivation ---------- The current configuration provider runs into issues when nodes are restarted in several edge cases. Also, shutdown handling was subject to race conditions, leaving threads behind and as a result kept JVMs running. Modifications ------------- This changeset makes several modifications to the provider: - A shutdown variable is introduced and crucial parts of the code are made aware of it, stopping their work if shutdown is true. - If a new carrier provider is set, the code now checks if there was an old one in place, shutting it down. This makes sure that no threads are left behind. - Only monitor the bucket in http bootstrap, avoiding weird stack overflows if monitoring and bootstrapping does not succeed (if only one node is used and it is restarted). In addition the used CouchbaseConfigConnection has been clarified a bit so that from a thread dump it is now visible if a data IO thread is meant or the config thread for carrier configs. Result ------ In general, the resulting provider should be more resilient to failure and should not leave threads behind on shutdown. Change-Id: Ic3c4026494c858e03d63767dd953675104687e0b Reviewed-on: http://review.couchbase.org/38364 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- pom.xml | 2 +- .../provider/BucketConfigurationProvider.java | 45 ++++++++++++++++--- .../provider/CouchbaseConfigConnection.java | 12 +++++ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index dd4def10..64ed147f 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ http://www.couchbase.com/develop/java/current - 2.11.0 + 2.11.3 diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index 2fd70e2d..90529eea 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -84,6 +84,7 @@ public class BucketConfigurationProvider extends SpyObject private final boolean disableHttpBootstrap; private volatile boolean isBinary; private volatile long lastRevision; + private volatile boolean shutdown; public BucketConfigurationProvider(final List seedNodes, final String bucket, final String password, @@ -108,10 +109,15 @@ public BucketConfigurationProvider(final List seedNodes, CouchbaseProperties.getProperty("disableCarrierBootstrap", "false")); disableHttpBootstrap = Boolean.parseBoolean( CouchbaseProperties.getProperty("disableHttpBootstrap", "false")); + shutdown = false; } @Override public Bucket bootstrap() { + if(shutdown) { + getLogger().debug("Omitting bootstrap since already shutdown."); + } + isBinary = false; if (!bootstrapBinary() && !bootstrapHttp()) { throw new ConfigurationException("Could not fetch a valid Bucket " @@ -125,7 +131,6 @@ public Bucket bootstrap() { + "HTTP."); } - monitorBucket(); return config.get(); } @@ -178,10 +183,13 @@ boolean bootstrapBinary() { */ private boolean tryBinaryBootstrapForNode(InetSocketAddress node) throws Exception { + if (binaryConnection.get() != null) { + return true; + } ConfigurationConnectionFactory fact = new ConfigurationConnectionFactory(seedNodes, bucket, password); CouchbaseConnectionFactory cf = connectionFactory; - CouchbaseConnection connection; + CouchbaseConnection connection = null; List initialObservers = new ArrayList(); final CountDownLatch latch = new CountDownLatch(1); @@ -210,6 +218,9 @@ public void connectionLost(SocketAddress socketAddress) { + " port in the given time interval."); } } catch (Exception ex) { + if (connection != null) { + connection.shutdown(); + } getLogger().debug("(Carrier Publication) Could not load config from " + node.getHostName() + ", trying next node.", ex); return false; @@ -238,8 +249,15 @@ public void connectionLost(SocketAddress socketAddress) { String appliedConfig = connection.replaceConfigWildcards( configs.get(0)); - Bucket config = configurationParser.parseBucket(appliedConfig); - setConfig(config); + try { + Bucket config = configurationParser.parseBucket(appliedConfig); + setConfig(config); + } catch(Exception ex) { + getLogger().warn("Could not parse config, retrying bootstrap.", ex); + connection.shutdown(); + return false; + } + connection.addObserver(new ConnectionObserver() { @Override public void connectionEstablished(SocketAddress sa, int reconnectCount) { @@ -252,12 +270,18 @@ public void connectionLost(SocketAddress sa) { CouchbaseConnection conn = binaryConnection.getAndSet(null); try { conn.shutdown(); + } catch (IOException e) { getLogger().debug("Could not shut down Carrier Config Connection", e); } signalOutdated(); } }); + + CouchbaseConnection old = binaryConnection.get(); + if (old != null) { + old.shutdown(); + } binaryConnection.set(connection); return true; } @@ -319,6 +343,7 @@ boolean bootstrapHttp() { try { Bucket config = httpProvider.get().getBucketConfiguration(bucket); setConfig(config); + monitorBucket(); isBinary = false; return true; } catch(Exception ex) { @@ -332,8 +357,8 @@ boolean bootstrapHttp() { * used. */ private void monitorBucket() { - if (!isBinary) { - httpProvider.get().subscribe(bucket, this); + if (!shutdown && !isBinary) { + httpProvider.get().subscribe(bucket, this); } } @@ -462,6 +487,11 @@ private static boolean seedNodesAreDifferent(List left, @Override public void signalOutdated() { + if (shutdown) { + getLogger().debug("Omitting signalOutdated since already shutdown."); + return; + } + if (isBinary) { if (binaryConnection.get() == null) { bootstrap(); @@ -495,13 +525,14 @@ public void signalOutdated() { @Override public void reloadConfig() { - if (isBinary) { + if (isBinary && !shutdown) { signalOutdated(); } } @Override public void shutdown() { + shutdown = true; if (httpProvider.get() != null) { httpProvider.get().shutdown(); } diff --git a/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java b/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java index ef265d43..70bf9653 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java @@ -26,6 +26,7 @@ import com.couchbase.client.CouchbaseConnectionFactory; import net.spy.memcached.ConnectionObserver; import net.spy.memcached.FailureMode; +import net.spy.memcached.MemcachedNode; import net.spy.memcached.OperationFactory; import net.spy.memcached.compat.log.Level; import net.spy.memcached.compat.log.Logger; @@ -55,6 +56,17 @@ public CouchbaseConfigConnection(int bufSize, CouchbaseConnectionFactory f, super(bufSize, f, a, obs, fm, opfactory); } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{CouchbaseConfigConnection to"); + for (MemcachedNode qa : locator.getAll()) { + sb.append(" ").append(qa.getSocketAddress()); + } + sb.append("}"); + return sb.toString(); + } + @Override protected Logger getLogger() { return LOGGER; From 02c4f367098beeff6846a98cdea89eb2cc965006 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Wed, 25 Jun 2014 13:42:06 +0200 Subject: [PATCH 15/47] JCBC-480: Complete replica read ops even if NOT_EXISTS Motivation ---------- get(s)FromReplica operations may timeout incorrectly if the document has not been stored on the server at all, since it waits for a successful response before it completes the latch and stores the response. Since a non-existing doc response returns NOT_EXISTS and false success, it will never complete and timeout. Modifications ------------- Complete the latch as well if the response is a NOT_EXISTS, indicating a good response but the document was just not found. This is perfectly fine under the assumptions that the response is allowed to be non-consistent by contract, even if it may not be replicated yet. Result ------ The behavior is now consistent for both found and not found docs. Change-Id: Ic310255e28c498ed6482101f64657978fad8c7ae Reviewed-on: http://review.couchbase.org/38796 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- .../com/couchbase/client/CouchbaseClient.java | 9 +++---- .../com/couchbase/client/ReplicaReadTest.java | 24 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseClient.java b/src/main/java/com/couchbase/client/CouchbaseClient.java index befaf582..dab78eef 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClient.java +++ b/src/main/java/com/couchbase/client/CouchbaseClient.java @@ -78,6 +78,7 @@ import net.spy.memcached.ops.ReplicaGetOperation; import net.spy.memcached.ops.ReplicaGetsOperation; import net.spy.memcached.ops.StatsOperation; +import net.spy.memcached.ops.StatusCode; import net.spy.memcached.transcoders.Transcoder; import org.apache.http.HttpRequest; import org.apache.http.HttpVersion; @@ -1042,7 +1043,7 @@ private Operation createOperationForReplicaGet(final String key, @Override public void receivedStatus(OperationStatus status) { future.set(val, status); - if (!replicaFuture.isDone() && status.isSuccess()) { + if (!replicaFuture.isDone() && (status.isSuccess() || status.getStatusCode() == StatusCode.ERR_NOT_FOUND)) { usedFuture = replicaFuture.setCompletedFuture(future); } } @@ -1070,7 +1071,7 @@ public void complete() { @Override public void receivedStatus(OperationStatus status) { future.set(val, status); - if (!replicaFuture.isDone() && status.isSuccess()) { + if (!replicaFuture.isDone() && (status.isSuccess() || status.getStatusCode() == StatusCode.ERR_NOT_FOUND)) { usedFuture = replicaFuture.setCompletedFuture(future); } } @@ -1106,7 +1107,7 @@ private Operation createOperationForReplicaGets(final String key, @Override public void receivedStatus(OperationStatus status) { future.set(val, status); - if (!replicaFuture.isDone() && status.isSuccess()) { + if (!replicaFuture.isDone() && (status.isSuccess() || status.getStatusCode() == StatusCode.ERR_NOT_FOUND)) { usedFuture = replicaFuture.setCompletedFuture(future); } } @@ -1134,7 +1135,7 @@ public void complete() { @Override public void receivedStatus(OperationStatus status) { future.set(val, status); - if (!replicaFuture.isDone() && status.isSuccess()) { + if (!replicaFuture.isDone() && (status.isSuccess() || status.getStatusCode() == StatusCode.ERR_NOT_FOUND)) { usedFuture = replicaFuture.setCompletedFuture(future); } } diff --git a/src/test/java/com/couchbase/client/ReplicaReadTest.java b/src/test/java/com/couchbase/client/ReplicaReadTest.java index fb71772b..05443909 100644 --- a/src/test/java/com/couchbase/client/ReplicaReadTest.java +++ b/src/test/java/com/couchbase/client/ReplicaReadTest.java @@ -25,19 +25,19 @@ import com.couchbase.client.clustermanager.BucketType; import com.couchbase.client.internal.ReplicaGetCompletionListener; import com.couchbase.client.internal.ReplicaGetFuture; -import java.net.URI; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - import net.spy.memcached.CASValue; import net.spy.memcached.ConnectionFactory; import net.spy.memcached.ReplicateTo; import net.spy.memcached.TestConfig; - import org.junit.Before; import org.junit.Test; + +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -153,4 +153,14 @@ public void onComplete(ReplicaGetFuture f) throws Exception { latch.await(5, TimeUnit.SECONDS)); } + @Test + public void testGetFromReplicaWithNonExistentKey() { + assertEquals(null, client.getFromReplica("nonexistentKey")); + } + + @Test + public void testGetsFromReplicaWithNonExistentKey() { + assertEquals(null, client.getsFromReplica("nonexistentKey")); + } + } From b634acbc477b5ff8f99be1126cdd32519c4564a9 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Fri, 27 Jun 2014 10:59:53 +0200 Subject: [PATCH 16/47] JCBC-482: Select proper replica node for getsFromReplica Motivation ---------- getsFromReplica did not work when the master node is currently failing because it had not been scheduled properly. This fix remedies that. Note that getFromReplica had always worked as expected. Modification ------------ This changeset adds support for getsFromReplica in the add operation codepath to properly select the replica node. Result ------ getsFromReplica now works like getFromReplica even when only replica nodes are available. Change-Id: I4362efbba4ea04ff063c85128949a3931c95bd58 Reviewed-on: http://review.couchbase.org/38866 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- .../java/com/couchbase/client/CouchbaseConnection.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index 3e776e32..12ce6e08 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -38,6 +38,7 @@ import net.spy.memcached.ops.OperationCallback; import net.spy.memcached.ops.OperationStatus; import net.spy.memcached.ops.ReplicaGetOperation; +import net.spy.memcached.ops.ReplicaGetsOperation; import net.spy.memcached.ops.VBucketAware; import java.io.IOException; @@ -198,8 +199,12 @@ public void addOperation(final String key, final Operation o) { MemcachedNode primary; if(o instanceof ReplicaGetOperation && locator instanceof VBucketNodeLocator) { - primary = ((VBucketNodeLocator)locator).getReplica(key, - ((ReplicaGetOperation)o).getReplicaIndex()); + primary = ((VBucketNodeLocator) locator).getReplica(key, + ((ReplicaGetOperation) o).getReplicaIndex()); + } else if(o instanceof ReplicaGetsOperation + && locator instanceof VBucketNodeLocator) { + primary = ((VBucketNodeLocator) locator).getReplica(key, + ((ReplicaGetsOperation) o).getReplicaIndex()); } else { primary = locator.getPrimary(key); } From 23040ca987095332a90b563030db8e7114987483 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Tue, 1 Jul 2014 07:28:33 +0200 Subject: [PATCH 17/47] Upgrade spymemcached to 2.11.4 Change-Id: I9971faf5b9511f8764667654fc326469e0ad8d9c Reviewed-on: http://review.couchbase.org/39006 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 880f647d..aef5e872 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.3 -spymemcached-test.version=2.11.3 +spymemcached.version=2.11.4 +spymemcached-test.version=2.11.4 From 75d57883e684e18e0eb5270a1128e427e237d531 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Wed, 16 Jul 2014 09:09:23 +0200 Subject: [PATCH 18/47] JCBC-488: Make sure headers are overriden, not appended on readd. Motivation ---------- If a view operation needs to be retried, the current code adds headers instead of overwriting them, which leads to broken http requests. As a result, the server responds with 400 and is unable to process the query. Modifications ------------- Changing the code from addHeader to setHeaders overrides the headers instead of appending them. Result ------ Valid HTTP requests, even if it needs to be retried. Change-Id: I13b72effc2998d17aaf2ac8acfc3c225aef30de3 Reviewed-on: http://review.couchbase.org/39429 Reviewed-by: Matt Ingenthron Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../com/couchbase/client/ClusterManager.java | 6 ++--- .../com/couchbase/client/ViewConnection.java | 23 +++++++++---------- .../protocol/views/HttpOperationImpl.java | 9 ++++---- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/couchbase/client/ClusterManager.java b/src/main/java/com/couchbase/client/ClusterManager.java index be6540bf..16989afe 100644 --- a/src/main/java/com/couchbase/client/ClusterManager.java +++ b/src/main/java/com/couchbase/client/ClusterManager.java @@ -450,10 +450,10 @@ private HttpResult sendRequest(final HttpRequest request) { HttpCoreContext coreContext = HttpCoreContext.create(); - request.addHeader("Authorization", "Basic " + request.setHeader("Authorization", "Basic " + Base64.encodeBase64String((username + ':' + password).getBytes())); - request.addHeader("Accept", "*/*"); - request.addHeader("Content-Type", "application/x-www-form-urlencoded"); + request.setHeader("Accept", "*/*"); + request.setHeader("Content-Type", "application/x-www-form-urlencoded"); for (HttpHost node : clusterNodes) { try { diff --git a/src/main/java/com/couchbase/client/ViewConnection.java b/src/main/java/com/couchbase/client/ViewConnection.java index af7419cc..983ec42a 100644 --- a/src/main/java/com/couchbase/client/ViewConnection.java +++ b/src/main/java/com/couchbase/client/ViewConnection.java @@ -29,17 +29,6 @@ import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.config.Bucket; import com.couchbase.client.vbucket.config.CouchbaseConfig; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; -import java.net.InetSocketAddress; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - import net.spy.memcached.compat.SpyObject; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; @@ -63,6 +52,16 @@ import org.apache.http.protocol.RequestTargetHost; import org.apache.http.protocol.RequestUserAgent; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.UnsupportedEncodingException; +import java.net.InetSocketAddress; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + /** * The {@link ViewConnection} is responsible for managing and multiplexing * HTTP connections to Couchbase View endpoints. @@ -216,7 +215,7 @@ public void addOp(final HttpOperation op) { HttpHost httpHost = getNextNode(); HttpRequest request = op.getRequest(); - request.addHeader(HTTP.TARGET_HOST, httpHost.toHostString()); + request.setHeader(HTTP.TARGET_HOST, httpHost.toHostString()); requester.execute( new BasicAsyncRequestProducer(httpHost, request), new BasicAsyncResponseConsumer(), diff --git a/src/main/java/com/couchbase/client/protocol/views/HttpOperationImpl.java b/src/main/java/com/couchbase/client/protocol/views/HttpOperationImpl.java index 1f2129ce..bd4dc1b3 100644 --- a/src/main/java/com/couchbase/client/protocol/views/HttpOperationImpl.java +++ b/src/main/java/com/couchbase/client/protocol/views/HttpOperationImpl.java @@ -22,20 +22,19 @@ package com.couchbase.client.protocol.views; -import java.io.IOException; -import java.text.ParseException; - import net.spy.memcached.ops.OperationCallback; import net.spy.memcached.ops.OperationErrorType; import net.spy.memcached.ops.OperationException; import net.spy.memcached.ops.OperationStatus; - import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.util.EntityUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import java.io.IOException; +import java.text.ParseException; + /** * An HttpOperationImpl. */ @@ -95,7 +94,7 @@ public void setException(OperationException e) { } public void addAuthHeader(String authzn) { - request.addHeader("Authorization", authzn); + request.setHeader("Authorization", authzn); } public abstract void handleResponse(HttpResponse response); From c7beac4f40d03e851cb316e29794d956c147bfd9 Mon Sep 17 00:00:00 2001 From: David Haikney Date: Wed, 16 Jul 2014 18:18:44 +0100 Subject: [PATCH 19/47] JCBC-495: Quotes round variables to accommodate pathnames with spaces Change-Id: I9dc381a1b7223c4ed3c23c14989e4261f50badba Reviewed-on: http://review.couchbase.org/39443 Reviewed-by: Matt Ingenthron Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- src/scripts/write-version-info.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/scripts/write-version-info.sh b/src/scripts/write-version-info.sh index fc85286f..31825115 100755 --- a/src/scripts/write-version-info.sh +++ b/src/scripts/write-version-info.sh @@ -43,9 +43,9 @@ host=`hostname` compiledate=`date` treeversion=`git describe` -mkdir -p ${outputdir} -git log > ${changesfile} -cat > ${buildinfofile} < "${changesfile}" +cat > "${buildinfofile}" < Date: Mon, 4 Aug 2014 17:09:59 +0200 Subject: [PATCH 20/47] JCBC-503: Release observers on shutdown. Motivation ---------- On shutdown of the provider, there are still observers referenced. Making sure they are not referenced anymore should be part of a clean shutdown procedure. Modifications ------------- Clear out the observers array to remove the references. Result ------ Cleaner shutdown. Change-Id: I710d1b8be9dea88aba0fe28162442951807610de Reviewed-on: http://review.couchbase.org/40263 Tested-by: Michael Nitschinger Reviewed-by: Matt Ingenthron --- .../client/vbucket/provider/BucketConfigurationProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index 90529eea..e5f5615e 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -532,6 +532,7 @@ public void reloadConfig() { @Override public void shutdown() { + observers.clear(); shutdown = true; if (httpProvider.get() != null) { httpProvider.get().shutdown(); From bf510d732978c4877296a20e7c7a1b0033c83657 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Wed, 30 Jul 2014 18:49:17 +0200 Subject: [PATCH 21/47] JCBC-505: Fix concurrency issue in Query builder. Motivation ---------- Similar to one fix earlier in SPY (SPY-170), only the regex matcher is thread safe (resetting it is not), leading to NPEs further down the stack. Modifications ------------- Always use a new builder and do not reset the same one. Result ------ Thread safety for the Query class and no exceptions like: java.lang.NullPointerException at java.util.regex.Matcher.getTextLength(Matcher.java:1283) at java.util.regex.Matcher.reset(Matcher.java:309) at java.util.regex.Matcher.reset(Matcher.java:329) at com.couchbase.client.protocol.views.Query.quote(Query.java:572) at com.couchbase.client.protocol.views.Query.setRangeStart(Query.java:400) at com.couchbase.client.protocol.views.Paginator.fetchNextPage(Paginator.java:180) at com.couchbase.client.protocol.views.Paginator.hasNext(Paginator.java:160) Change-Id: Icc578a03bdece1ebc9b344ee8fdbe36a2542b6cf Reviewed-on: http://review.couchbase.org/40058 Tested-by: Michael Nitschinger Reviewed-by: Matt Ingenthron --- .../java/com/couchbase/client/protocol/views/Query.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/java/com/couchbase/client/protocol/views/Query.java b/src/main/java/com/couchbase/client/protocol/views/Query.java index c3a85632..90a7d57f 100644 --- a/src/main/java/com/couchbase/client/protocol/views/Query.java +++ b/src/main/java/com/couchbase/client/protocol/views/Query.java @@ -27,7 +27,6 @@ import java.text.ParsePosition; import java.util.HashMap; import java.util.Map; -import java.util.regex.Matcher; import java.util.regex.Pattern; /** @@ -91,11 +90,6 @@ public class Query { private static final Pattern quotePattern = Pattern.compile("^(\".*|\\{.*|\\[.*|true|false|null|-?[\\d,]*([.Ee]\\d+)?)$"); - /** - * A pre allocated matcher for the quote pattern match. - */ - private final Matcher quoteMatcher = quotePattern.matcher(""); - /** * Number format to use to find matching numbers. */ @@ -569,7 +563,7 @@ protected String encode(final String source) { * @return maybe quoted target string. */ protected String quote(final String source) { - if (quoteMatcher.reset(source).matches()) { + if (quotePattern.matcher(source).matches()) { ParsePosition parsePosition = new ParsePosition(0); Number result = numberFormat.parse(source, parsePosition); if (parsePosition.getIndex() == source.length()) { From ffaaf977ff3c6f08465122dde6a505c7052a1cb1 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 14 Jul 2014 11:29:51 +0200 Subject: [PATCH 22/47] JCBC-490: Error log unexpected/unparsable JSON body content. Motivation ---------- Stack traces with invalid server responses have been reported where the root cause is unclear. The stack trace does not show the actual response however so its hard to decipher. Modifications ------------- Since normally such a case shouldn't happen, the actual body including the HTTP header are logged at ERROR level to raise proper attention and make it easier to find the actual cause. Result ------ Unparsable server results that lead to exceptions are now easier to decipher and to track down eventually. Change-Id: Id3c98a6488581c5025811433452551de8cc9a136 Reviewed-on: http://review.couchbase.org/39347 Reviewed-by: Matt Ingenthron Tested-by: Michael Nitschinger --- .../protocol/views/DocsOperationImpl.java | 12 +++++------- .../protocol/views/NoDocsOperationImpl.java | 12 +++++------- .../protocol/views/ReducedOperationImpl.java | 12 +++++------- .../protocol/views/ViewOperationImpl.java | 17 ++++++++++------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/couchbase/client/protocol/views/DocsOperationImpl.java b/src/main/java/com/couchbase/client/protocol/views/DocsOperationImpl.java index 47029c46..4be2f615 100644 --- a/src/main/java/com/couchbase/client/protocol/views/DocsOperationImpl.java +++ b/src/main/java/com/couchbase/client/protocol/views/DocsOperationImpl.java @@ -22,16 +22,15 @@ package com.couchbase.client.protocol.views; -import java.text.ParseException; -import java.util.Collection; -import java.util.LinkedList; -import java.util.logging.Level; - import org.apache.http.HttpRequest; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import java.text.ParseException; +import java.util.Collection; +import java.util.LinkedList; + /** * Implementation of a view that calls the map * function and includes the documents in the result. @@ -70,8 +69,7 @@ protected ViewResponseWithDocs parseResult(String json) } } if (base.has("debug_info")) { - LOGGER.log(Level.INFO, "Debugging View {0}: {1}", - new Object[]{getView().getURI(), json}); + LOGGER.info("Debugging View {0}: {1}", getView().getURI(), json); } if (base.has("errors")) { JSONArray ids = base.getJSONArray("errors"); diff --git a/src/main/java/com/couchbase/client/protocol/views/NoDocsOperationImpl.java b/src/main/java/com/couchbase/client/protocol/views/NoDocsOperationImpl.java index f40ae45b..d643adac 100644 --- a/src/main/java/com/couchbase/client/protocol/views/NoDocsOperationImpl.java +++ b/src/main/java/com/couchbase/client/protocol/views/NoDocsOperationImpl.java @@ -22,16 +22,15 @@ package com.couchbase.client.protocol.views; -import java.text.ParseException; -import java.util.Collection; -import java.util.LinkedList; -import java.util.logging.Level; - import org.apache.http.HttpRequest; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import java.text.ParseException; +import java.util.Collection; +import java.util.LinkedList; + /** * Implementation of a view that calls the map * function and excludes the documents in the result. @@ -71,8 +70,7 @@ protected ViewResponseNoDocs parseResult(String json) } } if (base.has("debug_info")) { - LOGGER.log(Level.INFO, "Debugging View {0}: {1}", - new Object[]{getView().getURI(), json}); + LOGGER.info("Debugging View {0}: {1}", getView().getURI(), json); } if (base.has("errors")) { JSONArray ids = base.getJSONArray("errors"); diff --git a/src/main/java/com/couchbase/client/protocol/views/ReducedOperationImpl.java b/src/main/java/com/couchbase/client/protocol/views/ReducedOperationImpl.java index c2ebf38c..c6501bd9 100644 --- a/src/main/java/com/couchbase/client/protocol/views/ReducedOperationImpl.java +++ b/src/main/java/com/couchbase/client/protocol/views/ReducedOperationImpl.java @@ -22,16 +22,15 @@ package com.couchbase.client.protocol.views; -import java.text.ParseException; -import java.util.Collection; -import java.util.LinkedList; -import java.util.logging.Level; - import org.apache.http.HttpRequest; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import java.text.ParseException; +import java.util.Collection; +import java.util.LinkedList; + /** * Implementation of a view that calls the map function * and the reduce function and gets the result. @@ -64,8 +63,7 @@ protected ViewResponseReduced parseResult(String json) } } if (base.has("debug_info")) { - LOGGER.log(Level.INFO, "Debugging View {0}: {1}", - new Object[]{getView().getURI(), json}); + LOGGER.info("Debugging View {0}: {1}", getView().getURI(), json); } if (base.has("errors")) { JSONArray ids = base.getJSONArray("errors"); diff --git a/src/main/java/com/couchbase/client/protocol/views/ViewOperationImpl.java b/src/main/java/com/couchbase/client/protocol/views/ViewOperationImpl.java index 7a8c3206..56bc6567 100644 --- a/src/main/java/com/couchbase/client/protocol/views/ViewOperationImpl.java +++ b/src/main/java/com/couchbase/client/protocol/views/ViewOperationImpl.java @@ -22,17 +22,17 @@ package com.couchbase.client.protocol.views; -import java.text.ParseException; -import java.util.logging.Logger; - +import net.spy.memcached.compat.log.Logger; +import net.spy.memcached.compat.log.LoggerFactory; import net.spy.memcached.ops.OperationCallback; import net.spy.memcached.ops.OperationErrorType; import net.spy.memcached.ops.OperationException; import net.spy.memcached.ops.OperationStatus; - import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; +import java.text.ParseException; + /** * A ViewOperationImpl. * @@ -42,8 +42,10 @@ public abstract class ViewOperationImpl extends HttpOperationImpl private final AbstractView view; - protected static final Logger LOGGER = Logger.getLogger( - ViewOperationImpl.class.getName()); + protected static final Logger LOGGER = LoggerFactory.getLogger( + ViewOperationImpl.class); + + public ViewOperationImpl(HttpRequest r, AbstractView view, OperationCallback cb) { @@ -71,8 +73,9 @@ public void handleResponse(HttpResponse response) { ((ViewCallback) callback).gotData(vr); callback.receivedStatus(status); } catch (ParseException e) { + LOGGER.error("Failed to parse JSON in response: " + response + ": " + json); setException(new OperationException(OperationErrorType.GENERAL, - "Error parsing JSON")); + "Error parsing JSON (" + e.getMessage() + "): " + json)); } callback.complete(); } From 669c9580674082b2acccd769d9da5e1b3bb3e417 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 14 Jul 2014 11:07:52 +0200 Subject: [PATCH 23/47] JCBC-464: Clear buckets for HTTP provider on (re)bootstrap. Motivation ---------- Prior to this change, a combination of (re)bootstrapping initiated by a full cluster shutdown and a previously loaded configuration led to a invalid state in the HTTP provider. Since it is also used for a cccp fallback, it always interfered in such a scenario. Modifications ------------- The internal problem was that once bootstrapped, the bucket config was stored and on a re-bootstrap, this old config was taken. That itself did not cause trouble, but the http streaming config attachment was broken because some parameters were not set properly (which would have been on a full bootstrap). This changeset clears out the bucket configs before a (re)bootstrap to make sure a full http walk cycle is always done. This provides more predictable behavior and also avoids a reported NPE. Result ------ Even if rebootstrapped, it now either gets to a valid configuration over HTTP and succeed or will retry, without failing with a NPE or ending up in an invalid state. Change-Id: Id8644b1fddf7b38168e663a3d5af51e17e56b9c4 Reviewed-on: http://review.couchbase.org/39346 Reviewed-by: Matt Ingenthron Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../client/vbucket/ConfigurationProviderHTTP.java | 15 +++++++++------ .../provider/BucketConfigurationProvider.java | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/couchbase/client/vbucket/ConfigurationProviderHTTP.java b/src/main/java/com/couchbase/client/vbucket/ConfigurationProviderHTTP.java index a5bd6af0..c1d20418 100644 --- a/src/main/java/com/couchbase/client/vbucket/ConfigurationProviderHTTP.java +++ b/src/main/java/com/couchbase/client/vbucket/ConfigurationProviderHTTP.java @@ -29,13 +29,14 @@ import com.couchbase.client.vbucket.config.ConfigurationParser; import com.couchbase.client.vbucket.config.ConfigurationParserJSON; import com.couchbase.client.vbucket.config.Pool; +import net.spy.memcached.AddrUtil; +import net.spy.memcached.compat.SpyObject; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; - import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; @@ -43,15 +44,11 @@ import java.net.URL; import java.net.URLConnection; import java.text.ParseException; - import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import net.spy.memcached.AddrUtil; -import net.spy.memcached.compat.SpyObject; - /** * A configuration provider. */ @@ -71,7 +68,7 @@ public class ConfigurationProviderHTTP extends SpyObject implements private volatile List baseList; private final String restUsr; private final String restPwd; - private URI loadedBaseUri; + private volatile URI loadedBaseUri; private final Map buckets = new ConcurrentHashMap(); @@ -487,4 +484,10 @@ public void updateBucket(final String config) { } } + /** + * Clears all stored bucket references to get back to a pre-bootstrap state. + */ + public void clearBuckets() { + this.buckets.clear(); + } } diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index e5f5615e..e1cb4242 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -341,6 +341,7 @@ boolean bootstrapHttp() { } try { + httpProvider.get().clearBuckets(); Bucket config = httpProvider.get().getBucketConfiguration(bucket); setConfig(config); monitorBucket(); From 9f6fdf8d260b079850d8872534bc9383b547e62d Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 18 Aug 2014 08:06:39 +0200 Subject: [PATCH 24/47] JCBC-510: Allow optional non-persistent view connections. Motivation ---------- With HTTP 1.1, all connections are by default persistent, unless connection control headers are set to close. In order to provide more flexbility in how the client handles those connection, an optional system property can be set to change it appropriately. Modifications ------------- A custom system property has been introduced and the view pipeline has been modified to take it into account. Tests have been added to verify the header settings. Result ------ It is now possible to change socket persistence behavior from a system property. Change-Id: I960bf22cb64f8fa93ee4853cfff0bd1103d487c2 Reviewed-on: http://review.couchbase.org/40685 Reviewed-by: Matt Ingenthron Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../com/couchbase/client/ViewConnection.java | 6 +- .../client/http/CustomRequestConnControl.java | 63 ++++++++++++++++++ .../http/CustomRequestConnControlTest.java | 64 +++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/couchbase/client/http/CustomRequestConnControl.java create mode 100644 src/test/java/com/couchbase/client/http/CustomRequestConnControlTest.java diff --git a/src/main/java/com/couchbase/client/ViewConnection.java b/src/main/java/com/couchbase/client/ViewConnection.java index 983ec42a..eff2c6c1 100644 --- a/src/main/java/com/couchbase/client/ViewConnection.java +++ b/src/main/java/com/couchbase/client/ViewConnection.java @@ -22,6 +22,7 @@ package com.couchbase.client; +import com.couchbase.client.http.CustomRequestConnControl; import com.couchbase.client.http.HttpResponseCallback; import com.couchbase.client.http.HttpUtil; import com.couchbase.client.http.ViewPool; @@ -46,7 +47,6 @@ import org.apache.http.protocol.HttpCoreContext; import org.apache.http.protocol.HttpProcessor; import org.apache.http.protocol.HttpProcessorBuilder; -import org.apache.http.protocol.RequestConnControl; import org.apache.http.protocol.RequestContent; import org.apache.http.protocol.RequestExpectContinue; import org.apache.http.protocol.RequestTargetHost; @@ -147,7 +147,7 @@ public ViewConnection(final CouchbaseConnectionFactory cf, this.user = user; this.password = password; connectionFactory = cf; - + boolean shouldClose = Boolean.parseBoolean(CouchbaseProperties.getProperty("closeViewConnections", "false")); viewNodes = Collections.synchronizedList(new ArrayList()); for (InetSocketAddress addr : seedAddrs) { viewNodes.add(createHttpHost(addr.getHostName(), addr.getPort())); @@ -156,7 +156,7 @@ public ViewConnection(final CouchbaseConnectionFactory cf, HttpProcessor httpProc = HttpProcessorBuilder.create() .add(new RequestContent()) .add(new RequestTargetHost()) - .add(new RequestConnControl()) + .add(new CustomRequestConnControl(shouldClose)) .add(new RequestUserAgent("JCBC/1.2")) .add(new RequestExpectContinue(true)).build(); diff --git a/src/main/java/com/couchbase/client/http/CustomRequestConnControl.java b/src/main/java/com/couchbase/client/http/CustomRequestConnControl.java new file mode 100644 index 00000000..7e16a0a9 --- /dev/null +++ b/src/main/java/com/couchbase/client/http/CustomRequestConnControl.java @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2009-2013 Couchbase, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALING + * IN THE SOFTWARE. + */ +package com.couchbase.client.http; + +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.protocol.HTTP; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.Args; + +import java.io.IOException; + +/** + * Custom connection control, derived from an example implementation and extended for + * optional non-persistent connections. + */ +public class CustomRequestConnControl implements HttpRequestInterceptor { + + private final boolean shouldClose; + + public CustomRequestConnControl(final boolean shouldClose) { + super(); + this.shouldClose = shouldClose; + } + + public void process(final HttpRequest request, final HttpContext context) + throws HttpException, IOException { + Args.notNull(request, "HTTP request"); + + final String method = request.getRequestLine().getMethod(); + if (method.equalsIgnoreCase("CONNECT")) { + return; + } + + if (!request.containsHeader(HTTP.CONN_DIRECTIVE)) { + if (shouldClose) { + request.addHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE); + } else { + request.addHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE); + } + } + } +} diff --git a/src/test/java/com/couchbase/client/http/CustomRequestConnControlTest.java b/src/test/java/com/couchbase/client/http/CustomRequestConnControlTest.java new file mode 100644 index 00000000..cfdbd774 --- /dev/null +++ b/src/test/java/com/couchbase/client/http/CustomRequestConnControlTest.java @@ -0,0 +1,64 @@ +/** + * Copyright (C) 2009-2013 Couchbase, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALING + * IN THE SOFTWARE. + */ +package com.couchbase.client.http; + +import org.apache.http.HttpRequest; +import org.apache.http.ProtocolVersion; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.protocol.HTTP; +import org.junit.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Verifies the functionality provided by {@link CustomRequestConnControl}. + */ +public class CustomRequestConnControlTest { + + @Test + public void shouldSetKeepAlive() throws Exception { + HttpRequest request = mock(HttpRequest.class); + when(request.getRequestLine()).thenReturn(new BasicRequestLine("GET", "/foo", + new ProtocolVersion("HTTP", 1, 1))); + + CustomRequestConnControl processor = new CustomRequestConnControl(false); + processor.process(request, null); + + verify(request, times(1)).addHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE); + } + + @Test + public void shouldSetClose() throws Exception { + HttpRequest request = mock(HttpRequest.class); + when(request.getRequestLine()).thenReturn(new BasicRequestLine("GET", "/foo", + new ProtocolVersion("HTTP", 1, 1))); + + CustomRequestConnControl processor = new CustomRequestConnControl(true); + processor.process(request, null); + + verify(request, times(1)).addHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE); + } + +} \ No newline at end of file From 98513344b55a2726f6d6fb39f4a6e785998a641e Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Fri, 26 Sep 2014 21:01:05 +0300 Subject: [PATCH 25/47] JCBC-566: Catch ConfigurationException to prevent IO loop death When CCCP protocol enabled, there is a likelihood that ConfigurationException will bubble up and kill the IO loop. When connectivity will be restored, the client cannot reconnect the nodes. Change-Id: I52de7e70008cddc602169356d4db0b592bcfb682 Reviewed-on: http://review.couchbase.org/41677 Reviewed-by: Matt Ingenthron Tested-by: Matt Ingenthron --- src/main/java/com/couchbase/client/CouchbaseConnection.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index 12ce6e08..af0a48fe 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -24,6 +24,7 @@ import com.couchbase.client.internal.AdaptiveThrottler; import com.couchbase.client.internal.ThrottleManager; +import com.couchbase.client.vbucket.ConfigurationException; import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.VBucketNodeLocator; import com.couchbase.client.vbucket.config.Bucket; @@ -332,6 +333,8 @@ public void run() { logRunException(e); } catch (ConcurrentModificationException e) { logRunException(e); + } catch (ConfigurationException e) { + getLogger().debug("Configuration unsuccessful. Will retry.", e); } } } From 5529322655b7056314b773e4afec2dbb0667d7b0 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Mon, 29 Sep 2014 02:49:36 +0300 Subject: [PATCH 26/47] JCBC-567: Remember initial bootstrap method Motivation ---------- In some situations like recovering connection after network outage, the client might stop using Carrier Publication protocol to receive cluster configuration. Modifications ------------- BucketConfigurationProvider.isBinary property refactored from simple boolean to type with three values to clearly describe the bootstrap protol or state that the bootstrap process have not decided on any of them. Result ------ When someone signals that configuration has been outdated (BucketConfigurationProvider.signalOutdated), the last method will be tried first, and then (if it was CCCP) the library will try to bootstrap from HTTP Change-Id: I6218ea6ceba7acddfcb4b43d81cc83a3cb6bbf6d Reviewed-on: http://review.couchbase.org/41734 Reviewed-by: Matt Ingenthron Tested-by: Sergey Avseyev --- .../provider/BootstrapProviderType.java | 49 +++++++++++++++++++ .../provider/BucketConfigurationProvider.java | 29 +++++------ 2 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 src/main/java/com/couchbase/client/vbucket/provider/BootstrapProviderType.java diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BootstrapProviderType.java b/src/main/java/com/couchbase/client/vbucket/provider/BootstrapProviderType.java new file mode 100644 index 00000000..b36d57b4 --- /dev/null +++ b/src/main/java/com/couchbase/client/vbucket/provider/BootstrapProviderType.java @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2014 Couchbase, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALING + * IN THE SOFTWARE. + */ + +package com.couchbase.client.vbucket.provider; + +public enum BootstrapProviderType { + + /** + * No bootstrap provider has been specified + */ + NONE, + + /** + * Configuration Carrier Publication Protocol (CCCP) + */ + CARRIER, + + /** + * HTTP-based protocol + */ + HTTP; + + public boolean isCarrier() { + return this.ordinal() == CARRIER.ordinal(); + } + + public boolean isHttp() { + return this.ordinal() == HTTP.ordinal(); + } +} diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index e1cb4242..5924ad58 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -65,7 +65,6 @@ public class BucketConfigurationProvider extends SpyObject implements ConfigurationProvider, Reconfigurable { - private static final int DEFAULT_BINARY_PORT = 11210; private static final String ANONYMOUS_BUCKET = "default"; @@ -82,7 +81,7 @@ public class BucketConfigurationProvider extends SpyObject private final AtomicReference binaryConnection; private final boolean disableCarrierBootstrap; private final boolean disableHttpBootstrap; - private volatile boolean isBinary; + private volatile BootstrapProviderType bootstrapProvider; private volatile long lastRevision; private volatile boolean shutdown; @@ -118,13 +117,13 @@ public Bucket bootstrap() { getLogger().debug("Omitting bootstrap since already shutdown."); } - isBinary = false; + bootstrapProvider = BootstrapProviderType.NONE; if (!bootstrapBinary() && !bootstrapHttp()) { throw new ConfigurationException("Could not fetch a valid Bucket " + "configuration."); } - if (isBinary) { + if (bootstrapProvider.isCarrier()) { getLogger().info("Could bootstrap through carrier publication."); } else { getLogger().info("Carrier config not available, bootstrapped through " @@ -148,7 +147,6 @@ boolean bootstrapBinary() { return false; } - isBinary = true; List nodes = new ArrayList(seedNodes.size()); for (URI seedNode : seedNodes) { @@ -158,18 +156,17 @@ boolean bootstrapBinary() { try { for (InetSocketAddress node : nodes) { if(tryBinaryBootstrapForNode(node)) { + bootstrapProvider = BootstrapProviderType.CARRIER; return true; } } getLogger().debug("Not a single node returned a carrier publication " + "config."); - isBinary = false; return false; } catch(Exception ex) { getLogger().info("Could not fetch config from carrier publication seed " + "nodes.", ex); - isBinary = false; return false; } } @@ -345,7 +342,7 @@ boolean bootstrapHttp() { Bucket config = httpProvider.get().getBucketConfiguration(bucket); setConfig(config); monitorBucket(); - isBinary = false; + bootstrapProvider = BootstrapProviderType.HTTP; return true; } catch(Exception ex) { getLogger().info("Could not fetch config from http seed nodes.", ex); @@ -358,7 +355,7 @@ boolean bootstrapHttp() { * used. */ private void monitorBucket() { - if (!shutdown && !isBinary) { + if (!shutdown && bootstrapProvider.isHttp()) { httpProvider.get().subscribe(bucket, this); } } @@ -391,7 +388,7 @@ public void setConfig(final Bucket config) { } } getLogger().debug("Applying new bucket config for bucket \"" + bucket - + "\" (carrier publication: " + isBinary + "): " + config); + + "\" (carrier publication: " + bootstrapProvider.isCarrier() + "): " + config); this.config.set(config); httpProvider.get().updateBucket(config.getName(), config); @@ -408,7 +405,7 @@ public void setConfig(final Bucket config) { * @param config the config to check. */ private void manageTaintedConfig(final Config config) { - if (!isBinary) { + if (bootstrapProvider != BootstrapProviderType.CARRIER) { return; } @@ -493,7 +490,7 @@ public void signalOutdated() { return; } - if (isBinary) { + if (bootstrapProvider.isCarrier()) { if (binaryConnection.get() == null) { bootstrap(); } else { @@ -514,6 +511,10 @@ public void signalOutdated() { } } } else { + if (disableHttpBootstrap) { + getLogger().info("Http bootstrap manually disabled, skipping."); + return; + } if (refreshingHttp.compareAndSet(false, true)) { Thread refresherThread = new Thread(new HttpProviderRefresher()); refresherThread.setName("HttpConfigurationProvider Reloader"); @@ -526,7 +527,7 @@ public void signalOutdated() { @Override public void reloadConfig() { - if (isBinary && !shutdown) { + if (bootstrapProvider.isCarrier() && !shutdown) { signalOutdated(); } } @@ -646,7 +647,7 @@ class BinaryConfigPoller implements Runnable { @Override public void run() { try { - while (isBinary && getConfig().getConfig().isTainted()) { + while (bootstrapProvider.isCarrier() && getConfig().getConfig().isTainted()) { getLogger().debug("Polling for new carrier configuration and " + "waiting " + waitPeriod + "ms (Attempt " + ++attempt + ")."); signalOutdated(); From fa5b5241ddfd880bdd6576962cf10cea99236213 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Mon, 29 Sep 2014 03:20:45 +0300 Subject: [PATCH 27/47] Do not reset thresholdLastCheck on each iteration Change-Id: I3d28ad5f8014cb46f1d3e9bd5e2beb8bde9add8a Reviewed-on: http://review.couchbase.org/41740 Tested-by: Sergey Avseyev Reviewed-by: Matt Ingenthron --- .../java/com/couchbase/client/CouchbaseConnectionFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java index 6280ab57..e9543aac 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java @@ -390,9 +390,9 @@ protected boolean pastReconnThreshold() { if (currentTime - thresholdLastCheck >= TimeUnit.SECONDS.toNanos(10)) { configThresholdCount.set(0); + thresholdLastCheck = currentTime; } - thresholdLastCheck = currentTime; if (configThresholdCount.incrementAndGet() >= maxConfigCheck) { return true; } From a7fcf503d603579bf5d96ebfe22e6c3d7bc46c7d Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 23 Sep 2014 23:21:37 +0300 Subject: [PATCH 28/47] JCBC-566: Remove dead code blocking configuration updates In this patch http://review.couchbase.org/32589 Resubscriber class was removed, but its flag was not. So the boolean state is never changed here. Change-Id: I9509e207d1c1c9215ac22de0bd57022dd9270aec Reviewed-on: http://review.couchbase.org/41741 Tested-by: Sergey Avseyev Reviewed-by: Matt Ingenthron --- .../com/couchbase/client/CouchbaseConnectionFactory.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java index e9543aac..03d9bbc4 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java @@ -151,7 +151,6 @@ public class CouchbaseConnectionFactory extends BinaryConnectionFactory { private static final Logger LOGGER = Logger.getLogger(CouchbaseConnectionFactory.class.getName()); private volatile boolean needsReconnect; - private final AtomicBoolean doingResubscribe = new AtomicBoolean(false); private volatile long thresholdLastCheck = System.nanoTime(); private final AtomicInteger configThresholdCount = new AtomicInteger(0); private final int maxConfigCheck = 10; //maximum allowed checks before we @@ -360,12 +359,7 @@ void checkConfigUpdate() { return; } - if (doingResubscribe.compareAndSet(false, true)) { - getConfigurationProvider().signalOutdated(); - } else { - LOGGER.log(Level.CONFIG, "Duplicate resubscribe for config updates" - + " suppressed."); - } + getConfigurationProvider().signalOutdated(); } else { LOGGER.log(Level.FINE, "No reconnect required, though check requested." + " Current config check is {0} out of a threshold of {1}.", From 5e233d219b24dd6080c4c99d77b95163ee3dce57 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 30 Sep 2014 23:46:51 +0300 Subject: [PATCH 29/47] Initialize bootstrapProvider member Fixes NPE in reloadConfig() public void reloadConfig() { if (bootstrapProvider.isCarrier() && !shutdown) { ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Change-Id: Ie88aeb0624968656e89819de2b931832843e0ad3 Reviewed-on: http://review.couchbase.org/41786 Reviewed-by: Matt Ingenthron Tested-by: Sergey Avseyev --- .../client/vbucket/provider/BucketConfigurationProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index 5924ad58..63606058 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -81,7 +81,7 @@ public class BucketConfigurationProvider extends SpyObject private final AtomicReference binaryConnection; private final boolean disableCarrierBootstrap; private final boolean disableHttpBootstrap; - private volatile BootstrapProviderType bootstrapProvider; + private volatile BootstrapProviderType bootstrapProvider = BootstrapProviderType.NONE; private volatile long lastRevision; private volatile boolean shutdown; From 7662ee138e3a3d74f7d67a78d2ba89d8c6bc282d Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Tue, 26 Aug 2014 16:02:13 +0200 Subject: [PATCH 30/47] JCBC-531: Add Diagnostics and dump them on startup. Change-Id: Ib8a3719ac0380cf64486bbea8fb208bb9606f770 Reviewed-on: http://review.couchbase.org/40913 Reviewed-by: Matt Ingenthron Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- .../com/couchbase/client/CouchbaseClient.java | 1 + .../com/couchbase/client/Diagnostics.java | 159 ++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 src/main/java/com/couchbase/client/Diagnostics.java diff --git a/src/main/java/com/couchbase/client/CouchbaseClient.java b/src/main/java/com/couchbase/client/CouchbaseClient.java index dab78eef..2d422c81 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClient.java +++ b/src/main/java/com/couchbase/client/CouchbaseClient.java @@ -257,6 +257,7 @@ public CouchbaseClient(CouchbaseConnectionFactory cf) throws IOException { super(cf, AddrUtil.getAddresses(cf.getVBucketConfig().getServers())); getLogger().info(cf.toString()); + getLogger().debug(Diagnostics.collectAndFormat()); cbConnFactory = cf; diff --git a/src/main/java/com/couchbase/client/Diagnostics.java b/src/main/java/com/couchbase/client/Diagnostics.java new file mode 100644 index 00000000..c8030684 --- /dev/null +++ b/src/main/java/com/couchbase/client/Diagnostics.java @@ -0,0 +1,159 @@ +/** + * Copyright (C) 2014 Couchbase, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALING + * IN THE SOFTWARE. + */ +package com.couchbase.client; + + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; +import java.lang.management.ThreadMXBean; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Provides access to various metrics helpful for system diagnosis. + */ +public class Diagnostics { + + public static final OperatingSystemMXBean OS_BEAN = ManagementFactory.getOperatingSystemMXBean(); + public static final MemoryMXBean MEM_BEAN = ManagementFactory.getMemoryMXBean(); + public static final RuntimeMXBean RUNTIME_BEAN = ManagementFactory.getRuntimeMXBean(); + public static final ThreadMXBean THREAD_BEAN = ManagementFactory.getThreadMXBean(); + + /** + * Collects system information as delivered from the {@link OperatingSystemMXBean}. + * + * @param infos a map where the infos are passed in. + */ + public static void systemInfo(final Map infos) { + infos.put("sys.os.name", OS_BEAN.getName()); + infos.put("sys.os.version", OS_BEAN.getVersion()); + infos.put("sys.os.arch", OS_BEAN.getArch()); + infos.put("sys.cpu.num", OS_BEAN.getAvailableProcessors()); + infos.put("sys.cpu.loadAvg", OS_BEAN.getSystemLoadAverage()); + + if (OS_BEAN instanceof com.sun.management.OperatingSystemMXBean) { + com.sun.management.OperatingSystemMXBean sunBean = (com.sun.management.OperatingSystemMXBean) OS_BEAN; + + infos.put("proc.cpu.time", sunBean.getProcessCpuTime()); + infos.put("mem.physical.total", sunBean.getTotalPhysicalMemorySize()); + infos.put("mem.physical.free", sunBean.getFreePhysicalMemorySize()); + infos.put("mem.virtual.comitted", sunBean.getCommittedVirtualMemorySize()); + infos.put("mem.swap.total", sunBean.getTotalSwapSpaceSize()); + infos.put("mem.swap.free", sunBean.getFreeSwapSpaceSize()); + } + } + + /** + * Collects system information as delivered from the {@link GarbageCollectorMXBean}. + * + * @param infos a map where the infos are passed in. + */ + public static void gcInfo(final Map infos) { + List mxBeans = ManagementFactory.getGarbageCollectorMXBeans(); + + for (GarbageCollectorMXBean mxBean : mxBeans) { + infos.put("gc." + mxBean.getName().toLowerCase() + ".collectionCount", mxBean.getCollectionCount()); + infos.put("gc." + mxBean.getName().toLowerCase() + ".collectionTime", mxBean.getCollectionTime()); + } + } + + /** + * Collects system information as delivered from the {@link MemoryMXBean}. + * + * @param infos a map where the infos are passed in. + */ + public static void memInfo(final Map infos) { + infos.put("heap.used", MEM_BEAN.getHeapMemoryUsage()); + infos.put("offHeap.used", MEM_BEAN.getNonHeapMemoryUsage()); + infos.put("heap.pendingFinalize", MEM_BEAN.getObjectPendingFinalizationCount()); + } + + /** + * Collects system information as delivered from the {@link RuntimeMXBean}. + * + * @param infos a map where the infos are passed in. + */ + public static void runtimeInfo(final Map infos) { + infos.put("runtime.vm", RUNTIME_BEAN.getVmVendor() + "/" + RUNTIME_BEAN.getVmName() + ": " + + RUNTIME_BEAN.getVmVersion()); + infos.put("runtime.startTime", RUNTIME_BEAN.getStartTime()); + infos.put("runtime.uptime", RUNTIME_BEAN.getUptime()); + infos.put("runtime.name", RUNTIME_BEAN.getName()); + infos.put("runtime.spec", RUNTIME_BEAN.getSpecVendor() + "/" + RUNTIME_BEAN.getSpecName() + ": " + + RUNTIME_BEAN.getSpecVersion()); + infos.put("runtime.sysProperties", RUNTIME_BEAN.getSystemProperties()); + } + + /** + * Collects system information as delivered from the {@link ThreadMXBean}. + * + * @param infos a map where the infos are passed in. + */ + public static void threadInfo(final Map infos) { + infos.put("thread.count", THREAD_BEAN.getThreadCount()); + infos.put("thread.peakCount", THREAD_BEAN.getPeakThreadCount()); + infos.put("thread.startedCount", THREAD_BEAN.getTotalStartedThreadCount()); + } + + /** + * Collects all available infos in one map. + * + * @return the map populated with the information. + */ + public static Map collect() { + Map infos = new TreeMap(); + + systemInfo(infos); + memInfo(infos); + threadInfo(infos); + gcInfo(infos); + runtimeInfo(infos); + + return infos; + } + + /** + * Collects all available infos and formats it in a better readable way. + * + * @return a formatted string of available information. + */ + public static String collectAndFormat() { + Map infos = collect(); + + StringBuilder sb = new StringBuilder(); + + sb.append("Diagnostics {\n"); + int count = 0; + for (Map.Entry info : infos.entrySet()) { + if (count++ > 0) { + sb.append(",\n"); + } + sb.append(" ").append(info.getKey()).append("=").append(info.getValue()); + } + sb.append("\n}"); + return sb.toString(); + } +} From c4fa34c32d67193125c8828ebaef25484c5d55ad Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 3 Nov 2014 15:22:06 +0100 Subject: [PATCH 31/47] JCBC-620: Make Diagnostics resilient for not found classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- The code uses com/sun namespaced packages - which we shouldn't be using - but are able to optionally utilize them to get more diagnostics information. The code previously did check if the classes are proper instances, but it was not resilient for environments where the class doesn't exist at all (that is on IBM JVMs, as well as OSGi containers and app servers like Wildfly). It prevents starting an application. Modifications ------------- The code now properly catches such an exception and logs a debug notice that extended info is not available and only reduced output is printed. Result ------ Properly degrading output and making sure app servers with reduced context are able to start. Change-Id: I7c3b2b22bd42dbc28ff46d5db6a3ca413aa6f744 Reviewed-on: http://review.couchbase.org/42711 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../com/couchbase/client/Diagnostics.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/couchbase/client/Diagnostics.java b/src/main/java/com/couchbase/client/Diagnostics.java index c8030684..f266f977 100644 --- a/src/main/java/com/couchbase/client/Diagnostics.java +++ b/src/main/java/com/couchbase/client/Diagnostics.java @@ -22,6 +22,9 @@ package com.couchbase.client; +import net.spy.memcached.compat.log.Logger; +import net.spy.memcached.compat.log.LoggerFactory; + import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; @@ -37,6 +40,8 @@ */ public class Diagnostics { + private static final Logger LOGGER = LoggerFactory.getLogger(Diagnostics.class); + public static final OperatingSystemMXBean OS_BEAN = ManagementFactory.getOperatingSystemMXBean(); public static final MemoryMXBean MEM_BEAN = ManagementFactory.getMemoryMXBean(); public static final RuntimeMXBean RUNTIME_BEAN = ManagementFactory.getRuntimeMXBean(); @@ -54,15 +59,19 @@ public static void systemInfo(final Map infos) { infos.put("sys.cpu.num", OS_BEAN.getAvailableProcessors()); infos.put("sys.cpu.loadAvg", OS_BEAN.getSystemLoadAverage()); - if (OS_BEAN instanceof com.sun.management.OperatingSystemMXBean) { - com.sun.management.OperatingSystemMXBean sunBean = (com.sun.management.OperatingSystemMXBean) OS_BEAN; + try { + if (OS_BEAN instanceof com.sun.management.OperatingSystemMXBean) { + com.sun.management.OperatingSystemMXBean sunBean = (com.sun.management.OperatingSystemMXBean) OS_BEAN; - infos.put("proc.cpu.time", sunBean.getProcessCpuTime()); - infos.put("mem.physical.total", sunBean.getTotalPhysicalMemorySize()); - infos.put("mem.physical.free", sunBean.getFreePhysicalMemorySize()); - infos.put("mem.virtual.comitted", sunBean.getCommittedVirtualMemorySize()); - infos.put("mem.swap.total", sunBean.getTotalSwapSpaceSize()); - infos.put("mem.swap.free", sunBean.getFreeSwapSpaceSize()); + infos.put("proc.cpu.time", sunBean.getProcessCpuTime()); + infos.put("mem.physical.total", sunBean.getTotalPhysicalMemorySize()); + infos.put("mem.physical.free", sunBean.getFreePhysicalMemorySize()); + infos.put("mem.virtual.comitted", sunBean.getCommittedVirtualMemorySize()); + infos.put("mem.swap.total", sunBean.getTotalSwapSpaceSize()); + infos.put("mem.swap.free", sunBean.getFreeSwapSpaceSize()); + } + } catch(final Throwable err) { + LOGGER.debug("com.sun.management.OperatingSystemMXBean not available, skipping extended system info."); } } From 13f0db05294364dcbdf99e16d8f6405b19b0ae6c Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 1 Dec 2014 08:20:48 +0100 Subject: [PATCH 32/47] JCBC-641: Upgrade Spymemcached to 2.11.5 Motivation ---------- Spymemcached 2.11.5 contains two important fixes, including one where reconnect can take longer than necessary. Modifications ------------- Upgrade the dependency. Change-Id: I40d4b73c9e0579088b5d35c284567d5168b06d9f Reviewed-on: http://review.couchbase.org/43756 Reviewed-by: Sergey Avseyev Tested-by: Michael Nitschinger --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index aef5e872..d658c2d0 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.4 -spymemcached-test.version=2.11.4 +spymemcached.version=2.11.5 +spymemcached-test.version=2.11.5 From fa1b11619c39c71c4a2083d60313b7f5950d9351 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Tue, 9 Dec 2014 09:06:02 +0100 Subject: [PATCH 33/47] JCBC-647: Avoid WARN level log on failed over replica with observe. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- During a failover/error condition, the actual number of replicas will differ from the configured ones. There has always been a check in place to not send a request to an inactive replica (identified by -1 in the config). The code path used printed out a WARN which, under high traffic, can seriously crowd the logs. Modifications ------------- The Locator already provides a pre-check to only return the active replicas, which is used by replica get queries. This change also makes the code use this path, only looping through the active replicas and avoiding the very verbose WARN log message over and over again. Result ------ Less verbose logs, making it easier to discover actual problems. Change-Id: Ie9577fd55f6171406a0176c8b7ec7a4ee4db4bcc Reviewed-on: http://review.couchbase.org/44141 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- src/main/java/com/couchbase/client/CouchbaseClient.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseClient.java b/src/main/java/com/couchbase/client/CouchbaseClient.java index 2d422c81..3b8f3eaf 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClient.java +++ b/src/main/java/com/couchbase/client/CouchbaseClient.java @@ -1657,8 +1657,9 @@ private Map observe(final String key, } if (toReplica) { - for (int i = 0; i < cfg.getReplicasCount(); i++) { - MemcachedNode replica = locator.getReplica(key, i); + List replicaIndexes = locator.getReplicaIndexes(key); + for (int index : replicaIndexes) { + MemcachedNode replica = locator.getReplica(key, index); if (replica != null) { bcastNodes.add(replica); } From 14bd22c00565e0018d2fe5db975f7c1d0b20fc35 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 12 Jan 2015 16:17:19 +0100 Subject: [PATCH 34/47] JCBC-681: Detect missing noop polls and rebootstrap config. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- If the node where the carrier config is attached to, dies silently, the config channel is kept open longer than its doing any good. We are already doing NOOP polls on a 5 second interval to make sure the connection does not get dropped by firewalls, so we can piggyback the responses to trigger an outdated config signal when they do not come back anymore. Modifications ------------- The changeset piggybacks on the NOOP interval which is done anyways and adds a simple counting mechanism. When more than (by default) 3 NOOPs are missed, after 3*5seconds a "outdated config" signal is issued to the configuration provider. Once the provider tries to grab a config but can't (since it won't return him one on the dead node), a full rebootstrap is initiated. Result ------ The client now switches over to another node in the cluster even before auto failover kicks in, so when it does it can very quickly retrieve a valid configuration and get back to a stable state. Change-Id: I7004d1923bf43cf899113da48dd8aa146a458625 Reviewed-on: http://review.couchbase.org/45214 Reviewed-by: Simon Baslé Tested-by: Michael Nitschinger --- .../couchbase/client/CouchbaseConnection.java | 8 +-- .../client/CouchbaseConnectionFactory.java | 32 ++++------ .../provider/BucketConfigurationProvider.java | 20 ++++++- .../provider/CouchbaseConfigConnection.java | 60 ++++++++++++++++++- 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index af0a48fe..abbea6b0 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -69,14 +69,14 @@ public class CouchbaseConnection extends MemcachedConnection implements * The amount in seconds after which a op broadcast is forced to detect * dead connections. */ - private static final int ALLOWED_IDLE_TIME = 5; + protected static final int ALLOWED_IDLE_TIME = 5; protected volatile boolean reconfiguring = false; - private final CouchbaseConnectionFactory cf; + protected final CouchbaseConnectionFactory cf; private final ThrottleManager throttleManager; private final boolean enableThrottling; - private volatile long lastWrite; + protected volatile long lastWrite; public CouchbaseConnection(int bufSize, CouchbaseConnectionFactory f, List a, Collection obs, @@ -412,7 +412,7 @@ private void logRunException(Exception e) { /** * Helper method to centralize updating the last write timestamp. */ - private void updateLastWrite() { + protected void updateLastWrite() { long now = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()); if (lastWrite != now) { lastWrite = now; diff --git a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java index 03d9bbc4..53e25abd 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java @@ -22,43 +22,35 @@ package com.couchbase.client; -import com.couchbase.client.vbucket.ConfigurationException; -import com.couchbase.client.vbucket.provider.BucketConfigurationProvider; -import com.couchbase.client.vbucket.provider.ConfigurationProvider; -import com.couchbase.client.vbucket.ConfigurationProviderHTTP; import com.couchbase.client.vbucket.CouchbaseNodeOrder; import com.couchbase.client.vbucket.Reconfigurable; import com.couchbase.client.vbucket.VBucketNodeLocator; -import com.couchbase.client.vbucket.config.Bucket; import com.couchbase.client.vbucket.config.Config; import com.couchbase.client.vbucket.config.ConfigType; - +import com.couchbase.client.vbucket.provider.BucketConfigurationProvider; +import com.couchbase.client.vbucket.provider.ConfigurationProvider; +import net.spy.memcached.BinaryConnectionFactory; +import net.spy.memcached.DefaultHashAlgorithm; +import net.spy.memcached.FailureMode; +import net.spy.memcached.HashAlgorithm; +import net.spy.memcached.KetamaNodeLocator; +import net.spy.memcached.MemcachedConnection; +import net.spy.memcached.MemcachedNode; +import net.spy.memcached.NodeLocator; +import net.spy.memcached.auth.AuthDescriptor; +import net.spy.memcached.auth.PlainCallbackHandler; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -import net.spy.memcached.BinaryConnectionFactory; -import net.spy.memcached.DefaultHashAlgorithm; -import net.spy.memcached.FailureMode; -import net.spy.memcached.HashAlgorithm; -import net.spy.memcached.KetamaNodeLocator; -import net.spy.memcached.MemcachedConnection; -import net.spy.memcached.MemcachedNode; -import net.spy.memcached.NodeLocator; -import net.spy.memcached.auth.AuthDescriptor; -import net.spy.memcached.auth.PlainCallbackHandler; - /** * Couchbase implementation of ConnectionFactory. * diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index 63606058..b12f98d1 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -43,7 +43,6 @@ import net.spy.memcached.ops.Operation; import net.spy.memcached.ops.OperationCallback; import net.spy.memcached.ops.OperationStatus; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -118,6 +117,14 @@ public Bucket bootstrap() { } bootstrapProvider = BootstrapProviderType.NONE; + CouchbaseConnection oldBinaryConnection = binaryConnection.getAndSet(null); + if (oldBinaryConnection != null) { + try { + oldBinaryConnection.shutdown(); + } catch (IOException e) { + getLogger().warn("Failed to shutdown old binary config connection.", e); + } + } if (!bootstrapBinary() && !bootstrapHttp()) { throw new ConfigurationException("Could not fetch a valid Bucket " + "configuration."); @@ -279,6 +286,8 @@ public void connectionLost(SocketAddress sa) { if (old != null) { old.shutdown(); } + + getLogger().debug("Properly bootstrapped carrier config through node: " + node.getHostName()); binaryConnection.set(connection); return true; } @@ -490,7 +499,7 @@ public void signalOutdated() { return; } - if (bootstrapProvider.isCarrier()) { + if (bootstrapProvider.isCarrier() || bootstrapProvider == BootstrapProviderType.NONE) { if (binaryConnection.get() == null) { bootstrap(); } else { @@ -666,13 +675,18 @@ public void run() { } } - static class ConfigurationConnectionFactory + class ConfigurationConnectionFactory extends CouchbaseConnectionFactory { ConfigurationConnectionFactory(List baseList, String bucketName, String password) throws IOException { super(baseList, bucketName, password); } + @Override + public synchronized ConfigurationProvider getConfigurationProvider() { + return BucketConfigurationProvider.this; + } + @Override public NodeLocator createLocator(List nodes) { return new ArrayModNodeLocator(nodes, getHashAlg()); diff --git a/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java b/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java index 70bf9653..785668a1 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/CouchbaseConfigConnection.java @@ -24,6 +24,8 @@ import com.couchbase.client.CouchbaseConnection; import com.couchbase.client.CouchbaseConnectionFactory; +import com.couchbase.client.CouchbaseProperties; +import net.spy.memcached.BroadcastOpFactory; import net.spy.memcached.ConnectionObserver; import net.spy.memcached.FailureMode; import net.spy.memcached.MemcachedNode; @@ -31,11 +33,15 @@ import net.spy.memcached.compat.log.Level; import net.spy.memcached.compat.log.Logger; import net.spy.memcached.compat.log.LoggerFactory; - +import net.spy.memcached.ops.Operation; +import net.spy.memcached.ops.OperationCallback; +import net.spy.memcached.ops.OperationStatus; import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collection; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * A custom {@link CouchbaseConnection} that is used for binary config @@ -47,13 +53,28 @@ */ public class CouchbaseConfigConnection extends CouchbaseConnection { - private static final Logger LOGGER = new LoggerProxy(LoggerFactory.getLogger( - CouchbaseConfigConnection.class));; + /** + * If 3 consecutive NOOPS are missed, an outdated config is signaled. + */ + private static final String DEFAULT_MISSED_NOOPS_THRESHOLD = "3"; + private final short noopsThreshold; + private volatile short outstandingNoops = 0; + + private static final Logger LOGGER = new LoggerProxy( + LoggerFactory.getLogger(CouchbaseConfigConnection.class) + ); public CouchbaseConfigConnection(int bufSize, CouchbaseConnectionFactory f, List a, Collection obs, FailureMode fm, OperationFactory opfactory) throws IOException { super(bufSize, f, a, obs, fm, opfactory); + + noopsThreshold = Short.decode(CouchbaseProperties.getProperty( + "configPollThreshold", + DEFAULT_MISSED_NOOPS_THRESHOLD + )); + + LOGGER.debug("Using config noop threshold of " + noopsThreshold); } @Override @@ -67,6 +88,39 @@ public String toString() { return sb.toString(); } + @Override + protected void handleWokenUpSelector() { + long now = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()); + long diff = now - lastWrite; + if (lastWrite > 0 && diff >= ALLOWED_IDLE_TIME) { + if (outstandingNoops >= noopsThreshold) { + cf.getConfigurationProvider().signalOutdated(); + outstandingNoops = 0; + } + + updateLastWrite(); + getLogger().debug("Wakeup counter triggered, broadcasting noops."); + final OperationFactory fact = cf.getOperationFactory(); + outstandingNoops++; + broadcastOperation(new BroadcastOpFactory() { + @Override + public Operation newOp(MemcachedNode n, final CountDownLatch latch) { + return fact.noop(new OperationCallback() { + @Override + public void receivedStatus(OperationStatus status) { + if (status.isSuccess() && outstandingNoops > 0) { + outstandingNoops--; + } + } + + @Override + public void complete() {} + }); + } + }); + } + } + @Override protected Logger getLogger() { return LOGGER; From 84e1d041aa417c933df23803fa9063211e5a5026 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 19 Jan 2015 11:15:09 +0100 Subject: [PATCH 35/47] Fix unit tests with > 3.0.0 compat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation & Modifications -------------------------- Spatial views have changed and currently it is not planned to support the new format going forward on the 1.4 branch. It needs to be conditionally disabled. Also, the test config did incorrectly parse out the version of the nodes, this has been corrected by backporting the Version regex from 2.1. Change-Id: I43e9ad9ab2a48ec7fb5394564d380557cf869665 Reviewed-on: http://review.couchbase.org/45557 Reviewed-by: Simon Baslé Tested-by: Michael Nitschinger --- .../com/couchbase/client/CbTestConfig.java | 37 +++++++++++++++---- .../com/couchbase/client/SpatialViewTest.java | 27 +++++++++++--- .../BucketConfigurationProviderTest.java | 17 +++------ 3 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/test/java/com/couchbase/client/CbTestConfig.java b/src/test/java/com/couchbase/client/CbTestConfig.java index e70d46d5..103a79d4 100644 --- a/src/test/java/com/couchbase/client/CbTestConfig.java +++ b/src/test/java/com/couchbase/client/CbTestConfig.java @@ -22,6 +22,9 @@ package com.couchbase.client; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * A typical configuration for the test suite. */ @@ -59,26 +62,46 @@ private CbTestConfig() { */ public static class Version { + private static final Pattern VERSION_PATTERN = Pattern.compile("^(\\d+)(?:\\.(\\d+))?(?:\\.(\\d+))?.*"); + private final int major; private final int minor; private final int bugfix; public Version(String raw) { - String[] tokens = raw.replaceAll("_.*$", "").split("\\."); - major = Integer.parseInt(tokens[0]); - minor = Integer.parseInt(tokens[1]); - bugfix = Integer.parseInt(tokens[2]); + Matcher matcher = VERSION_PATTERN.matcher(raw); + if (matcher.matches() && matcher.groupCount() == 3) { + major = Integer.parseInt(matcher.group(1)); + minor = matcher.group(2) != null ? Integer.parseInt(matcher.group(2)) : 0; + bugfix = matcher.group(3) != null ? Integer.parseInt(matcher.group(3)) : 0; + } else { + throw new IllegalArgumentException( + "Expected a version string starting with X[.Y[.Z]], was " + raw); + } } public boolean greaterOrEqualThan(int major, int minor, int bugfix) { - return this.major >= major - && this.minor >= minor - && this.bugfix >= bugfix; + if (this.major > major) { + return true; + } else if (this.major == major) { + if (this.minor > minor) { + return true; + } else if (this.minor == minor) { + if (this.bugfix >= bugfix) { + return true; + } + } + } + return false; } public boolean isCarrierConfigAware() { return greaterOrEqualThan(2, 5, 0); } + + public boolean isOldSpatialAware() { + return !greaterOrEqualThan(3, 0, 0); + } } } diff --git a/src/test/java/com/couchbase/client/SpatialViewTest.java b/src/test/java/com/couchbase/client/SpatialViewTest.java index 9984581e..f08aa646 100644 --- a/src/test/java/com/couchbase/client/SpatialViewTest.java +++ b/src/test/java/com/couchbase/client/SpatialViewTest.java @@ -30,19 +30,22 @@ import com.couchbase.client.protocol.views.Stale; import com.couchbase.client.protocol.views.ViewResponse; import com.couchbase.client.protocol.views.ViewRow; -import java.net.URI; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.TimeUnit; import net.spy.memcached.PersistTo; import net.spy.memcached.TestConfig; +import net.spy.memcached.compat.log.Logger; +import net.spy.memcached.compat.log.LoggerFactory; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.net.URI; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -52,12 +55,17 @@ * Verifies the correct functionality of spatial view queries. */ public class SpatialViewTest { + + private static final Logger LOGGER = + LoggerFactory.getLogger(SpatialViewTest.class); + protected static TestingClient client = null; private static final ArrayList CITY_DOCS; private static final String SERVER_URI = "http://" + TestConfig.IPV4_ADDR + ":8091/pools"; public static final String DESIGN_DOC = "cities"; public static final String VIEW_NAME_SPATIAL = "all_cities"; + private static boolean isOldSpatial = false; static { CITY_DOCS = new ArrayList(); @@ -148,6 +156,15 @@ public String success(long elapsedTime) { bucketTool.poll(callback); bucketTool.waitForWarmup(client); + ArrayList versions = new ArrayList( + client.getVersions().values()); + if (versions.size() > 0) { + CbTestConfig.Version version = new CbTestConfig.Version(versions.get(0)); + isOldSpatial = version.isOldSpatialAware(); + } + + Assume.assumeTrue(isOldSpatial); + String docUri = "/default/_design/" + TestingClient.MODE_PREFIX + DESIGN_DOC; String view = "{\"language\":\"javascript\",\"spatial\":{\"" diff --git a/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java b/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java index 506440c6..f7d3b2b6 100644 --- a/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java +++ b/src/test/java/com/couchbase/client/vbucket/provider/BucketConfigurationProviderTest.java @@ -31,6 +31,7 @@ import net.spy.memcached.TestConfig; import net.spy.memcached.compat.log.Logger; import net.spy.memcached.compat.log.LoggerFactory; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -104,10 +105,7 @@ public void resetProperties() { @Test public void shouldBootstrapBothBinaryAndHttp() throws Exception { - if (!isCCCPAware) { - LOGGER.info("Skipping Test because cluster is not CCCP aware."); - return; - } + Assume.assumeTrue(isCCCPAware); BucketConfigurationProvider provider = new BucketConfigurationProvider( seedNodes, @@ -162,10 +160,8 @@ public void shouldReloadHttpConfigOnSignalOutdated() throws Exception { @Test public void shouldReloadBinaryConfigOnSignalOutdated() throws Exception { - if (!isCCCPAware) { - LOGGER.info("Skipping Test because cluster is not CCCP aware."); - return; - } + Assume.assumeTrue(isCCCPAware); + List seedNodes = Arrays.asList( new URI("http://foobar:8091/pools"), @@ -209,10 +205,7 @@ public void shouldIgnoreInvalidNodeOnBootstrap() throws Exception { @Test public void shouldSkipBinaryOnManualDisable() throws Exception { - if (!isCCCPAware) { - LOGGER.info("Skipping Test because cluster is not CCCP aware."); - return; - } + Assume.assumeTrue(isCCCPAware); System.setProperty("cbclient.disableCarrierBootstrap", "true"); BucketConfigurationProvider provider = new BucketConfigurationProvider( From 4cede5077846ed903e9a504703a461c0c8ca4e24 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 19 Jan 2015 10:46:32 +0100 Subject: [PATCH 36/47] Depend on Spymemcached 2.11.6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I2e12877d0885482d7f8b66624021c6633f12ac92 Reviewed-on: http://review.couchbase.org/45555 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index d658c2d0..5d961067 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.5 -spymemcached-test.version=2.11.5 +spymemcached.version=2.11.6 +spymemcached-test.version=2.11.6 From f004c6cc5c8ca78392723f97f2cf6b25f6325bb5 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 16 Feb 2015 08:18:52 +0100 Subject: [PATCH 37/47] JCBC-700: Set ObserveFuture isDone() properly. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- When a ObserveFuture is returned (the underlying one which is always returned when PersistTo or ReplicateTo overloads are used) the current code always returns false on isDone, even when done. Modifications ------------- Overriding the set() method which is called by the callback to set the proper status and count down the latch to also set the "done" flag. Adding a test case to verify its working. Result ------ Proper behavior when calling isDone() on observe overloads. Change-Id: I783d10753c1cbe7bbba4d3940922b78161c9023e Reviewed-on: http://review.couchbase.org/46873 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../client/internal/ObserveFuture.java | 7 +++++ .../couchbase/client/CouchbaseClientTest.java | 28 ++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/couchbase/client/internal/ObserveFuture.java b/src/main/java/com/couchbase/client/internal/ObserveFuture.java index 90be085c..033b1d3c 100644 --- a/src/main/java/com/couchbase/client/internal/ObserveFuture.java +++ b/src/main/java/com/couchbase/client/internal/ObserveFuture.java @@ -23,6 +23,7 @@ package com.couchbase.client.internal; import net.spy.memcached.internal.OperationFuture; +import net.spy.memcached.ops.OperationStatus; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -65,4 +66,10 @@ public boolean isCancelled() { public boolean isDone() { return done; } + + @Override + public void set(T o, OperationStatus s) { + super.set(o, s); + done = true; + } } diff --git a/src/test/java/com/couchbase/client/CouchbaseClientTest.java b/src/test/java/com/couchbase/client/CouchbaseClientTest.java index 9ff50f38..156e59e7 100644 --- a/src/test/java/com/couchbase/client/CouchbaseClientTest.java +++ b/src/test/java/com/couchbase/client/CouchbaseClientTest.java @@ -25,17 +25,6 @@ import com.couchbase.client.BucketTool.FunctionCallback; import com.couchbase.client.clustermanager.BucketType; - -import java.net.SocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - import net.spy.memcached.BinaryClientTest; import net.spy.memcached.CASResponse; import net.spy.memcached.CASValue; @@ -48,10 +37,19 @@ import net.spy.memcached.internal.OperationCompletionListener; import net.spy.memcached.internal.OperationFuture; import net.spy.memcached.ops.OperationStatus; - import org.junit.Ignore; -import static org.junit.Assume.assumeTrue; import org.junit.Test; +import java.net.SocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assume.assumeTrue; /** * A CouchbaseClientTest. @@ -471,7 +469,11 @@ public void testAsyncCASObserve() throws Exception { OperationFuture setFuture = client.set(key, "value", PersistTo.MASTER); + assertFalse(setFuture.isDone()); assertTrue(setFuture.get()); + assertTrue(setFuture.isDone()); + assertTrue(setFuture.getStatus().isSuccess()); + final CountDownLatch latch = new CountDownLatch(1); client.asyncCas(key, setFuture.getCas(), "value2", PersistTo.MASTER) From 4a07360dbd1e40df7d02f7503626ffe866744a5b Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Wed, 1 Apr 2015 11:21:55 +0200 Subject: [PATCH 38/47] JCBC-741: Always poll master on observe. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- To consistently detect concurrent modifications of the observed document, the client needs to always poll the master cas state, since it can't be reliably detected on the client. So even when just ReplicateTo.* is used, the master is polled and used for a cas crosscheck. Modifications ------------- The isMaster() check blocks have been removed which will enable master polling all the time, leading to cas mismatch detection. Result ------ Consistent concurrent modification check behaviour, independent if PersistTo is used or not. Change-Id: Ia48cb827ff90a1d7511a42fd10fd058dd3dcb845 Reviewed-on: http://review.couchbase.org/49023 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../java/com/couchbase/client/CouchbaseClient.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseClient.java b/src/main/java/com/couchbase/client/CouchbaseClient.java index 3b8f3eaf..5558f5d7 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClient.java +++ b/src/main/java/com/couchbase/client/CouchbaseClient.java @@ -1642,19 +1642,17 @@ public void onComplete(OperationFuture future) throws Exception { } private Map observe(final String key, - final long cas, final boolean toMaster, final boolean toReplica) { + final long cas, final boolean toReplica) { Config cfg = ((CouchbaseConnectionFactory) connFactory).getVBucketConfig(); VBucketNodeLocator locator = (VBucketNodeLocator) mconn.getLocator(); final int vb = locator.getVBucketIndex(key); List bcastNodes = new ArrayList(); - if (toMaster) { MemcachedNode primary = locator.getPrimary(key); if (primary != null) { bcastNodes.add(primary); } - } if (toReplica) { List replicaIndexes = locator.getReplicaIndexes(key); @@ -1710,7 +1708,7 @@ public void complete() { @Override public Map observe(final String key, final long cas) { - return observe(key, cas, true, true); + return observe(key, cas, true); } @Override @@ -1782,8 +1780,6 @@ public void observePoll(final String key, final long cas, PersistTo persist, final int shouldPersistTo = persist.getValue() > 0 ? persist.getValue() - 1 : 0; final int shouldReplicateTo = replicate.getValue(); final boolean shouldPersistToMaster = persist.getValue() > 0; - - final boolean toMaster = persist.getValue() > 0; final boolean toReplica = replicate.getValue() > 0 || persist.getValue() > 1; int donePolls = 0; @@ -1802,8 +1798,7 @@ public void observePoll(final String key, final long cas, PersistTo persist, + TimeUnit.MILLISECONDS.toSeconds(timeTried) + " seconds."); } - Map response = observe(key, cas, toMaster, - toReplica); + Map response = observe(key, cas, toReplica); MemcachedNode master = locator.getPrimary(key); alreadyPersistedTo = 0; @@ -1813,7 +1808,7 @@ public void observePoll(final String key, final long cas, PersistTo persist, MemcachedNode node = r.getKey(); ObserveResponse observeResponse = r.getValue(); - boolean isMaster = node == master ? true : false; + boolean isMaster = node == master; if (isMaster && observeResponse == ObserveResponse.MODIFIED) { throw new ObservedModifiedException("Key was modified"); } From 8bab8926afd10259421f4e3bef38486406e40030 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Wed, 1 Apr 2015 11:43:23 +0200 Subject: [PATCH 39/47] JCBC-738: Avoid casting op in HttpFuture on timeout. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- A bug was reported that the HttpFuture incorrectly casted the operation to a net.spy one, which is unexpected. In fact, the cast itself is unnecessary and can be removed. Modifications ------------- The timeout exception is now directly exposed to be in line with the other timeout case on the codepath and as a result is not subject to a classcast exception anymore as well as consistently throws a TimeoutException in the timeout case. In fact, the method directly allows a TimeoutException to be thrown as a checked exception. Result ------ No classcast exceptions and correct exception type thrown on all timeout codepaths. Change-Id: I4e0cda493a4cba961c42cef3b24fdea49e18fb81 Reviewed-on: http://review.couchbase.org/49020 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../com/couchbase/client/internal/HttpFuture.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/couchbase/client/internal/HttpFuture.java b/src/main/java/com/couchbase/client/internal/HttpFuture.java index bf59cea4..8a38acc8 100644 --- a/src/main/java/com/couchbase/client/internal/HttpFuture.java +++ b/src/main/java/com/couchbase/client/internal/HttpFuture.java @@ -23,6 +23,9 @@ package com.couchbase.client.internal; import com.couchbase.client.protocol.views.HttpOperation; +import net.spy.memcached.internal.AbstractListenableFuture; +import net.spy.memcached.internal.GenericCompletionListener; +import net.spy.memcached.ops.OperationStatus; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -31,13 +34,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; -import net.spy.memcached.compat.SpyObject; -import net.spy.memcached.internal.AbstractListenableFuture; -import net.spy.memcached.internal.CheckedOperationTimeoutException; -import net.spy.memcached.internal.GenericCompletionListener; -import net.spy.memcached.internal.OperationCompletionListener; -import net.spy.memcached.ops.Operation; -import net.spy.memcached.ops.OperationStatus; /** * A future http response. @@ -105,8 +101,7 @@ protected void waitForAndCheckOperation(long duration, TimeUnit units) if (op != null && op.isTimedOut()) { status = new OperationStatus(false, "Timed out"); - throw new ExecutionException(new CheckedOperationTimeoutException( - "Operation timed out.", (Operation)op)); + throw new TimeoutException("Timed out waiting for operation"); } } From c4441e8bd1523bfff66b489f4af306c09f972b0c Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Fri, 3 Apr 2015 09:54:30 +0200 Subject: [PATCH 40/47] Upgrade Spymemcached to 2.11.7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I5a1d869ede982f2d70d6c2a996474fa22d23798f Reviewed-on: http://review.couchbase.org/49157 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 5d961067..debb4066 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.6 -spymemcached-test.version=2.11.6 +spymemcached.version=2.11.7 +spymemcached-test.version=2.11.7 From 7ae27ec8cc1935c64e2204cac13765614b9ce48d Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 18 May 2015 07:39:32 +0200 Subject: [PATCH 41/47] JCBC-770: Memcached buckets fail to pick up new configuration. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- For memcached buckets, the client was able to pick up an initial configuration, but failed to properly establish a streaming connection to get subsequent configuration updates. This resulted in the problem that when memcached nodes are added, it is not picked up properly and since the ketama algorithm doesn't make it fail it could only be observed by looking in the UI. Also, node removals are not properly picked up. Modifications ------------- The root cause of the problem is that the HTTP streaming connection is not properly attached to one of the servers, and the reason for that is that the correct bootstrap provider type was not set and therefore an assertion failed. Result ------ After setting the bootstrap provider before monitoring the bucket through a streaming connection, new configs are now properly picked up (both node add and removals) and delivered to the subscribers. Change-Id: I4565b1d7f6c28773736be0a18a020029f0b7964e Reviewed-on: http://review.couchbase.org/51153 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../client/vbucket/provider/BucketConfigurationProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index b12f98d1..b026db0c 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -350,8 +350,8 @@ boolean bootstrapHttp() { httpProvider.get().clearBuckets(); Bucket config = httpProvider.get().getBucketConfiguration(bucket); setConfig(config); - monitorBucket(); bootstrapProvider = BootstrapProviderType.HTTP; + monitorBucket(); return true; } catch(Exception ex) { getLogger().info("Could not fetch config from http seed nodes.", ex); From c5feb5fdd1b4320281e5234f569d3f9f0f88d68a Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Thu, 23 Jul 2015 12:17:54 +0200 Subject: [PATCH 42/47] JCBC-816: Make reconfiguration more resilient during edge cases. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- There are two edge cases which can happen either individually or together: - Full cluster restarts (also single node cluster restarts) - Low ops/s. (especially during testing on a single thread in a loop, blocking with the default timeout) If they happen, the client is not able to get to an alternative node immediately and also the incentive for a reconfiguration (triggered through ops not finding their primary node) is lowered. There are some steps which can be done to make it more resilient and recovering more quickly. Modifications ------------- A bunch of smaller fixes combined provide better reliability semantics. They are: - If an exception happens during bootstrap, it is retried. But if an exception happens again during the re-bootstrap, it is now caught and ignored to "keep going" and wait for new attempts through the other heuristics. Propagating the exception into the nirvana only has the side effect of potentially shooting off live threads which just want to signal a stale connection. - The current code had too strict boundary checks in place for when a new reconfiguration is triggered, mainly for legacy reasons when http streaming only was available. With CCCP, things changed a bit. So the code now only makes sure that not more than one "outdated" attempt is made per second (the 10 ops in a 10 sec interval boundary has been removed). This makes sure that even under low load, we eventually reach a new configuration as quickly as possible. Tests have been adapted to get rid of the removed boundary check, as well as a potential NPE has been removed. Result ------ Under these edge cases, the client now more reliably and more quickly gets back to a valid configuration. Change-Id: I3e19e1d0a859cfd345db18457377b7373e643605 Reviewed-on: http://review.couchbase.org/53589 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../client/CouchbaseConnectionFactory.java | 54 +++--------------- .../provider/BucketConfigurationProvider.java | 41 ++++++++------ .../CouchbaseConnectionFactoryTest.java | 56 ++----------------- 3 files changed, 35 insertions(+), 116 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java index 53e25abd..7f235e1a 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnectionFactory.java @@ -46,8 +46,6 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -143,8 +141,6 @@ public class CouchbaseConnectionFactory extends BinaryConnectionFactory { private static final Logger LOGGER = Logger.getLogger(CouchbaseConnectionFactory.class.getName()); private volatile boolean needsReconnect; - private volatile long thresholdLastCheck = System.nanoTime(); - private final AtomicInteger configThresholdCount = new AtomicInteger(0); private final int maxConfigCheck = 10; //maximum allowed checks before we // reconnect in a 10 sec interval private volatile long configProviderLastUpdateTimestamp; @@ -328,20 +324,17 @@ void setMinReconnectInterval(long reconnIntervalMsecs) { } /** - * Check if a configuration update is needed. + * Check for a new configuration update. * - * There are two main reasons that can trigger a configuration update. Either - * there is a configuration update happening in the cluster, or operations - * added to the queue can not find their corresponding node. For the latter, - * see the {@link #pastReconnThreshold()} method for further details. + * In many different cases, it is possible that for a new configuration should be checked + * proactively. This internally performs a {@link ConfigurationProvider#signalOutdated()} + * call under the covers to work with the configuration provider on a new configuration. * - * If a configuration update is needed, a resubscription for configuration - * updates is triggered. Note that since reconnection takes some time, - * the method will also wait a time period given by - * {@link #getMinReconnectInterval()} before the resubscription is triggered. + * To not ask too often, only if more time than specified through + * {@link #getMinReconnectInterval()} passed, the {@link ConfigurationProvider#signalOutdated()} + * will be executed. */ void checkConfigUpdate() { - if (needsReconnect || pastReconnThreshold()) { long now = System.currentTimeMillis(); long intervalWaited = now - this.configProviderLastUpdateTimestamp; if (intervalWaited < this.getMinReconnectInterval()) { @@ -352,38 +345,7 @@ void checkConfigUpdate() { } getConfigurationProvider().signalOutdated(); - } else { - LOGGER.log(Level.FINE, "No reconnect required, though check requested." - + " Current config check is {0} out of a threshold of {1}.", - new Object[]{configThresholdCount, maxConfigCheck}); - } - } - - /** - * Checks if there have been more requests than allowed through - * maxConfigCheck in a 10 second period. - * - * If this is the case, then true is returned. If the timeframe between - * two distinct requests is more than 10 seconds, a fresh timeframe starts. - * This means that 10 calls every second would trigger an update while - * 1 operation, then a 11 second sleep and one more operation would not. - * - * @return true if there were more config check requests than maxConfigCheck - * in the 10 second period. - */ - protected boolean pastReconnThreshold() { - long currentTime = System.nanoTime(); - - if (currentTime - thresholdLastCheck >= TimeUnit.SECONDS.toNanos(10)) { - configThresholdCount.set(0); - thresholdLastCheck = currentTime; - } - - if (configThresholdCount.incrementAndGet() >= maxConfigCheck) { - return true; - } - - return false; + configProviderLastUpdateTimestamp = System.currentTimeMillis(); } /** diff --git a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java index b026db0c..ea0cc6a8 100644 --- a/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java +++ b/src/main/java/com/couchbase/client/vbucket/provider/BucketConfigurationProvider.java @@ -273,9 +273,10 @@ public void connectionLost(SocketAddress sa) { getLogger().debug("Carrier Config Connection lost from " + sa); CouchbaseConnection conn = binaryConnection.getAndSet(null); try { - conn.shutdown(); - - } catch (IOException e) { + if (conn != null) { + conn.shutdown(); + } + } catch (Exception e) { getLogger().debug("Could not shut down Carrier Config Connection", e); } signalOutdated(); @@ -500,24 +501,28 @@ public void signalOutdated() { } if (bootstrapProvider.isCarrier() || bootstrapProvider == BootstrapProviderType.NONE) { - if (binaryConnection.get() == null) { - bootstrap(); - } else { - try { - List configs = getConfigsFromBinaryConnection(binaryConnection.get()); - if (configs.isEmpty()) { + try { + if (binaryConnection.get() == null) { + bootstrap(); + } else { + try { + List configs = getConfigsFromBinaryConnection(binaryConnection.get()); + if (configs.isEmpty()) { + bootstrap(); + return; + } + String appliedConfig = binaryConnection.get().replaceConfigWildcards( + configs.get(0)); + Bucket config = configurationParser.parseBucket(appliedConfig); + setConfig(config); + } catch(Exception ex) { + getLogger().info("Could not load config from existing " + + "connection, rerunning bootstrap.", ex); bootstrap(); - return; } - String appliedConfig = binaryConnection.get().replaceConfigWildcards( - configs.get(0)); - Bucket config = configurationParser.parseBucket(appliedConfig); - setConfig(config); - } catch(Exception ex) { - getLogger().info("Could not load config from existing " - + "connection, rerunning bootstrap.", ex); - bootstrap(); } + } catch (Exception ex) { + getLogger().debug("Exception received during signalOutdated, ignoring to keep going.", ex); } } else { if (disableHttpBootstrap) { diff --git a/src/test/java/com/couchbase/client/CouchbaseConnectionFactoryTest.java b/src/test/java/com/couchbase/client/CouchbaseConnectionFactoryTest.java index dced85cc..1d5afa43 100644 --- a/src/test/java/com/couchbase/client/CouchbaseConnectionFactoryTest.java +++ b/src/test/java/com/couchbase/client/CouchbaseConnectionFactoryTest.java @@ -22,21 +22,15 @@ */ package com.couchbase.client; +import com.couchbase.client.vbucket.config.Config; +import net.spy.memcached.TestConfig; +import org.junit.Before; +import org.junit.Test; import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.List; -import java.util.concurrent.TimeUnit; -import com.couchbase.client.vbucket.ConfigurationProvider; -import com.couchbase.client.vbucket.ConfigurationProviderMemcacheMock; -import com.couchbase.client.vbucket.CouchbaseNodeOrder; -import com.couchbase.client.vbucket.config.Config; -import net.spy.memcached.TestConfig; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -73,48 +67,6 @@ public void shouldThrowIfPasswordIsNull() throws Exception { new CouchbaseConnectionFactory(uris, "default", null); } - /** - * Make sure that the first calls to pastReconnThreshold() yield false - * and the first one who is over getMaxConfigCheck() yields true. - * - * @throws IOException - */ - @Test - public void testPastReconnectThreshold() throws IOException { - CouchbaseConnectionFactory connFact = buildFactory(); - - for(int i=1; i Date: Tue, 17 Nov 2015 10:05:55 +0100 Subject: [PATCH 43/47] JCBC-881: Expose custom transcoder with durability requirements. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- On the CouchbaseClientIF, all the mutation methods expose a custom transcoder, but only without durability requirements. It is a valid use case to use a custom transcoder with durability requirements. Modifications ------------- Add API which allows to use the durability requirements together with a custom transcoder. The changeset itself is low risk, since the methods are available internally, only the approrpiate overloads have been added which now call the underlying methods explicitly with the transcoder. Result ------ Mutations with a custom transcoder and durability requirements can now be used easily. Change-Id: I5eca2162d3b5c844635dbf1b8d0aa7c03edcdb8f Reviewed-on: http://review.couchbase.org/57103 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- .../com/couchbase/client/CouchbaseClient.java | 85 +++++--- .../couchbase/client/CouchbaseClientIF.java | 194 ++++++++++++++++++ 2 files changed, 249 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseClient.java b/src/main/java/com/couchbase/client/CouchbaseClient.java index 5558f5d7..50f0ff90 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClient.java +++ b/src/main/java/com/couchbase/client/CouchbaseClient.java @@ -1271,15 +1271,14 @@ public OperationFuture set(String key, Object value) { } @Override - public OperationFuture set(String key, int exp, - Object value, PersistTo req, ReplicateTo rep) { - + public OperationFuture set(String key, int exp, T value, + PersistTo req, ReplicateTo rep, Transcoder tc) { if(mconn instanceof CouchbaseMemcachedConnection) { throw new IllegalArgumentException("Durability options are not supported" - + " on memcached type buckets."); + + " on memcached type buckets."); } - OperationFuture setOp = set(key, exp, value); + OperationFuture setOp = set(key, exp, value, tc); if(req == PersistTo.ZERO && rep == ReplicateTo.ZERO) { return setOp; } @@ -1287,6 +1286,12 @@ public OperationFuture set(String key, int exp, return asyncObserveStore(key, setOp, req, rep, "Set", false); } + @Override + public OperationFuture set(String key, int exp, + Object value, PersistTo req, ReplicateTo rep) { + return set(key, exp, value, req, rep, transcoder); + } + @Override public OperationFuture set(String key, Object value, PersistTo req, ReplicateTo rep) { @@ -1322,15 +1327,14 @@ public OperationFuture set(String key, Object value, } @Override - public OperationFuture add(String key, int exp, - Object value, PersistTo req, ReplicateTo rep) { - + public OperationFuture add(String key, int exp, T value, + PersistTo req, ReplicateTo rep, Transcoder tc) { if(mconn instanceof CouchbaseMemcachedConnection) { throw new IllegalArgumentException("Durability options are not supported" - + " on memcached type buckets."); + + " on memcached type buckets."); } - OperationFuture addOp = add(key, exp, value); + OperationFuture addOp = add(key, exp, value, tc); if(req == PersistTo.ZERO && rep == ReplicateTo.ZERO) { return addOp; } @@ -1338,6 +1342,12 @@ public OperationFuture add(String key, int exp, return asyncObserveStore(key, addOp, req, rep, "Add", false); } + @Override + public OperationFuture add(String key, int exp, + Object value, PersistTo req, ReplicateTo rep) { + return add(key, exp, value, req, rep, transcoder); + } + @Override public OperationFuture add(String key, Object value, PersistTo req, ReplicateTo rep) { @@ -1373,15 +1383,14 @@ public OperationFuture add(String key, Object value, } @Override - public OperationFuture replace(String key, int exp, - Object value, PersistTo req, ReplicateTo rep) { - + public OperationFuture replace(String key, int exp, T value, + PersistTo req, ReplicateTo rep, Transcoder tc) { if(mconn instanceof CouchbaseMemcachedConnection) { throw new IllegalArgumentException("Durability options are not supported" - + " on memcached type buckets."); + + " on memcached type buckets."); } - OperationFuture replaceOp = replace(key, exp, value); + OperationFuture replaceOp = replace(key, exp, value, tc); if (req == PersistTo.ZERO && rep == ReplicateTo.ZERO) { return replaceOp; } @@ -1389,6 +1398,12 @@ public OperationFuture replace(String key, int exp, return asyncObserveStore(key, replaceOp, req, rep, "Replace", false); } + @Override + public OperationFuture replace(String key, int exp, + Object value, PersistTo req, ReplicateTo rep) { + return replace(key, exp, value, req, rep, transcoder); + } + /** * Helper method to chain asynchronous observe calls. * @@ -1496,13 +1511,13 @@ public CASResponse cas(String key, long cas, } @Override - public CASResponse cas(String key, long cas, int exp, - Object value, PersistTo req, ReplicateTo rep) { + public CASResponse cas(String key, long cas, int exp, T value, PersistTo req, + ReplicateTo rep, Transcoder tc) { CASResponse casr = null; try { OperationFuture casOp = asyncCas(key, cas, exp, value, req, - rep); + rep, tc); long timeout = cbConnFactory.getObsTimeout(); if (req == PersistTo.ZERO && rep == ReplicateTo.ZERO) { @@ -1524,6 +1539,12 @@ public CASResponse cas(String key, long cas, int exp, } } + @Override + public CASResponse cas(String key, long cas, int exp, Object value, + PersistTo req, ReplicateTo rep) { + return cas(key, cas, exp, value, req, rep, transcoder); + } + @Override public CASResponse cas(String key, long cas, Object value, PersistTo req) { @@ -1579,21 +1600,19 @@ public OperationFuture asyncCas(String key, long cas, int exp, } @Override - public OperationFuture asyncCas(final String key, long cas, - int exp, Object value, final PersistTo req, final ReplicateTo rep) { - + public OperationFuture asyncCas(final String key, long cas, int exp, T value, + final PersistTo req, final ReplicateTo rep, Transcoder tc) { if (mconn instanceof CouchbaseMemcachedConnection) { throw new IllegalArgumentException("Durability options are not supported" - + " on memcached type buckets."); + + " on memcached type buckets."); } - OperationFuture casOp = asyncCAS(key, cas, exp, value, - transcoder); + OperationFuture casOp = asyncCAS(key, cas, exp, value, tc); final CountDownLatch latch = new CountDownLatch(1); final ObserveFuture observeFuture = - new ObserveFuture(key, latch, cbConnFactory.getObsTimeout(), - executorService); + new ObserveFuture(key, latch, cbConnFactory.getObsTimeout(), + executorService); casOp.addListener(new OperationCompletionListener() { @Override @@ -1613,7 +1632,7 @@ public void onComplete(OperationFuture future) throws Exception { } if((casr != CASResponse.OK) - || (req == PersistTo.ZERO && rep == ReplicateTo.ZERO)) { + || (req == PersistTo.ZERO && rep == ReplicateTo.ZERO)) { latch.countDown(); observeFuture.signalComplete(); return; @@ -1624,13 +1643,13 @@ public void onComplete(OperationFuture future) throws Exception { observeFuture.set(casr, future.getStatus()); } catch (ObservedException e) { observeFuture.set(CASResponse.OBSERVE_ERROR_IN_ARGS, - new OperationStatus(false, e.getMessage())); + new OperationStatus(false, e.getMessage())); } catch (ObservedTimeoutException e) { observeFuture.set(CASResponse.OBSERVE_TIMEOUT, - new OperationStatus(false, e.getMessage())); + new OperationStatus(false, e.getMessage())); } catch (ObservedModifiedException e) { observeFuture.set(CASResponse.OBSERVE_MODIFIED, - new OperationStatus(false, e.getMessage())); + new OperationStatus(false, e.getMessage())); } latch.countDown(); @@ -1641,6 +1660,12 @@ public void onComplete(OperationFuture future) throws Exception { return observeFuture; } + @Override + public OperationFuture asyncCas(String key, long cas, int exp, + Object value, PersistTo req, ReplicateTo rep) { + return asyncCas(key, cas, exp, value, req, rep, transcoder); + } + private Map observe(final String key, final long cas, final boolean toReplica) { Config cfg = ((CouchbaseConnectionFactory) connFactory).getVBucketConfig(); diff --git a/src/main/java/com/couchbase/client/CouchbaseClientIF.java b/src/main/java/com/couchbase/client/CouchbaseClientIF.java index f23656fd..c5f2d8d6 100644 --- a/src/main/java/com/couchbase/client/CouchbaseClientIF.java +++ b/src/main/java/com/couchbase/client/CouchbaseClientIF.java @@ -1510,4 +1510,198 @@ SpatialView getSpatialView(String designDocumentName, OperationFuture> getKeyStats(String key); + /** + * Set a value with durability options. + * + * To make sure that a value is stored the way you want it to in the + * cluster, you can use the PersistTo and ReplicateTo arguments. The + * operation will block until the desired state is satisfied or + * otherwise an exception is raised. There are many reasons why this could + * happen, the more frequent ones are as follows: + * + * - The given replication settings are invalid. + * - The operation could not be completed within the timeout. + * - Something goes wrong and a cluster failover is triggered. + * + * The client does not attempt to guarantee the given durability + * constraints, it just reports whether the operation has been completed + * or not. If it is not achieved, it is the responsibility of the + * application code using this API to re-retrieve the items to verify + * desired state, redo the operation or both. + * + * Note that even if an exception during the observation is raised, + * this doesn't mean that the operation has failed. A normal set() + * operation is initiated and after the OperationFuture has returned, + * the key itself is observed with the given durability options (watch + * out for Observed*Exceptions) in this case. + * + * @param key the key to store. + * @param exp the expiry value to use. + * @param value the value of the key. + * @param req the amount of nodes the item should be persisted to before + * returning. + * @param rep the amount of nodes the item should be replicated to before + * returning. + * @param tc a custom transcoder. + * @return the future result of the set operation. + */ + OperationFuture set(String key, int exp, + T value, PersistTo req, ReplicateTo rep, Transcoder tc); + + /** + * Add a value with durability options. + * + * To make sure that a value is stored the way you want it to in the + * cluster, you can use the PersistTo and ReplicateTo arguments. The + * operation will block until the desired state is satisfied or + * otherwise an exception is raised. There are many reasons why this could + * happen, the more frequent ones are as follows: + * + * - The given replication settings are invalid. + * - The operation could not be completed within the timeout. + * - Something goes wrong and a cluster failover is triggered. + * + * The client does not attempt to guarantee the given durability + * constraints, it just reports whether the operation has been completed + * or not. If it is not achieved, it is the responsibility of the + * application code using this API to re-retrieve the items to verify + * desired state, redo the operation or both. + * + * Note that even if an exception during the observation is raised, + * this doesn't mean that the operation has failed. A normal add() + * operation is initiated and after the OperationFuture has returned, + * the key itself is observed with the given durability options (watch + * out for Observed*Exceptions) in this case. + * + * @param key the key to store. + * @param exp the expiry value to use. + * @param value the value of the key. + * @param req the amount of nodes the item should be persisted to before + * returning. + * @param rep the amount of nodes the item should be replicated to before + * returning. + * @param tc a custom transcoder. + * @return the future result of the add operation. + */ + OperationFuture add(String key, int exp, + T value, PersistTo req, ReplicateTo rep, Transcoder tc); + + + /** + * Replace a value with durability options. + * + * To make sure that a value is stored the way you want it to in the + * cluster, you can use the PersistTo and ReplicateTo arguments. The + * operation will block until the desired state is satisfied or + * otherwise an exception is raised. There are many reasons why this could + * happen, the more frequent ones are as follows: + * + * - The given replication settings are invalid. + * - The operation could not be completed within the timeout. + * - Something goes wrong and a cluster failover is triggered. + * + * The client does not attempt to guarantee the given durability + * constraints, it just reports whether the operation has been completed + * or not. If it is not achieved, it is the responsibility of the + * application code using this API to re-retrieve the items to verify + * desired state, redo the operation or both. + * + * Note that even if an exception during the observation is raised, + * this doesn't mean that the operation has failed. A normal replace() + * operation is initiated and after the OperationFuture has returned, + * the key itself is observed with the given durability options (watch + * out for Observed*Exceptions) in this case. + * + * @param key the key to store. + * @param exp the expiry value to use. + * @param value the value of the key. + * @param req the amount of nodes the item should be persisted to before + * returning. + * @param rep the amount of nodes the item should be replicated to before + * returning. + * @param tc a custom transcoder. + * @return the future result of the replace operation. + */ + OperationFuture replace(String key, int exp, + T value, PersistTo req, ReplicateTo rep, Transcoder tc); + + /** + * Set a value with a CAS and durability options. + * + * To make sure that a value is stored the way you want it to in the + * cluster, you can use the PersistTo and ReplicateTo arguments. The + * operation will block until the desired state is satisfied or + * otherwise an exception is raised. There are many reasons why this could + * happen, the more frequent ones are as follows: + * + * - The given replication settings are invalid. + * - The operation could not be completed within the timeout. + * - Something goes wrong and a cluster failover is triggered. + * + * The client does not attempt to guarantee the given durability + * constraints, it just reports whether the operation has been completed + * or not. If it is not achieved, it is the responsibility of the + * application code using this API to re-retrieve the items to verify + * desired state, redo the operation or both. + * + * Note that even if an exception during the observation is raised, + * this doesn't mean that the operation has failed. A normal asyncCAS() + * operation is initiated and after the OperationFuture has returned, + * the key itself is observed with the given durability options (watch + * out for Observed*Exceptions) in this case. + * + * @param key the key to store. + * @param cas the CAS value to use. + * @param exp expiration time for the key. + * @param value the value of the key. + * @param req the amount of nodes the item should be persisted to before + * returning. + * @param rep the amount of nodes the item should be replicated to before + * returning. + * @param tc a custom transcoder. + * @return the future result of the CAS operation. + */ + CASResponse cas(String key, long cas, int exp, T value, PersistTo req, + ReplicateTo rep, Transcoder tc); + + /** + * Set a value with a CAS and durability options. + * + * To make sure that a value is stored the way you want it to in the + * cluster, you can use the PersistTo and ReplicateTo arguments. The + * operation will block until the desired state is satisfied or + * otherwise an exception is raised. There are many reasons why this could + * happen, the more frequent ones are as follows: + * + * - The given replication settings are invalid. + * - The operation could not be completed within the timeout. + * - Something goes wrong and a cluster failover is triggered. + * + * The client does not attempt to guarantee the given durability + * constraints, it just reports whether the operation has been completed + * or not. If it is not achieved, it is the responsibility of the + * application code using this API to re-retrieve the items to verify + * desired state, redo the operation or both. + * + * Note that even if an exception during the observation is raised, + * this doesn't mean that the operation has failed. A normal asyncCAS() + * operation is initiated and after the OperationFuture has returned, + * the key itself is observed with the given durability options (watch + * out for Observed*Exceptions) in this case. + * + * @param key the key to store. + * @param cas the CAS value to use. + * @param exp expiration time for the key. + * @param value the value of the key. + * @param req the amount of nodes the item should be persisted to before + * returning. + * @param rep the amount of nodes the item should be replicated to before + * returning. + * @param tc a custom transcoder. + * @return the future result of the CAS operation. + */ + OperationFuture asyncCas(String key, long cas, int exp, + T value, PersistTo req, ReplicateTo rep, Transcoder tc); + + } From 21b6fdc80b06ad24897dad8605221e192b926863 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Tue, 17 Nov 2015 10:33:50 +0100 Subject: [PATCH 44/47] JCBC-776: Fix typo in log message. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I1ff1e0c712d7c9bb7a828ca8d21c3c55b21c8cee Reviewed-on: http://review.couchbase.org/57106 Reviewed-by: Simon Baslé Reviewed-by: Sergey Avseyev Tested-by: Simon Baslé --- .../java/com/couchbase/client/CouchbaseMemcachedConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java index ae2c2623..b05c6be7 100644 --- a/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java @@ -157,7 +157,7 @@ public void reconfigure(Bucket bucket) { // schedule shutdown for the oddNodes for(MemcachedNode shutDownNode : oddNodes) { getLogger().info("Scheduling Node " - + shutDownNode.getSocketAddress() + "for shutdown."); + + shutDownNode.getSocketAddress() + " for shutdown."); } nodesToShutdown.addAll(oddNodes); } catch (IOException e) { From e1133d9d0ff2b440ccca7880780716403649f797 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Fri, 29 Jan 2016 13:59:44 +0100 Subject: [PATCH 45/47] JCBC-912: Explicitly check that the retry information is not null. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation ---------- If the server returns a NMVB with an empty response, the code needs to check for this case and do not potentially throw a NPE if this case remains undetected. Modifications ------------- The code checks if the buffer which potentially contains the new config is not null and if it is just returns (nothing to do here at this point). Result ------ Better resiliency for the NMVB config optimizations implemented in the server for the watson timeframe. Change-Id: Ie236ed0beb26251c4f9d84ef911906e508b0d41a Reviewed-on: http://review.couchbase.org/59258 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- src/main/java/com/couchbase/client/CouchbaseConnection.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/couchbase/client/CouchbaseConnection.java b/src/main/java/com/couchbase/client/CouchbaseConnection.java index abbea6b0..cae55d18 100644 --- a/src/main/java/com/couchbase/client/CouchbaseConnection.java +++ b/src/main/java/com/couchbase/client/CouchbaseConnection.java @@ -343,6 +343,10 @@ public void run() { @Override protected void handleRetryInformation(byte[] retryMessage) { + if (retryMessage == null) { + return; + } + String message = new String(retryMessage).trim(); if (message.startsWith("{")) { cf.getConfigurationProvider().setConfig( From 4e6f4b3bfb1d5cae2da1b43a63b39ec5dcbd7951 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Thu, 14 Apr 2016 17:08:26 +0200 Subject: [PATCH 46/47] Update Spymemcached to 2.12.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: Ib52d636b7c01dd222d2523676151c607745699a5 Reviewed-on: http://review.couchbase.org/62849 Tested-by: Michael Nitschinger Reviewed-by: Simon Baslé --- ivy/libraries.properties | 4 ++-- pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index debb4066..4941434d 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.11.7 -spymemcached-test.version=2.11.7 +spymemcached.version=2.12.1 +spymemcached-test.version=2.12.1 diff --git a/pom.xml b/pom.xml index 64ed147f..03bd80b9 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ http://www.couchbase.com/develop/java/current - 2.11.3 + 2.12.1 @@ -131,4 +131,4 @@ - \ No newline at end of file + From e2358f8221f22a5b9db9c535edc6ac071657c533 Mon Sep 17 00:00:00 2001 From: Michael Nitschinger Date: Mon, 1 May 2017 10:55:03 +0200 Subject: [PATCH 47/47] Update Spymemcached to 2.12.3 Change-Id: I94f5df56b0e5a08b77cd3229a42c21f012ea9cf3 Reviewed-on: http://review.couchbase.org/77541 Reviewed-by: Michael Nitschinger Tested-by: Michael Nitschinger --- ivy/libraries.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 4941434d..a9a8dcf0 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -37,5 +37,5 @@ jettison.version=1.1 junit.version=4.7 junit-addons.version=1.4 netty.version=3.5.5.Final -spymemcached.version=2.12.1 -spymemcached-test.version=2.12.1 +spymemcached.version=2.12.3 +spymemcached-test.version=2.12.3