我们最近在我们的cassandra集群中看到了这个问题。
com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException: org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 3 responses.
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2201) ~[guava-16.0.jar:na]
at com.google.common.cache.LocalCache.get(LocalCache.java:3934) ~[guava-16.0.jar:na]
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3938) ~[guava-16.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4821) ~[guava-16.0.jar:na]
at org.apache.cassandra.auth.PermissionsCache.getPermissions(PermissionsCache.java:72) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.auth.Auth.getPermissions(Auth.java:75) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.service.ClientState.authorize(ClientState.java:353) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.service.ClientState.ensureHasPermission(ClientState.java:251) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:245) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:229) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.cql3.statements.ModificationStatement.checkAccess(ModificationStatement.java:132) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:235) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:260) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:122) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:439) [apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:335) [apache-cassandra-2.1.14.jar:2.1.14]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333) [netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32) [netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324) [netty-all-4.0.23.Final.jar:4.0.23.Final]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_05]
at org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) [apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) [apache-cassandra-2.1.14.jar:2.1.14]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_05]
Caused by: java.lang.RuntimeException: org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 3 responses.
at org.apache.cassandra.auth.Auth.selectUser(Auth.java:276) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.auth.Auth.isSuperuser(Auth.java:97) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.auth.AuthenticatedUser.isSuper(AuthenticatedUser.java:50) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.auth.CassandraAuthorizer.authorize(CassandraAuthorizer.java:67) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:124) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:121) ~[apache-cassandra-2.1.14.jar:2.1.14]
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3524) ~[guava-16.0.jar:na]
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2317) ~[guava-16.0.jar:na]
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2280) ~[guava-16.0.jar:na]
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2195) ~[guava-16.0.jar:na]
... 23 common frames omitted
Caused by: org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 3 responses.
at org.apache.cassandra.service.ReadCallback.get(ReadCallback.java:105) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.service.AbstractReadExecutor.get(AbstractReadExecutor.java:146) ~[apache-cassandra-2.1.14.jar:2.1.14]
at org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1397) ~[apache-cassandra-2.1.14.jar:2.1.14]
为什么这种情况突然发生，然后cassandra的使用变得正常？ 这个问题的所有可能性是什么？