Frequent failures of MoveReplicaHDFSTest.testFailedMove

   [junit4]   2> 129275 INFO  (qtp1647120030-248) [n:127.0.0.1:55469_solr c:MoveReplicaHDFSTest_failed_coll_true s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n4] o.a.s.c.S.Request [MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n4]  webapp=/solr path=/select params={q=*:*&_stateVer_=MoveReplicaHDFSTest_failed_coll_true:9&wt=javabin&version=2} status=503 QTime=0
   [junit4]   2> 129278 ERROR (qtp148844424-682) [n:127.0.0.1:54855_solr c:MoveReplicaHDFSTest_failed_coll_true s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n6] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: no servers hosting shard: shard1
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.prepDistributed(HttpShardHandler.java:436)
   [junit4]   2> 	at org.apache.solr.handler.component.SearchHandler.getAndPrepShardHandler(SearchHandler.java:226)
   [junit4]   2> 	at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:264)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:195)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2503)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:711)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:517)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:384)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:330)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:530)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
   [junit4]   2> 	at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 129279 INFO  (qtp148844424-682) [n:127.0.0.1:54855_solr c:MoveReplicaHDFSTest_failed_coll_true s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n6] o.a.s.c.S.Request [MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n6]  webapp=/solr path=/select params={q=*:*&_stateVer_=MoveReplicaHDFSTest_failed_coll_true:9&wt=javabin&version=2} status=503 QTime=1
   [junit4]   2> 129280 ERROR (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[26ECC6481A5A977A]) [    ] o.a.s.c.s.i.CloudSolrClient Request to collection [MoveReplicaHDFSTest_failed_coll_true] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54855/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard1, retry? 1
   [junit4]   2> 129281 INFO  (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[26ECC6481A5A977A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testFailedMove
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=MoveReplicaHDFSTest -Dtests.method=testFailedMove -Dtests.seed=26ECC6481A5A977A -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=is-IS -Dtests.timezone=Asia/Macau -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   21.0s J10 | MoveReplicaHDFSTest.testFailedMove <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:54855/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:55469/solr/MoveReplicaHDFSTest_failed_coll_true]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([26ECC6481A5A977A:8C2115BAAD8942AA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
   [junit4]    > 	at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:309)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54855/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	... 46 more