java.nio.channels.AsynchronousCloseException

I am going to switch from solr cloud 6.6 to solrcloud 8.5. I installed solr 8.5 parallel with solr 6.6. they have same data, same configset.

Solr 8.5 have many log errors about java.nio.channels.AsynchronousCloseException and Reset cancel_stream_error while solr 6.6 doesn't have.

2020-04-09 05:05:00.254 WARN (Thread-34437) [ ] o.a.s.u.p.DistributedZkUpdateProcessor Error sending update to http://192.168.1.106:8301/solr => org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: null
at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:417)
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: null
at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:417) ~[?:?]
at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:753) ~[?:?]
at org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:369) ~[?:?]
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290) ~[?:?]
at org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:344) ~[?:?]
at org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:333) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_231]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_231]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_231]
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) ~[metrics-core-4.1.2.jar:4.1.2]
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_231]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_231]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_231]
Caused by: java.nio.channels.AsynchronousCloseException
at org.eclipse.jetty.http2.client.http.HttpConnectionOverHTTP2.close(HttpConnectionOverHTTP2.java:144) ~[?:?]
at org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2.onClose(HttpClientTransportOverHTTP2.java:170) ~[?:?]
at org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2$SessionListenerPromise.onClose(HttpClientTransportOverHTTP2.java:232) ~[?:?]
at org.eclipse.jetty.http2.api.Session$Listener.onClose(Session.java:206) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.HTTP2Session.notifyClose(HTTP2Session.java:1131) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.HTTP2Session.onGoAway(HTTP2Session.java:439) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.parser.Parser$Listener$Wrapper.onGoAway(Parser.java:396) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.parser.BodyParser.notifyGoAway(BodyParser.java:192) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.parser.GoAwayBodyParser.onGoAway(GoAwayBodyParser.java:169) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.parser.GoAwayBodyParser.parse(GoAwayBodyParser.java:139) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.parser.Parser.parseBody(Parser.java:198) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.parser.Parser.parse(Parser.java:127) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.HTTP2Connection$HTTP2Producer.produce(HTTP2Connection.java:248) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.HTTP2Connection.produce(HTTP2Connection.java:170) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.HTTP2Connection.onFillable(HTTP2Connection.java:125) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.http2.HTTP2Connection$FillableCallback.succeeded(HTTP2Connection.java:348) ~[http2-common-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) ~[jetty-io-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) ~[jetty-io-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.Invocable.invokeNonBlocking(Invocable.java:69) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.invokeTask(EatWhatYouKill.java:348) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:303) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) ~[jetty-util-9.4.24.v20191120.jar:9.4.24.v20191120]
at org.eclipse.

2020-04-09 05:15:00.255 ERROR (qtp479397964-36118) [c:dbcustomer s:shard1 r:core_node4 x:dbcustomer_shard1_replica_n3] o.a.s.s.HttpSolrCall null:org.eclipse.jetty.io.EofException: Reset cancel_stream_error
at org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory$HTTPServerSessionListener.onReset(HTTP2ServerConnectionFactory.java:157)
at org.eclipse.jetty.http2.HTTP2Stream.notifyReset(HTTP2Stream.java:574)
at org.eclipse.jetty.http2.HTTP2Stream.onReset(HTTP2Stream.java:343)
at org.eclipse.jetty.http2.HTTP2Stream.process(HTTP2Stream.java:252)
at org.eclipse.jetty.http2.HTTP2Session.onReset(HTTP2Session.java:295)
at org.eclipse.jetty.http2.parser.Parser$Listener$Wrapper.onReset(Parser.java:372)
at org.eclipse.jetty.http2.parser.BodyParser.notifyReset(BodyParser.java:144)
at org.eclipse.jetty.http2.parser.ResetBodyParser.onReset(ResetBodyParser.java:97)
at org.eclipse.jetty.http2.parser.ResetBodyParser.parse(ResetBodyParser.java:66)
at org.eclipse.jetty.http2.parser.Parser.parseBody(Parser.java:198)
at org.eclipse.jetty.http2.parser.Parser.parse(Parser.java:127)
at org.eclipse.jetty.http2.parser.ServerParser.parse(ServerParser.java:115)
at org.eclipse.jetty.http2.HTTP2Connection$HTTP2Producer.produce(HTTP2Connection.java:248)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at org.eclipse.jetty.http2.HTTP2Connection.produce(HTTP2Connection.java:170)
at org.eclipse.jetty.http2.HTTP2Connection.onFillable(HTTP2Connection.java:125)
at org.eclipse.jetty.http2.HTTP2Connection$FillableCallback.succeeded(HTTP2Connection.java:348)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
Suppressed: java.lang.Throwable: HttpInput failure
at org.eclipse.jetty.server.HttpInput.failed(HttpInput.java:823)
at org.eclipse.jetty.http2.server.HttpChannelOverHTTP2.onFailure(HttpChannelOverHTTP2.java:323)
at org.eclipse.jetty.http2.server.HTTP2ServerConnection.onStreamFailure(HTTP2ServerConnection.java:221)