Description
Could you please add [core name] to each log entry? Thanks!
For example, it's hard for us to know the exact core having a such issue and the sequence, if there are too many cores in a solr node.
This line is a good example:
2015-04-16 13:12:07.244; org.apache.solr.core.SolrCore; [collection3_shard5_replica2] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
This is a bad example:
WARN - 2015-04-16 13:12:11.136; org.apache.solr.handler.SnapPuller$DirectoryFileFetcher; Error in fetching packets
java.io.EOFException
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:154)
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:146)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchPackets(SnapPuller.java:1211)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1174)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
WARN - 2015-04-16 13:12:11.287; org.apache.solr.handler.SnapPuller$DirectoryFileFetcher; Error in fetching packets
java.io.EOFException
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:154)
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:146)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchPackets(SnapPuller.java:1211)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1174)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
WARN - 2015-04-16 13:12:11.465; org.apache.solr.handler.SnapPuller$DirectoryFileFetcher; Error in fetching packets
java.io.EOFException
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:154)
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:146)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchPackets(SnapPuller.java:1211)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1174)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
WARN - 2015-04-16 13:12:11.586; org.apache.solr.handler.SnapPuller$DirectoryFileFetcher; Error in fetching packets
java.io.EOFException
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:154)
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:146)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchPackets(SnapPuller.java:1211)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1174)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
WARN - 2015-04-16 13:12:11.768; org.apache.solr.handler.SnapPuller$DirectoryFileFetcher; Error in fetching packets
java.io.EOFException
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:154)
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:146)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchPackets(SnapPuller.java:1211)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1174)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
WARN - 2015-04-16 13:12:11.895; org.apache.solr.handler.SnapPuller$DirectoryFileFetcher; Error in fetching packets
java.io.EOFException
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:154)
at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:146)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchPackets(SnapPuller.java:1211)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1174)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
ERROR - 2015-04-16 13:12:11.924; org.apache.solr.common.SolrException; SnapPull failed :org.apache.solr.common.SolrException: Unable to download segments_1a8f completely. Downloaded 0!=931
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.cleanup(SnapPuller.java:1305)
at org.apache.solr.handler.SnapPuller$DirectoryFileFetcher.fetchFile(SnapPuller.java:1185)
at org.apache.solr.handler.SnapPuller.downloadIndexFiles(SnapPuller.java:771)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:421)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:322)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:155)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)
ERROR - 2015-04-16 13:12:11.924; org.apache.solr.common.SolrException; Error while trying to recover:org.apache.solr.common.SolrException: Replication for recovery failed.
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:158)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:437)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)