Major Description
3 QFS smoke tests out of 11 are failing on Jenkins.
https://ci.bigtop.apache.org/job/Bigtop-trunk-smoke-tests/166/COMPONENTS=hdfs.yarn.qfs@qfs,OS=centos-7-x86_64-deploy/console
org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[teragen] FAILED
java.lang.AssertionError: Example teragen /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar teragen 1000 teragen1600846854331 failed
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:167)
org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[terasort] FAILED
java.lang.AssertionError: Example terasort /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar terasort teragen1600846854331 terasort1600846854331 failed
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:167)
org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[teravalidate] FAILED
java.lang.AssertionError: Example teravalidate /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar teravalidate terasort1600846854331 tervalidate1600846854331 failed
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:167)
Gradle Test Executor 2 finished executing tests.
11 tests completed, 3 failed
The terasort and teravalidate depend on teragen, so the root cause is teragen. I tried to install Hadoop and QFS built with Bigtop locally and run teragen manually, then I got FileAlreadyExistsException for the output directory, which didn't exist actually, for some reason.
# hadoop-qfs fs -ls /user/yarn
Found 5 items
drwx------ - yarn yarn 0 2020-09-23 17:48 /user/yarn/.staging
-rw-r--r-- 1 yarn yarn 585 2020-09-23 17:45 /user/yarn/cachedir.jar
-rw-r--r-- 1 yarn yarn 39 2020-09-23 17:45 /user/yarn/input.txt
-rw-r--r-- 1 yarn yarn 805 2020-09-23 17:45 /user/yarn/map.sh
-rw-r--r-- 1 yarn yarn 50 2020-09-23 17:45 /user/yarn/part-00001.snappy
# sudo -u yarn hadoop-qfs jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar teragen 1 /user/yarn/output
20/09/24 09:32:31 INFO client.RMProxy: Connecting to ResourceManager at sekikn-thinkcentre-m715q.flets-east.jp/127.0.1.1:8032
org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory qfs://sekikn-thinkcentre-m715q.flets-east.jp:20000/user/yarn/output already exists
at org.apache.hadoop.examples.terasort.TeraOutputFormat.checkOutputSpecs(TeraOutputFormat.java:117)
at org.apache.hadoop.mapreduce.JobSubmitter.checkSpecs(JobSubmitter.java:279)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:145)
at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1570)
at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1567)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1567)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1588)
at org.apache.hadoop.examples.terasort.TeraGen.run(TeraGen.java:301)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.hadoop.examples.terasort.TeraGen.main(TeraGen.java:305)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:71)
at org.apache.hadoop.util.ProgramDriver.run(ProgramDriver.java:144)
at org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:244)
at org.apache.hadoop.util.RunJar.main(RunJar.java:158)
# hadoop-qfs fs -ls /user/yarn
Found 5 items
drwx------ - yarn yarn 0 2020-09-23 17:48 /user/yarn/.staging
-rw-r--r-- 1 yarn yarn 585 2020-09-23 17:45 /user/yarn/cachedir.jar
-rw-r--r-- 1 yarn yarn 39 2020-09-23 17:45 /user/yarn/input.txt
-rw-r--r-- 1 yarn yarn 805 2020-09-23 17:45 /user/yarn/map.sh
-rw-r--r-- 1 yarn yarn 50 2020-09-23 17:45 /user/yarn/part-00001.snappy
#
QFS leverages MapReduce smoke test, but the latter succeeds if executed without QFS. So there seems to be a problem on the QFS side.
