Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-9250

HdfsOrcScanner crash in calling orc::createReader

    XMLWordPrintableJSON

Details

    Description

      Encounter a crash when running the test_scanners_fuzz test for orc:

      Crash reason:  SIGSEGV
      Crash address: 0x0
      Process uptime: not available
      
      Thread 313 (crashed)
       0  libstdc++.so.6.0.21 + 0xd0328
       1  impalad!orc::TypeImpl::addStructField(std::string const&, std::unique_ptr<orc::Type, std::default_delete<orc::Type> >) [new_allocator.h : 120 + 0xd]
       2  impalad!orc::convertType(orc::proto::Type const&, orc::proto::Footer const&) [TypeImpl.cc : 414 + 0x10]
       3  impalad!orc::ReaderImpl::ReaderImpl(std::shared_ptr<orc::FileContents>, orc::ReaderOptions const&, unsigned long, unsigned long) [Reader.cc : 413 + 0x9]
       4  impalad!orc::createReader(std::unique_ptr<orc::InputStream, std::default_delete<orc::InputStream> >, orc::ReaderOptions const&) [Reader.cc : 1135 + 0x20]
       5  impalad!impala::HdfsOrcScanner::ProcessFileTail() [hdfs-orc-scanner.cc : 250 + 0x4e]
       6  impalad!impala::HdfsOrcScanner::Open(impala::ScannerContext*) [hdfs-orc-scanner.cc : 168 + 0x19]
       7  impalad!impala::HdfsScanNodeBase::CreateAndOpenScannerHelper(impala::HdfsPartitionDescriptor*, impala::ScannerContext*, boost::scoped_ptr<impala::HdfsScanner>*) [hdfs-scan-node-base.cc : 730 + 0x29]
       8  impalad!impala::HdfsScanNode::ProcessSplit(std::vector<impala::FilterContext, std::allocator<impala::FilterContext> > const&, impala::MemPool*, impala::io::ScanRange*, long*) [hdfs-scan-node.cc : 494 + 0x2b]
       9  impalad!impala::HdfsScanNode::ScannerThread(bool, long) [hdfs-scan-node.cc : 416 + 0x2a]
      10  impalad!impala::HdfsScanNode::ThreadTokenAvailableCb(impala::ThreadResourcePool*)::{lambda()#1}::operator()() const + 0x30
      11  impalad!boost::detail::function::void_function_obj_invoker0<impala::HdfsScanNode::ThreadTokenAvailableCb(impala::ThreadResourcePool*)::<lambda()>, void>::invoke [function_template.hpp : 153 + 0xc]
      12  impalad!boost::function0<void>::operator()() const [function_template.hpp : 767 + 0x11]
      13  impalad!impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) [thread.cc : 360 + 0xf]
      14  impalad!void boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) [bind.hpp : 525 + 0x15]
      15  impalad!boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() [bind_template.hpp : 20 + 0x22]
      16  impalad!boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() [thread.hpp : 116 + 0x12]
      17  impalad!thread_proxy + 0xda
      18  libpthread-2.23.so + 0x76ba
      19  libc-2.23.so + 0x10741d
      

      Another similar crash:

      Thread 302 (crashed)
       0  libstdc++.so.6.0.21 + 0xd0328
       1  impalad!orc::TypeImpl::addStructField(std::string const&, std::unique_ptr<orc::Type, std::default_delete<orc::Type> >) [new_allocator.h : 120 + 0xd]
       2  impalad!orc::convertType(orc::proto::Type const&, orc::proto::Footer const&) [TypeImpl.cc : 414 + 0x10]
       3  impalad!orc::ReaderImpl::ReaderImpl(std::shared_ptr<orc::FileContents>, orc::ReaderOptions const&, unsigned long, unsigned long) [Reader.cc : 413 + 0x9]
       4  impalad!orc::createReader(std::unique_ptr<orc::InputStream, std::default_delete<orc::InputStream> >, orc::ReaderOptions const&) [Reader.cc : 1135 + 0x20]
       5  impalad!impala::HdfsOrcScanner::ProcessFileTail() [hdfs-orc-scanner.cc : 250 + 0x4e]
       6  impalad!impala::HdfsOrcScanner::Open(impala::ScannerContext*) [hdfs-orc-scanner.cc : 168 + 0x19]
       7  impalad!impala::HdfsScanNodeBase::CreateAndOpenScannerHelper(impala::HdfsPartitionDescriptor*, impala::ScannerContext*, boost::scoped_ptr<impala::HdfsScanner>*) [hdfs-scan-node-base.cc : 730 + 0x29]
       8  impalad!impala::HdfsScanNode::ProcessSplit(std::vector<impala::FilterContext, std::allocator<impala::FilterContext> > const&, impala::MemPool*, impala::io::ScanRange*, long*) [hdfs-scan-node.cc : 494 + 0x2b]
       9  impalad!impala::HdfsScanNode::ScannerThread(bool, long) [hdfs-scan-node.cc : 416 + 0x2a]
      10  impalad!impala::HdfsScanNode::ThreadTokenAvailableCb(impala::ThreadResourcePool*)::{lambda()#1}::operator()() const + 0x30
      11  impalad!boost::detail::function::void_function_obj_invoker0<impala::HdfsScanNode::ThreadTokenAvailableCb(impala::ThreadResourcePool*)::<lambda()>, void>::invoke [function_template.hpp : 153 + 0xc]
      12  impalad!boost::function0<void>::operator()() const [function_template.hpp : 767 + 0x11]
      13  impalad!impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) [thread.cc : 360 + 0xf]
      14  impalad!void boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) [bind.hpp : 525 + 0x15]
      15  impalad!boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() [bind_template.hpp : 20 + 0x22]
      16  impalad!boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() [thread.hpp : 116 + 0x12]
      17  impalad!thread_proxy + 0xda
      18  libpthread-2.23.so + 0x76ba
      19  libc-2.23.so + 0x10741d
      

      Attached the corrupt orc file that causes the first crash. Create the table by:

      CREATE TABLE test_fuzz_alltypes_b5a5881f.alltypes (
        id INT COMMENT 'Add a comment',
        bool_col BOOLEAN,
        tinyint_col TINYINT,
        smallint_col SMALLINT,
        int_col INT,
        bigint_col BIGINT,
        float_col FLOAT,
        double_col DOUBLE,
        date_string_col STRING,
        string_col STRING,
        timestamp_col TIMESTAMP
      )
      PARTITIONED BY (
        year INT, 
        month INT
      )
      STORED AS ORC
      

      Put the file into this table and run a "select *" on it.

      Attachments

        Issue Links

          Activity

            People

              stigahuang Quanlong Huang
              stigahuang Quanlong Huang
              Votes:
              0 Vote for this issue
              Watchers:
              1 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: