Details
-
Bug
-
Status: Closed
-
Major
-
Resolution: Fixed
-
0.12.2
-
0.5
Description
After flink creates and writes the table, the spark alter table reports an error
* flink
CREATE CATALOG myhudi WITH( 'type' = 'hudi', 'default-database' = 'default', 'mode' = 'hms', 'hive.conf.dir' = 'hdfs:///gdp-dev-3/flink/flink-conf' ); create database if not exists myhudi.test_hudi3; drop table if exists myhudi.test_hudi3.hudi_test19; create table if not exists myhudi.test_hudi3.hudi_test19 (id bigint not null comment 'comment_test', `name` string not null comment 'comment_test2', ts3 bigint ,PRIMARY KEY (`id`) NOT ENFORCED ) with('connector' = 'hudi', 'table.type' = 'COPY_ON_WRITE', 'index.type'= 'BUCKET', 'precombine.field' = 'ts3', 'hoodie.datasource.hive_sync.sync_comment' = 'true'); show create table myhudi.test_hudi3.hudi_test19; insert into myhudi.test_hudi3.hudi_test19 values (1, 'cc1', 1);
- spark
alter table test_hudi3.hudi_test19 add column c string;
* error
org.apache.kyuubi.KyuubiSQLException: org.apache.kyuubi.KyuubiSQLException: Error operating ExecuteStatement: org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to alter table. The following columns have types incompatible with the existing columns in their respective positions : col at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:110) at org.apache.spark.sql.hive.HiveExternalCatalog.alterTableDataSchema(HiveExternalCatalog.scala:679) at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.alterTableDataSchema(ExternalCatalogWithListener.scala:124) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.alterTableDataSchema(SessionCatalog.scala:459) at org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand.refreshSchemaInMeta(AlterHoodieTableAddColumnsCommand.scala:90) at org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand.run(AlterHoodieTableAddColumnsCommand.scala:70) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73) at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84) at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:98) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:109) at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:169) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:95) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:98) at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:94) at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:584) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:176) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:584)
Attachments
Issue Links
- links to