Skip to content

Commit 423fc83

Browse files
VenuReddy2103Venu Reddy
VenuReddy2103
and
Venu Reddy
authored
HIVE-29481: Fix create table with transactional=false and transactional_properties=insert_only to create a translated to external table (#6432)
Co-authored-by: Venu Reddy <kvenureddy@cloudera.com>
1 parent 78f50fd commit 423fc83

File tree

4 files changed

+156
-14
lines changed

4 files changed

+156
-14
lines changed

‎itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1276,7 +1276,7 @@ public void testTransformerAlterTable() throws Exception {
12761276
table_params = new StringBuilder();
12771277
table_params.append("key1=val1");
12781278
table_params.append(";");
1279-
table_params.append("transactional_properties=insert_only");
1279+
table_params.append("transactional=true;transactional_properties=insert_only");
12801280
tProps.put("PROPERTIES", table_params.toString());
12811281

12821282
List<String> capabilities = new ArrayList<>();

‎ql/src/test/queries/clientpositive/translated_external_create.q‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
set hive.support.concurrency=true;
2+
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
3+
set hive.create.as.acid=true;
4+
set hive.default.fileformat.managed=ORC;
5+
set hive.metastore.client.capabilities=HIVEFULLACIDWRITE,HIVEMANAGEDINSERTWRITE;
6+
set metastore.metadata.transformer.class=org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer;
7+
8+
-- Should create translated external table when transactional=false
9+
create table translated_table1 (i int) tblproperties('transactional'='false');
10+
desc formatted translated_table1;
11+
12+
-- Should create translated external table when transactional=false and have transactional_properties
13+
create table translated_table2 (i int) tblproperties('transactional'='false', 'transactional_properties'='default');
14+
desc formatted translated_table2;
15+
16+
create table translated_table3 (i int) tblproperties('transactional'='false', 'transactional_properties'='insert_only');
17+
desc formatted translated_table3;

‎ql/src/test/results/clientpositive/llap/translated_external_create.q.out‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
PREHOOK: query: create table translated_table1 (i int) tblproperties('transactional'='false')
2+
PREHOOK: type: CREATETABLE
3+
PREHOOK: Output: database:default
4+
PREHOOK: Output: default@translated_table1
5+
POSTHOOK: query: create table translated_table1 (i int) tblproperties('transactional'='false')
6+
POSTHOOK: type: CREATETABLE
7+
POSTHOOK: Output: database:default
8+
POSTHOOK: Output: default@translated_table1
9+
PREHOOK: query: desc formatted translated_table1
10+
PREHOOK: type: DESCTABLE
11+
PREHOOK: Input: default@translated_table1
12+
POSTHOOK: query: desc formatted translated_table1
13+
POSTHOOK: type: DESCTABLE
14+
POSTHOOK: Input: default@translated_table1
15+
# col_name data_type comment
16+
i int
17+
18+
# Detailed Table Information
19+
Database: default
20+
#### A masked pattern was here ####
21+
Retention: 0
22+
#### A masked pattern was here ####
23+
Table Type: EXTERNAL_TABLE
24+
Table Parameters:
25+
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"i\":\"true\"}}
26+
EXTERNAL TRUE
27+
TRANSLATED_TO_EXTERNAL TRUE
28+
bucketing_version 2
29+
external.table.purge TRUE
30+
numFiles 0
31+
numRows 0
32+
rawDataSize 0
33+
totalSize #Masked#
34+
#### A masked pattern was here ####
35+
36+
# Storage Information
37+
SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
38+
InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
39+
OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
40+
Compressed: No
41+
Num Buckets: -1
42+
Bucket Columns: []
43+
Sort Columns: []
44+
Storage Desc Params:
45+
serialization.format 1
46+
PREHOOK: query: create table translated_table2 (i int) tblproperties('transactional'='false', 'transactional_properties'='default')
47+
PREHOOK: type: CREATETABLE
48+
PREHOOK: Output: database:default
49+
PREHOOK: Output: default@translated_table2
50+
POSTHOOK: query: create table translated_table2 (i int) tblproperties('transactional'='false', 'transactional_properties'='default')
51+
POSTHOOK: type: CREATETABLE
52+
POSTHOOK: Output: database:default
53+
POSTHOOK: Output: default@translated_table2
54+
PREHOOK: query: desc formatted translated_table2
55+
PREHOOK: type: DESCTABLE
56+
PREHOOK: Input: default@translated_table2
57+
POSTHOOK: query: desc formatted translated_table2
58+
POSTHOOK: type: DESCTABLE
59+
POSTHOOK: Input: default@translated_table2
60+
# col_name data_type comment
61+
i int
62+
63+
# Detailed Table Information
64+
Database: default
65+
#### A masked pattern was here ####
66+
Retention: 0
67+
#### A masked pattern was here ####
68+
Table Type: EXTERNAL_TABLE
69+
Table Parameters:
70+
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"i\":\"true\"}}
71+
EXTERNAL TRUE
72+
TRANSLATED_TO_EXTERNAL TRUE
73+
bucketing_version 2
74+
external.table.purge TRUE
75+
numFiles 0
76+
numRows 0
77+
rawDataSize 0
78+
totalSize #Masked#
79+
#### A masked pattern was here ####
80+
81+
# Storage Information
82+
SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
83+
InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
84+
OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
85+
Compressed: No
86+
Num Buckets: -1
87+
Bucket Columns: []
88+
Sort Columns: []
89+
Storage Desc Params:
90+
serialization.format 1
91+
PREHOOK: query: create table translated_table3 (i int) tblproperties('transactional'='false', 'transactional_properties'='insert_only')
92+
PREHOOK: type: CREATETABLE
93+
PREHOOK: Output: database:default
94+
PREHOOK: Output: default@translated_table3
95+
POSTHOOK: query: create table translated_table3 (i int) tblproperties('transactional'='false', 'transactional_properties'='insert_only')
96+
POSTHOOK: type: CREATETABLE
97+
POSTHOOK: Output: database:default
98+
POSTHOOK: Output: default@translated_table3
99+
PREHOOK: query: desc formatted translated_table3
100+
PREHOOK: type: DESCTABLE
101+
PREHOOK: Input: default@translated_table3
102+
POSTHOOK: query: desc formatted translated_table3
103+
POSTHOOK: type: DESCTABLE
104+
POSTHOOK: Input: default@translated_table3
105+
# col_name data_type comment
106+
i int
107+
108+
# Detailed Table Information
109+
Database: default
110+
#### A masked pattern was here ####
111+
Retention: 0
112+
#### A masked pattern was here ####
113+
Table Type: EXTERNAL_TABLE
114+
Table Parameters:
115+
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"i\":\"true\"}}
116+
EXTERNAL TRUE
117+
TRANSLATED_TO_EXTERNAL TRUE
118+
bucketing_version 2
119+
external.table.purge TRUE
120+
numFiles 0
121+
numRows 0
122+
rawDataSize 0
123+
totalSize #Masked#
124+
#### A masked pattern was here ####
125+
126+
# Storage Information
127+
SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
128+
InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
129+
OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
130+
Compressed: No
131+
Num Buckets: -1
132+
Bucket Columns: []
133+
Sort Columns: []
134+
Storage Desc Params:
135+
serialization.format 1

‎standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_NONE;
2121
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY;
2222
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READWRITE;
23-
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.CTAS_LEGACY_CONFIG;
2423
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_TRANSACTIONAL;
2524
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES;
2625
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.EXTERNAL_TABLE_PURGE;
@@ -630,10 +629,6 @@ public Table transformCreateTable(Table table, List<String> processorCapabilitie
630629
params = new HashMap<>();
631630
}
632631
String tableType = newTable.getTableType();
633-
String txnal = null;
634-
String txn_properties = null;
635-
boolean isInsertAcid = false;
636-
637632
String dbName = table.getDbName();
638633
Database db = null;
639634
try {
@@ -642,13 +637,9 @@ public Table transformCreateTable(Table table, List<String> processorCapabilitie
642637
throw new MetaException("Database " + dbName + " for table " + table.getTableName() + " could not be found");
643638
}
644639

645-
if (TableType.MANAGED_TABLE.name().equals(tableType)) {
640+
if (TableType.MANAGED_TABLE.name().equals(tableType)) {
646641
LOG.debug("Table is a MANAGED_TABLE");
647-
txnal = params.get(TABLE_IS_TRANSACTIONAL);
648-
txn_properties = params.get(TABLE_TRANSACTIONAL_PROPERTIES);
649-
isInsertAcid = (txn_properties != null && txn_properties.equalsIgnoreCase("insert_only"));
650-
boolean ctas_legacy_config = params.containsKey(CTAS_LEGACY_CONFIG) && params.get(CTAS_LEGACY_CONFIG).equalsIgnoreCase("true") ? true : false;
651-
if (((txnal == null || txnal.equalsIgnoreCase("FALSE")) && !isInsertAcid) || (ctas_legacy_config && (txnal == null || txnal.equalsIgnoreCase("FALSE")))) { // non-ACID MANAGED TABLE
642+
if (!Boolean.parseBoolean(params.get(TABLE_IS_TRANSACTIONAL))) { // non-ACID MANAGED TABLE
652643
LOG.info("Converting " + newTable.getTableName() + " to EXTERNAL tableType for " + processorId);
653644
newTable.setTableType(TableType.EXTERNAL_TABLE.toString());
654645
params.remove(TABLE_IS_TRANSACTIONAL);
@@ -682,9 +673,8 @@ public Table transformCreateTable(Table table, List<String> processorCapabilitie
682673
throw new MetaException("Processor has no capabilities, cannot create an ACID table.");
683674
}
684675

685-
686676
newTable = validateTablePaths(table);
687-
if (isInsertAcid) { // MICRO_MANAGED Tables
677+
if (MetaStoreUtils.isInsertOnlyTableParam(table.getParameters())) { // MICRO_MANAGED Tables
688678
if (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE)) {
689679
LOG.debug("Processor has required capabilities to be able to create INSERT-only tables");
690680
return newTable;

0 commit comments

Comments
 (0)