Question

我需要从下面的代码中提取创建表语句。

    #!/bin/bash
rm -f tableNames.txt
rm -f HiveTableDDL.txt
beeline --showHeader=false --outputformat=tsv2 -u jdbc:hive2:// -n hive -e "show tables like 'test*';" > tableNames.txt 
wait
while read LINE
do
   beeline --showHeader=false --outputformat=tsv2 -u jdbc:hive2:// -n hive -e "show create table $LINE" | perl -ne 'BEGIN{$x=qx(cat test.txt);$x=~s/(.+)(create table.+?)(ROW FORMAT SERDE|STORED AS INPUTFORMAT|ROW FORMAT SERDE|OUTPUTFORMAT|LOCATION|TBLPROPERTIES)(.*)/$2/osm; print "$x STORED AS ORC\n" ; exit } '
   printf  ";\n\n" 
done < tableNames.txt >> HiveTableDDL.txt
rm -f tableNames.txt
echo "Table DDL generated"

我想要下面的东西

CREATE TABLE `test`(
  `id` string COMMENT '',
  `age` string COMMENT '',
  `city` string COMMENT '') stored as orc;

CREATE TABLE `test_2`(
  `id` string COMMENT '',
  `age` string COMMENT '',
  `city` string COMMENT '') stored as orc;

Answer 1

检查是否适合您。

> cat hive_table.txt2
show create table hive_table:
create table hive_table(id number,age number)
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
show create table hive_table2:
create table hive_table2(id number,age number)
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
show create table hive_table3:
create table hive_table3(id number,age number)
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
>
> hive_table.pl hive_table.txt2
create table hive_table(id number,age number)
 stored as orc
create table hive_table2(id number,age number)
 stored as orc
create table hive_table3(id number,age number)
 stored as orc
>

脚本是

> cat hive_table.pl
#!/usr/bin/perl
$file=$ARGV[0];
$x=qx(cat $file);
while($x=~m/(.+?)(create table.+?)(CREATE TABLE.+?)(PARTITIONED BY|STRED AS INPUTFORMAT|ROW FORMAT SERDE|OUTPUTFORMAT|LOCATION|TBLPROPERTIES)(.*)/iosm)
{
$x=$5;
$table_desc=$3;
print "$table_desc stored as orc\n";
}

>

EDIT1：

> cat hive_table.pl
#!/usr/bin/perl
$file=$ARGV[0];
$x=qx(cat $file);
while($x=~m/(.+?)(create table.+?)(PARTITIONED BY|STRED AS INPUTFORMAT|ROW FORMAT SERDE|OUTPUTFORMAT|LOCATION|TBLPROPERTIES)(.*)/iosm)
{
$x=$4;
$table_desc=$2;
print "$table_desc stored as orc\n";
}

>

EDIT2：

> cat hive_table.txt3
create table hive_table(id number,age number)
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
create table hive_table2(id number,age number)
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
create table hive_table3(id number,age number)
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ..)
>  hive_table.pl hive_table.txt3
create table hive_table(id number,age number)
 stored as orc
create table hive_table2(id number,age number)
 stored as orc
create table hive_table3(id number,age number)
 stored as orc
/etl/stage3/CAM/AN06599/work_2018/stack> cat hive_table.pl
#!/usr/bin/perl
$file=$ARGV[0];
$x=qx(cat $file);
$x="dummy".$x."dummy";
while($x=~m/(.+?)(create table.+?)(PARTITIONED BY|STRED AS INPUTFORMAT|ROW FORMAT SERDE|OUTPUTFORMAT|LOCATION|TBLPROPERTIES)(.*)/iosm)
{
$x=$4;
$table_desc=$2;
print "$table_desc stored as orc\n";
}

>

Answer 2

不适用于以下文件

 cat test.txt
create table hive_table(id number,age number)
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
create table hive_table2(id number,age number)
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ....)
create table hive_table3(id number,age number)
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION  'hdfs:/path/'
TBLPROPERTIES (   'spark.sql.sources ..)

在Hive中提取多个创建表语句

2 个答案: