Hive优化1---分区、桶、index

1、分区表:分区是目录。

------------------
//创建分区表
CREATE TABLE custs
(
id int,
name string ,
age int
)
PARTITIONED BY (prov string, city string)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;

//添加分区
alter table custs add PARTITION (prov='hebei', city='baoding') PARTITION (prov='hebei', city='shijiazhuang');

//查看分区
SHOW PARTITIONS custs;

//删除分区
alter table custs drop partition partition(prov='hebei',city='shijizhuang') ; 

//加载数据到分区
load data local inpath '/home/centos/cust.txt' into table custs partition(prov='hebei',city='baoding') ;

//按照分区查询
select * from custs where city = 'baoding' ;

2、桶表:桶表是文件。

--------------
//创建桶表
CREATE TABLE buck
(
id int,
name string ,
age int
)
CLUSTERED BY (id) INTO 3 BUCKETS
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;

//桶的数量确定标准
避免桶内的数据量过大或者过小，一般以数据块的2倍为宜。

//设置map个数和强行分桶
set map.reduce.tasks = 2;

set hive.enforce.bucketing = true;

3.index
能够排序，使用折半查找。
创建的索引对应一张表。内容为空
索引表中存放的字段值和对应的文件以及偏移量，如果数据发生改变，需要rebuild索引.
//创建索引，生成索引表
CREATE INDEX idx_emp_name ON TABLE employee(name) AS 'COMPACT' WITH DEFERRED REBUILD;
//重建索引值
ALTER INDEX idx_emp_name ON employee REBUILD;
//删除索引
drop index idx_emp_name on employee;

[创建部门索引]
CREATE INDEX idx_emp_deptnum ON TABLE employee(dept_num) AS 'COMPACT' WITH DEFERRED REBUILD;
//重建索引值
ALTER INDEX idx_emp_deptnum ON employee REBUILD;