ES 在HIVE中创建表存储数据

来源：互联网发布：禁止windows自动更新编辑：程序博客网时间：2024/06/10 06:18

所有操作使用shell脚本方式执行

1. 创建指定mapping文件

more ./create/es_reg_user_goods_article_recommend_mapping.json{                "et_reg_user_goods_article_recommend" : {                        "properties" : {                                "user_id" : {"type" : "long", "store": "yes"},                                "article_ids" : {"type" : "string", "store": "yes", "index": "not_analyzed"},                                "create_date" : {"type" : "string", "store": "yes", "index": "not_analyzed"}                        }                }}

2. 在ES中创建索引

# del index # 删除索引curl -XDELETE http://192.168.1.1:9200/es_reg_user_article_recommend# create index# 创建索引curl -XPUT http://192.168.1.1:9200/es_reg_user_goods_article_recommend/ -d'{        "mappings" : {                "_default_" : {                        "dynamic" : "strict"                }        }}'# create alias# 创建别名curl -XPOST 'http://192.168.1.1:9200/_aliases' -d'{        "actions" : [{ "add" : { "index" : "es_reg_user_goods_article_recommend", "alias" : "reg_user_goods_article_recommend_index"}}]}'# modify alias# 修改别名curl -XPOST 'http://192.168.1.1:9200/_aliases' -d'{        "actions" : [                { "remove" : { "index" : "es_reg_user_article_recommend", "alias" : "reg_user_article_recommend_index" }},                { "add" : { "index" : "es_reg_user_article_recommend_new", "alias" : "reg_user_article_recommend_index"}}        ]}'# create mapping# 创建mappingcurl -XPUT http://192.168.1.1:9200/es_reg_user_goods_article_recommend/et_reg_user_goods_article_recommend/_mapping?pretty -d @/opt/DATA/goldmine/src/mining/user_recommend_mining/goods_article_recommend/create/es_reg_user_goods_article_recommend_mapping.json

3. 创建 HIVE 表

more ./create/hive_to_es_reg_user_goods_article_recommend.sh#!/bin/sh#### 1. del jarhadoop fs -rm /user/hive/warehouse/jars/elasticsearch-hadoop-2.4.1.jar# 2. put jarhadoop fs -put /usr/local/goldmine/hive/default/dg-lib/elasticsearch-hadoop-2.4.1.jar /user/hive/warehouse/jars/# 3. create indexsh create_es_reg_user_article_recommend_mapping####--org.apache.commons.httpclient_3.1.0.v201012070820.jar这个jar包一般是不用加的，除非在报httpclient相关错误的时候才加#add jar hdfs:/opt/data/jar/org.apache.commons.httpclient_3.1.0.v201012070820.jar;\# 4. create table# /usr/local/goldmine/hive/default/dg-lib/elasticsearch-hadoop-2.4.1.jarreg_user_goods_article_recommend_create_table="add jar hdfs:/user/hive/warehouse/jars/elasticsearch-hadoop-2.4.1.jar;\        create external table if not exists des_reg_user_goods_article_recommend(                user_id bigint,                article_ids string,                create_date string        )         STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'         TBLPROPERTIES(                'es.nodes'='192.168.1.1:9200',                'es.resource' = 'es_reg_user_goods_article_recommend/et_reg_user_goods_article_recommend',                'es.index.auto.create'='false',                'es.mapping.id'='user_id',                'es.mapping.names'='article_ids:article_ids,create_date:create_date');"#       --es.resource斜线前面为索引，斜线后面为mapping名称。mapping可以不用事先建好，当数据插入时会自动根据数据类型创建mapping。hive -e "${reg_user_goods_article_recommend_create_table}"

阅读全文

1 0