oozie Hive demo
来源:互联网 发布:2016年新疆网络管制 编辑:程序博客网 时间:2024/05/21 11:12
job.properties
nameNode=hdfs://nameservicejobTracker=yarnRMqueueName=defaultexamplesRoot=examples#oozie.libpath=${nameNode}/user/${user.name}/liboozie.wf.application.path=${nameNode}/user/${user.name}/oozie/hive#outputdir=output-${data.M}
workflow.xml
<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.--><workflow-app xmlns="uri:oozie:workflow:0.2" name="hive-wf"> <start to="hive-node"/> <action name="hive-node"> <hive xmlns="uri:oozie:hive-action:0.2"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <prepare> <delete path="${nameNode}/user/${wf:user()}/output-data/hive"/> <mkdir path="${nameNode}/user/${wf:user()}/output-data"/> </prepare> <configuration> <property> <name>oozie.hive.defaults</name> <value>my-hive-default.xml</value> </property> <property> <name>mapred.job.queue.name</name> <value>${queueName}</value> </property> <property> <name>hive.metastore.local</name> <value>false</value> <description>controls whether to connect to remove metastore server or open a new metastore server in Hive Client JVM</description> </property> <property> <name>hive.metastore.uris</name> <value>thrift://crxy163:9083</value> <description>host and port for the thrift metastore server</description> </property> <property> <name>hive.metastore.warehouse.dir</name> <value>/user/hive/warehouse</value> <description>location of default database for the warehouse</description> </property> </configuration> <script>script.q</script> <param>INPUT=/user/${wf:user()}/input-data/table</param> <param>OUTPUT=/user/${wf:user()}/output-data/hive</param> </hive> <ok to="end"/> <error to="fail"/> </action> <kill name="fail"> <message>Hive failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> </kill> <end name="end"/></workflow-app>
my-hive-default.xml
<?xml version="1.0" encoding="UTF-8"?><!--Autogenerated by Cloudera CM on 2013-04-07T07:44:36.305Z--><configuration> <property> <name>hive.metastore.local</name> <value>false</value> </property> <property> <name>hive.metastore.uris</name> <value>thrift://crxy163:9083</value> </property> <property> <name>hive.metastore.warehouse.dir</name> <value>/user/hive/warehouse</value> </property> <property> <name>hive.warehouse.subdir.inherit.perms</name> <value>true</value> </property> <property> <name>mapred.reduce.tasks</name> <value>-1</value> </property> <property> <name>hive.exec.reducers.bytes.per.reducer</name> <value>1073741824</value> </property> <property> <name>hive.exec.reducers.max</name> <value>999</value> </property> <property> <name>hive.metastore.execute.setugi</name> <value>true</value> </property></configuration>
script.q
create external table IF NOT EXISTS test( IDC_ID bigint, IDC_NAME string, HOUSE_ID string, HOUSE_NAME string, SRCIP string, DESTIP string, SRC_PORT int, DEST_PORT int, DOMAIN_NAME string, URL string, ACCESS_TIME string)partitioned by (datepart string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|' STORED AS SEQUENCEFILElocation '/user/boco/impala/test/';
0 0
- oozie Hive demo
- oozie hive demo
- oozie hive action注意事项
- Hive In Oozie Workflow
- oozie运行hive
- Hive In Oozie Workflow
- oozie HDFS demo
- oozie mapreduce demo
- Oozie mapreduce demo
- oozie hdfs demo
- sqoop+hive+shell+oozie 示例
- Hue、Hive、Sentry、Airflow、Oozie
- Oozie调度sqoop导入hive
- Hive / pig / Sqoop/ Oozie 学习资料
- Oozie Hive action XML Schema版本冲突
- Cloudera Manager 中Oozie 配置HIVE workflow
- Oozie
- Oozie
- 华南师大外国语言文化学院考研历年分数线汇总【2010-2016】
- C#登录界面知识储备
- MySql最左前缀原则
- MySQL调优三步曲(慢查询、explain profile)
- iOS10获取相册/通讯录/相机/麦克风权限崩溃解决办法
- oozie Hive demo
- handler机制的原理
- android 拨打紧急号码
- Ultra-Pull-To-Refresh的一些问题
- Android Studio 单元测试(instrumentation)
- 欢迎使用CSDN-markdown编辑器
- VS中F11
- RTK与网络RTK技术的发展现状
- 字符编码