oozie——mapreduce、mysql2hdfs、hdfs2hive
来源:互联网 发布:淘宝店铺利益点介绍 编辑:程序博客网 时间:2024/05/22 04:43
<!-- Copyright (c) 2010 Yahoo! Inc. All rights reserved. Licensed under the
Apache License, Version 2.0 (the "License"); you may not use this file except
in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License.
See accompanying LICENSE file. -->
<workflow-app xmlns="uri:oozie:workflow:0.1" name="wfmaven">
<start to="mr-node" />
<action name="mr-node">
<map-reduce>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<prepare>
<delete path="${nameNode}/user/${wf:user()}/output-data/${outputDir}" />
</prepare>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
<property>
<name>mapred.mapper.class</name>
<value>com.example.WordCount$TokenizerMapper</value>
</property>
<property>
<name>mapred.reducer.class</name>
<value>com.example.WordCount$IntSumReducer</value>
</property>
<property>
<name>mapred.mapoutput.key.class</name>
<value>org.apache.hadoop.io.Text</value>
</property>
<property>
<name>mapred.mapoutput.value.class</name>
<value>org.apache.hadoop.io.IntWritable</value>
</property>
<property>
<name>mapred.output.key.class</name>
<value>org.apache.hadoop.io.Text</value>
</property>
<property>
<name>mapred.output.value.class</name>
<value>org.apache.hadoop.io.IntWritable</value>
</property>
<property>
<name>mapred.map.tasks</name>
<value>1</value>
</property>
<property>
<name>mapred.input.dir</name>
<value>/user/${wf:user()}/input-data/text</value>
</property>
<property>
<name>mapred.output.dir</name>
<value>/user/${wf:user()}/output-data/${outputDir}</value>
</property>
</configuration>
</map-reduce>
<ok to="db_dimAd_import" />
<error to="fail" />
</action>
<action name="db_dimAd_import">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<prepare>
<delete path="${wf_job_base_path}/sqoop/dimAd/old"/>
</prepare>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>import</arg>
<arg>-D</arg>
<arg>mapred.output.compress=false</arg>
<arg>--connect</arg>
<arg>jdbc:mysql://${db_hostname}/${db_name}</arg>
<arg>--table</arg>
<arg>dimAd</arg>
<arg>--target-dir</arg>
<arg>${wf_job_base_path}/sqoop/dimAd/old</arg>
<arg>--num-mappers</arg>
<arg>${sqoop_export_mappers_num}</arg>
<arg>--username</arg>
<arg>${db_user}</arg>
<arg>--password</arg>
<arg>${db_password}</arg>
<arg>--fields-terminated-by</arg>
<arg>\001</arg>
</sqoop>
<ok to="synchronize_bi_dimdb_to_hive"/>
<error to="fail"/>
</action>
<action name="synchronize_bi_dimdb_to_hive">
<hive xmlns="uri:oozie:hive-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.output.compress</name>
<value>false</value>
</property>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
<property>
<name>oozie.hive.defaults</name>
<value>hive-site.xml</value>
</property>
</configuration>
<script>synchronize_bi_dimdb_to_hive.q</script>
<param>app_hive_db_name=${app_hive_db_name}</param>
<param>dimAd_input=${wf_job_base_path}/sqoop/dimAd/old</param>//此处不写old/ 则可以找到期下的part-m-00000 加了找不到
</hive>
<ok to="end"/>
<error to="fail"/>
</action>
<kill name="fail">
<message>Map/Reduce failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<end name="end" />
</workflow-app>
Apache License, Version 2.0 (the "License"); you may not use this file except
in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License.
See accompanying LICENSE file. -->
<workflow-app xmlns="uri:oozie:workflow:0.1" name="wfmaven">
<start to="mr-node" />
<action name="mr-node">
<map-reduce>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<prepare>
<delete path="${nameNode}/user/${wf:user()}/output-data/${outputDir}" />
</prepare>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
<property>
<name>mapred.mapper.class</name>
<value>com.example.WordCount$TokenizerMapper</value>
</property>
<property>
<name>mapred.reducer.class</name>
<value>com.example.WordCount$IntSumReducer</value>
</property>
<property>
<name>mapred.mapoutput.key.class</name>
<value>org.apache.hadoop.io.Text</value>
</property>
<property>
<name>mapred.mapoutput.value.class</name>
<value>org.apache.hadoop.io.IntWritable</value>
</property>
<property>
<name>mapred.output.key.class</name>
<value>org.apache.hadoop.io.Text</value>
</property>
<property>
<name>mapred.output.value.class</name>
<value>org.apache.hadoop.io.IntWritable</value>
</property>
<property>
<name>mapred.map.tasks</name>
<value>1</value>
</property>
<property>
<name>mapred.input.dir</name>
<value>/user/${wf:user()}/input-data/text</value>
</property>
<property>
<name>mapred.output.dir</name>
<value>/user/${wf:user()}/output-data/${outputDir}</value>
</property>
</configuration>
</map-reduce>
<ok to="db_dimAd_import" />
<error to="fail" />
</action>
<action name="db_dimAd_import">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<prepare>
<delete path="${wf_job_base_path}/sqoop/dimAd/old"/>
</prepare>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>import</arg>
<arg>-D</arg>
<arg>mapred.output.compress=false</arg>
<arg>--connect</arg>
<arg>jdbc:mysql://${db_hostname}/${db_name}</arg>
<arg>--table</arg>
<arg>dimAd</arg>
<arg>--target-dir</arg>
<arg>${wf_job_base_path}/sqoop/dimAd/old</arg>
<arg>--num-mappers</arg>
<arg>${sqoop_export_mappers_num}</arg>
<arg>--username</arg>
<arg>${db_user}</arg>
<arg>--password</arg>
<arg>${db_password}</arg>
<arg>--fields-terminated-by</arg>
<arg>\001</arg>
</sqoop>
<ok to="synchronize_bi_dimdb_to_hive"/>
<error to="fail"/>
</action>
<action name="synchronize_bi_dimdb_to_hive">
<hive xmlns="uri:oozie:hive-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.output.compress</name>
<value>false</value>
</property>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
<property>
<name>oozie.hive.defaults</name>
<value>hive-site.xml</value>
</property>
</configuration>
<script>synchronize_bi_dimdb_to_hive.q</script>
<param>app_hive_db_name=${app_hive_db_name}</param>
<param>dimAd_input=${wf_job_base_path}/sqoop/dimAd/old</param>//此处不写old/ 则可以找到期下的part-m-00000 加了找不到
</hive>
<ok to="end"/>
<error to="fail"/>
</action>
<kill name="fail">
<message>Map/Reduce failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<end name="end" />
</workflow-app>
0 0
- oozie——mapreduce、mysql2hdfs、hdfs2hive
- oozie mapreduce demo
- Oozie mapreduce demo
- OOZIE mapreduce节点使用新API
- Oozie MapReduce Action配置的要点
- 运行oozie自带mapreduce例子
- Oozie自带MapReduce示例的运行
- Oozie分布式工作流——EL表达式
- Oozie分布式工作流——EL表达式
- Oozie
- Oozie
- OOZIE
- oozie
- oozie
- oozie
- oozie
- Oozie
- oozie fork多mapreduce任务并行处理示例
- eclipse中添加Server后需要双击Server,配置tomcat所在的路径,不然会404哦
- RSS阅读器Web的更好用
- activiti_SpringEnvironment
- Oracle DB 移动数据
- Jquery绑定事件(bind和live的区别)
- oozie——mapreduce、mysql2hdfs、hdfs2hive
- Android 特殊界面效果之——透明界面
- 用C/C++创建windows服务程序
- [Cocos2D-x For WP8]Sprite精灵
- Oracle Database Server Messages(二) .
- Java 版本Thrift Server 分析
- 清除myeclipse中的web browser里的历史记录
- 谈.Net委托与线程——解决窗体假死
- extjs 跨域问题