oozie hive demo

job.properties

nameNode=hdfs://nameservicejobTracker=yarnRMqueueName=defaultexamplesRoot=examples#oozie.libpath=${nameNode}/user/${user.name}/liboozie.wf.application.path=${nameNode}/user/${user.name}/oozie/hive#outputdir=output-${data.M}
1
2
3
4
5
6
7
8
1
2
3
4
5
6
7
8

workflow.xml

<?xml version="1.0" encoding="UTF-8"?><!--  Licensed to the Apache Software Foundation (ASF) under one  or more contributor license agreements.  See the NOTICE file  distributed with this work for additional information  regarding copyright ownership.  The ASF licenses this file  to you under the Apache License, Version 2.0 (the  "License"); you may not use this file except in compliance  with the License.  You may obtain a copy of the License at       http://www.apache.org/licenses/LICENSE-2.0  Unless required by applicable law or agreed to in writing, software  distributed under the License is distributed on an "AS IS" BASIS,  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the License for the specific language governing permissions and  limitations under the License.--><workflow-app xmlns="uri:oozie:workflow:0.2" name="hive-wf">    <start to="hive-node"/>    <action name="hive-node">        <hive xmlns="uri:oozie:hive-action:0.2">            <job-tracker>${jobTracker}</job-tracker>            <name-node>${nameNode}</name-node>            <prepare>                <delete path="${nameNode}/user/${wf:user()}/output-data/hive"/>                <mkdir path="${nameNode}/user/${wf:user()}/output-data"/>            </prepare>            <configuration>                <property>                      <name>oozie.hive.defaults</name>                      <value>my-hive-default.xml</value>                  </property>                 <property>                    <name>mapred.job.queue.name</name>                    <value>${queueName}</value>                </property>                <property>                        <name>hive.metastore.local</name>                        <value>false</value>                        <description>controls whether to connect to remove metastore server or open a new metastore server in Hive Client JVM</description>                      </property>                      <property>                        <name>hive.metastore.uris</name>                        <value>thrift://crxy163:9083</value>                        <description>host and port for the thrift metastore server</description>                      </property>                      <property>                        <name>hive.metastore.warehouse.dir</name>                        <value>/user/hive/warehouse</value>                        <description>location of default database for the warehouse</description>                      </property>             </configuration>            <script>script.q</script>            <param>INPUT=/user/${wf:user()}/input-data/table</param>            <param>OUTPUT=/user/${wf:user()}/output-data/hive</param>        </hive>        <ok to="end"/>        <error to="fail"/>    </action>    <kill name="fail">        <message>Hive failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>    </kill>    <end name="end"/></workflow-app>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

my-hive-default.xml

<?xml version="1.0" encoding="UTF-8"?><!--Autogenerated by Cloudera CM on 2013-04-07T07:44:36.305Z--><configuration>  <property>    <name>hive.metastore.local</name>    <value>false</value>  </property>  <property>    <name>hive.metastore.uris</name>    <value>thrift://crxy163:9083</value>  </property>  <property>    <name>hive.metastore.warehouse.dir</name>    <value>/user/hive/warehouse</value>  </property>  <property>    <name>hive.warehouse.subdir.inherit.perms</name>    <value>true</value>  </property>  <property>    <name>mapred.reduce.tasks</name>    <value>-1</value>  </property>  <property>    <name>hive.exec.reducers.bytes.per.reducer</name>    <value>1073741824</value>  </property>  <property>    <name>hive.exec.reducers.max</name>    <value>999</value>  </property>  <property>    <name>hive.metastore.execute.setugi</name>    <value>true</value>  </property></configuration>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

script.q

create external table IF NOT EXISTS  test( IDC_ID    bigint, IDC_NAME    string, HOUSE_ID  string, HOUSE_NAME    string, SRCIP    string, DESTIP    string, SRC_PORT    int, DEST_PORT    int, DOMAIN_NAME    string, URL    string, ACCESS_TIME    string)partitioned by (datepart string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\|' STORED AS SEQUENCEFILElocation '/user/boco/impala/test/';