增量实现

来源：互联网发布：淘宝介入买家怎么赢编辑：程序博客网时间：2024/06/08 05:01

###参数
source ExitCodeCheck.sh
queueName=$1;
hive_db_name_360=$2;
stat_date=$3;
end_date=$4;

#### 新理赔新增 1
hive -v -e "use ${hive_db_name_360};
set mapred.job.queue.name=queue_gbd_ide_02;
set hive.cli.print.header=false;
set hive.map.aggr=true;
set hive.groupby.skewindata=true;

set hive.exec.parallel=true;

insert overwrite table GBD_DM_PAC_SAFE.MID_T1_T1_AUTO_CLAIM_PROPERTY_NEW
select A.REPORT_NO, A.CASE_TIMES
from (select A.REPORT_NO, A.CASE_TIMES
from cx_edw_safe.claim_claimdata_clm_whole_case_base a
where to_date(A.updated_date) < to_date('${end_date}')
and to_date(A.updated_date) >= to_date('${stat_date}')
and a.MIGRATE_FROM in ('N', 'n')
union all
select A.REPORT_NO, A.CASE_TIMES
from cx_edw_safe.claim_claimdata_clm_investigation_task a
where to_date(A.updated_date) < to_date('${end_date}')
and to_date(A.updated_date) >= to_date('${stat_date}')
union all
select A.REPORT_NO, A.CASE_TIMES
from cx_edw_safe.claim_claimdata_clm_car_rob_info a
where to_date(A.updated_date) < to_date('${end_date}')
and to_date(A.updated_date) >= to_date('${stat_date}')
union all
select A.REPORT_NO, A.CASE_TIMES
from cx_edw_safe.claim_claimdata_clm_task_department a
where to_date(A.updated_date) < to_date('${end_date}')
and to_date(A.updated_date) >= to_date('${stat_date}')
and task_code = 'clm_029') A
;
";
exitCodeCheck $?

echo "Run Done 主题临时表!"

#!/bin/sh
source ExitCodeCheck.sh

opts=$@

getparam(){
arg=$1
echo $opts |xargs -n1 |cut -b 2- |awk -F'=' '{if($1=="'"$arg"'") print $2}'
}

IncStart=`getparam inc_start`
IncEnd=`getparam inc_end`
oracle_connection=`getparam jdbc_str`
oracle_username=`getparam db_user`
oracle_password=`getparam db_psw`
dataName=`getparam db_sid`
queueName=`getparam hdp_queue`
hdfshostname=`getparam hdfs_host`;

IncStartYear=`echo ${IncStart:0:4}`;
IncStartMonth=`echo ${IncStart:4:2}`;
IncStartDay=`echo ${IncStart:6:2}`;
IncStartAll=${IncStartYear}"-"${IncStartMonth}"-"${IncStartDay}" 00:00:00.0";
IncEndYear=`echo ${IncEnd:0:4}`;
IncEndMonth=`echo ${IncEnd:4:2}`;
IncEndDay=`echo ${IncEnd:6:2}`;
IncEndAll=${IncEndYear}"-"${IncEndMonth}"-"${IncEndDay}" 00:00:00.0";

hadoop dfs -rmr /apps-data/hduser0101/sx_360_safe/import/MDMDATA_QUESTON_PROJECT;

sqoop import -D mapred.job.queue.name=${queueName} \
--connect ${oracle_connection} \
--username ${oracle_username} \
--password ${oracle_password} \
--query "SELECT /*+ parallel(4) */ \
PHONE_NO, \
AGENT_NO, \
DATE_INVEST, \
SEQUENCE_NO, \
PRODUCT_NO, \
PRODUCT_NAME, \
QUESTION_DESC, \
OPTION_DESC, \
OPTION_DETAIL, \
TRUNC(DATE_EFF,'dd') as DATE_EFF, \
TRUNC(DATE_INVALID,'dd') as DATE_INVALID, \
decode(NAME,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || NAME) as NAME, \
decode(ID_TYPE,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || decode(ID_TYPE,'港澳回乡证或台胞证','港澳台证件','港澳台通行证','港澳台证件',ID_TYPE)) as ID_DESCRIPTION, \
decode(ID_NO,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || ID_NO) as ID_NO, \
decode(DATE_BIRTH,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || DATE_BIRTH) as DATE_BIRTH, \
decode(SEX,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || decode(SEX,'9',NULL,SEX)) as SEX, \
LBS_CLIENT_TYPE, \
CUSTOMER_FLAG, \
CHANNEL_SOURCE, \
DATA_SOURCE, \
CASE WHEN TRUNC(DATE_EFF,'dd') <= last_day(to_date('${IncStart}','yyyymmdd')) \
AND TRUNC(DATE_INVALID,'dd') > last_day(to_date('${IncStart}','yyyymmdd')) \
THEN 'Y' \
ELSE 'N' \
END AS IS_VALID_FLAG, \
QUESTION_NO, \
OPTION_NO, \
ID_TYPE_2 \
from TMP_2015051318174439 \
where 1=1 \
and \$CONDITIONS " \
-m 1 \
--hive-table SX_360_SAFE.MDMDATA_QUESTON_PROJECT \
--hive-drop-import-delims \
--fetch-size 5000 \
--target-dir /apps-data/hduser0101/sx_360_safe/import/mdmdata_queston_project \
--hive-overwrite \
--null-string '\\N' \
--null-non-string '\\N' \
--hive-import;
exitCodeCheck $?

0 0