FTP linux shell script

#!/bin/ksh


########################################################
#
# argument 1: Buniness Unit
# argument 2: File Type
# argument 3: File extension
#
########################################################


scriptDIR=`dirname $0`
JOBNAME=`basename $0`
CurDate=`date +%Y%m%d`
JobStartDate=`date +%m/%d/%y`
JobStartTime=`date +%H:%M:%S`
curr_datetime=`date +%Y%m%d_%H%M%S`
HOST=`hostname`
TMPFILE=/tmp/`basename $0`.tmp.$$
TMP_OKAY_FLIST=/tmp/`basename $0`_okay_flist.${curr_datetime}.$$
OK_EXT=OK
CURR_DIR=`pwd`


### 20120912
FTP_COUNT=1
FTP_FAILED="N"
### 20120912


###


if [ ! -f ${scriptDIR}/set_env.sh ];
then
echo "Cannot found ${scriptDIR}/set_env.sh, please check"
exit 1
else
echo "Sourcing ${scriptDIR}/set_env.sh environment file ..."
echo 
. ${scriptDIR}/set_env.sh
fi


if [ $# -ne 3 ]; 
then
echo "Usage: $0 <BU> <File Type> <File extension>"
exit 1
else
BU=$1
F_TYPE=$2
F_EXT=$3


echo "BU            : $BU"
echo "File Type     : $F_TYPE"
echo "File Extension: $F_EXT"
echo 
fi


get_env_var () {
SEARCH_STR="${BU}_FTP_${F_TYPE}"
echo 
echo "Search string for ${scriptDIR}/set_env.sh: $SEARCH_STR"
echo 
cat ${scriptDIR}/set_env.sh | grep -v '\#' | grep $SEARCH_STR > ${TMPFILE}
. ${TMPFILE}
rm $TMPFILE

nameref ftp_srv=${SEARCH_STR}_SRV
nameref ftp_user=${SEARCH_STR}_USER
nameref ftp_passwd=${SEARCH_STR}_PASSWD
nameref ftp_src_dir=${SEARCH_STR}_SRC_DIR
nameref ftp_dest_dir=${SEARCH_STR}_DEST_DIR
nameref ftp_log_dir=${SEARCH_STR}_LOG_DIR
nameref ftp_processed_dir=${SEARCH_STR}_PROCESSED_DIR
ftp_logfile="${ftp_log_dir}/`basename $0`_${curr_datetime}.log.$$"




echo 
echo "${SEARCH_STR}_SRV          : $ftp_srv"
echo "${SEARCH_STR}_USER         : $ftp_user"
echo "${SEARCH_STR}_PASSWD       : $ftp_passwd"
echo "${SEARCH_STR}_SRC_DIR      : $ftp_src_dir"
echo "${SEARCH_STR}_DEST_DIR     : $ftp_dest_dir"
echo "${SEARCH_STR}_LOG_DIR      : $ftp_log_dir"
echo "${SEARCH_STR}_PROCESSED_DIR: $ftp_processed_dir"
# echo "ftp_logfile: $ftp_logfile"
echo 
do_checking
}




do_checking () {


if [[ -z ${ftp_srv} ]]; then
echo "ftp server value is null "
exit 1
fi


if [[ -z ${ftp_user} ]]; then
echo "ftp user value is null "
exit 1
fi


if [[ -z ${ftp_passwd} ]]; then
echo "ftp user password value is null "
exit 1
fi


if [ ! -x $ftp_src_dir -o -z  $ftp_src_dir ]; then
echo "Cannot access directory ftp source directory "
exit 1
fi


if [ ! -x $ftp_log_dir -o -z  $ftp_log_dir ]; then
echo "Cannot access directory ftp log directory "
exit 1
fi


if [ ! -x $ftp_processed_dir -o -z  $ftp_processed_dir ]; then
echo "Cannot access directory ftp processed_dir "
exit 1
fi
}




do_ftp () {
        ftpserver=$1
        ftpuser=$2
        ftppasswd=$3
        change_dir=$4


TMP_CMDS="/tmp/`basename $0`_${BU}_${F_TYPE}_${curr_datetime}.ftp.$$"


        if [ -f $TMP_CMDS ];then
           rm $TMP_CMDS
        fi


        echo "ftp -inv $ftp_srv 2>&1 1> $ftp_logfile << EOF" >> ${TMP_CMDS}
        echo "user $ftp_user $ftp_passwd" >> ${TMP_CMDS}
        echo "cd $change_dir" >> ${TMP_CMDS}
        echo "binary" >> ${TMP_CMDS}
        echo "lcd $ftp_src_dir" >> ${TMP_CMDS}
cat $TMP_OKAY_FLIST | sed -e 's/^/put /g' >> ${TMP_CMDS}
        echo "bye" >> ${TMP_CMDS}
        echo "EOF" >> ${TMP_CMDS}
echo 
echo "ftp command file location:  $TMP_CMDS"

### Added 20120912 Ben Lu
while [ ${FTP_COUNT} -lt 4 ]; do
### Added 20120912 Ben Lu
     sh  ${TMP_CMDS}


### Added 20120912 Ben Lu
    if [ ${FTP_COUNT} -lt 3 ]; then 
      echo "TRY FTP ($FTP_COUNT) time"
      FTP_COUNT=`expr ${FTP_COUNT} + 1` 




grep -i  "^Not connected." $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP CONNECT"
  FTP_FAILED="Y"
fi


grep  -i "No such file" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
  echo "$ftp_srv ERROR FTP NO SUCH FILE"
  FTP_FAILED="Y"
fi


grep  -i "access denied" $ftp_logfile  > /dev/null 2>&1
if [  $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP ACCESS DENIED"
  FTP_FAILED="Y"
        fi


        grep  -i  "^Please login" $ftp_logfile > /dev/null 2>&1
        if [ $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP LOGIN"
  FTP_FAILED="Y"
        fi


        grep  -i "Login failed" $ftp_logfile > /dev/null 2>&1
        if [ $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP LOGIN"
  FTP_FAILED="Y"
        fi


     if [ ${FTP_FAILED} = "Y" ]; then 
       FTP_FAILED="N"


       if [ -f $ftp_logfile ]; then
         rm $ftp_logfile
       fi       
       


     else
      FTP_COUNT=4
     fi
    else
      echo "TRY FTP ($FTP_COUNT) time"
### Added 20120912 Ben Lu
     FTP_COUNT=`expr ${FTP_COUNT} + 1` 
grep -i  "^Not connected." $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP CONNECT"
  exit 2
fi


grep  -i "No such file" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
  echo "$ftp_srv ERROR FTP NO SUCH FILE"
  exit 2
fi


grep  -i "access denied" $ftp_logfile  > /dev/null 2>&1
if [  $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP ACCESS DENIED"
  exit 2
        fi


        grep  -i  "^Please login" $ftp_logfile > /dev/null 2>&1
        if [ $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP LOGIN"
  exit 2
        fi


        grep  -i "Login failed" $ftp_logfile > /dev/null 2>&1
        if [ $? -eq 0 ]; then
           echo "$ftp_srv ERROR FTP LOGIN"
  exit 2
        fi


        if [ -f $TMP_CMDS ]; then
           rm $TMP_CMDS
        fi


### Added 20120912 Ben Lu
    fi
done
### Added 20120912 Ben Lu
}


move_ftpfile_to_process_dir () {
        echo "OKAY file list: $TMP_OKAY_FLIST"


cd $ftp_src_dir
echo "Current directory: `pwd`"
if [ -f $TMP_OKAY_FLIST ]; then
size=`stat -c %s $TMP_OKAY_FLIST`
        
if [ $size -gt 0 ]; then
            if [ "${F_EXT}" != "${OK_EXT}" ]; then
     cat $TMP_OKAY_FLIST | grep -v "${OK_EXT}" | while read have_okay_fname
     do
 #f_suffix=`basename $have_okay_fname .zip`
 f_suffix=`basename $have_okay_fname .${F_EXT}`
 f_filename1=${f_suffix}.TMP
                  sed -e '1s/^\xEF\xBB\xBF//' $have_okay_fname > $f_filename1
                  mv $f_filename1 $have_okay_fname
 f_suffix=`basename $have_okay_fname .${F_EXT}`
 echo "mv ${f_suffix}* $ftp_processed_dir"
 mv ${f_suffix}* $ftp_processed_dir
     done
            else
     cat $TMP_OKAY_FLIST | grep "${OK_EXT}" | while read have_okay_fname
     do
 mv_fname=`basename $have_okay_fname`
 echo "mv ${mv_fname} $ftp_processed_dir"
 mv ${mv_fname}* $ftp_processed_dir
     done
            fi  
 fi
fi
}


print_footer () {
echo "
===============================================================
=
= FTP transfer log: $ftp_logfile
= `date`
=
===============================================================
"
}


do_cleanup () {
if [ -f $TMPFILE ]; then
rm $TMPFILE
fi


if [ -f $TMP_OKAY_FLIST ]; then
rm $TMP_OKAY_FLIST
fi
}




#############
# Main
#############




## get ftp related variables
get_env_var


cd $ftp_src_dir
chk_acxiom=`echo ${BU} | awk -F'_' '{print $2}'`
## check for acxiom file type
if [ "${chk_acxiom}" = "ACXIOM" ]; then
ftp_file_pattern="${F_TYPE}*.${F_EXT}"
acxiom_ftype="yes"
else
ftp_file_pattern="${F_TYPE}_${BU}*.${F_EXT}"
acxiom_ftype="no"
fi
flist=`ls -1 $ftp_file_pattern`


echo 
echo "Current Directory : $ftp_src_dir"
echo "Search pattern    : $ftp_file_pattern"
echo 
echo 


if [[ -z ${flist} ]]
then
echo 
echo "No files found for pattern $ftp_file_pattern"
echo 
exit 0
else
        flst_size=`stat -c %s $flist`
        for fname in $flist
do
  file_suffix=`basename $fname .${F_EXT}`
  okay_filename="${file_suffix}.${OK_EXT}"
  if [ -f $okay_filename ];then
                if [[ ${3} != "${OK_EXT}" || $flst_size -gt 1 ]];then
 #zip_fname="${file_suffix}.zip"
 zip_fname=${fname}
 echo "Zipping file $fname to ${zip_fname}"
 echo ""


 #zip ${zip_fname} ${fname}
 if [ $? -eq 0 ]; then
    echo "$zip_fname" >> $TMP_OKAY_FLIST
    echo "$okay_filename" >> $TMP_OKAY_FLIST
    echo "$fname, $okay_filename OK found"
 else
    echo "Problem in zipping file $fname"
    exit 2
 fi 
                else
                  echo "$okay_filename" >> $TMP_OKAY_FLIST 
                  echo "$fname, $okay_filename OK found"
                fi
  else
echo "$fname, $okay_filename OK file NOT found" 
  fi
done

        if [ -f $TMP_OKAY_FLIST ]; then
        size=`stat -c %s $TMP_OKAY_FLIST`
         if [ $size -gt 0 ]; then
      do_ftp $ftp_srv $ftp_user $ftp_passwd $ftp_dest_dir
fi
fi
fi


######
#move uploaded files to processed directory
#######
if [ -f $TMP_OKAY_FLIST ]; then
total_okay_file=`wc -l $TMP_OKAY_FLIST | awk '{print $1}'`
else
total_okay_file=0
fi


if [ -f $ftp_logfile ]; then
   if [ ${acxiom_ftype} = "yes" ];then
total_file_ftp=`cat $ftp_logfile | grep $F_TYPE | grep -v '^150' | wc -l |awk '{print $1}'`
   else
total_file_ftp=`cat $ftp_logfile | grep $BU| grep $F_TYPE | grep -v '^150' | wc -l |awk '{print $1}'`
   fi
else
total_file_ftp=0
fi


echo 
echo "Total number of files need to upload: $total_okay_file"
echo "Total number of files ftp to $ftp_srv: $total_file_ftp"
echo 


if [ $total_okay_file -eq $total_file_ftp ]; then
if [ $total_okay_file -gt 0 ]; then
  move_ftpfile_to_process_dir
  print_footer 
fi
exit 0
else
echo 
echo "Number of files upload does not match !!"
echo 
print_footer 
exit 1
fi


do_cleanup
cd $CURR_DIR