mysql 导入hive的脚本

Wesley13
• 阅读 587

#!/bin/bash
#tanj
#2017-09-11

DATE=`date +%Y-%m-%d`
TIME=`date +%H:%M:%S`
echo "$DATE $TIME-----------------start-----------------"

if  [ ! -n "$1" ] ;then
    echo "#you have not input a mysqldb!"
    exit 0;
else
    echo "#the mysqldb you input is $1";
    param1=$1;
fi

if  [ ! -n "$2" ] ;then
    echo "#you have not input a pgid!"
    exit 0;
else
    echo "#the pgid you input is $2";
    pgid=$2;
fi

#if flag is 0-- work is successful-- so,  insert result    
#if flag is not 0-- work is failed-- so, not insert result
#flag=1 --createTable is failed
#flag=2 --import      is failed
#flag=3 --appendData  is failed
#flag=4 --insertNum   is failed
#flag=5 --connect     is failed
#flag=6 --tuomin      is failed
#flag=7 --tuominsql   is failed
flag=10

function updateFlag() {
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set end_time='$DATE $TIME',result='$result',resultcode='$flag',status='end'  where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"
}

function createTable() {
echo "-----------------create table-------------------"
CREATEHOST=$HOST
CREATEPORT=$PORT
CREATEUSER=$USER
CREATEPASSWD=$PASSWD
CREATEMYSQLDB=$MYSQL_DB
CREATEHIVEDB=$HIVE_DB
CREATELOCATION=$HDFS_DIR
CREATEMYSQLTABLE=$HIVE_TABLE
CREATEHIVETABLE=$HIVE_TABLE

echo "#mysql -h $CREATEHOST -u $CREATEUSER -p$CREATEPASSWD -N -e \" use $CREATEMYSQLDB ; describe $CREATEMYSQLTABLE ;\"  |awk '{print \$1,\$2}' ";
Table_Fields=`mysql -h ${CREATEHOST} -u ${CREATEUSER} -p${CREATEPASSWD} -N -e " use $CREATEMYSQLDB ; describe ${CREATEMYSQLTABLE} ;" |awk '{print $1,$2}' `
echo "${Table_Fields}";
Fields_Split=(${Table_Fields//,,,/ })
Fields_Len=${#Fields_Split[@]}
echo "$Fields_Len"
create_Table_Str="CREATE TABLE IF NOT EXISTS ${CREATEHIVEDB}.${CREATEHIVETABLE} ("
for (( createI=0; createI < $Fields_Len-1; createI=createI+2 ))
do
field=${Fields_Split[$createI]}
fieldType="string"
field_Type=${Fields_Split[$createI+1]}

echo $field_Type |grep -q "int"
if [ $? -eq 0 ]
then
   fieldType="int"
fi

echo $field_Type | grep -q "decimal"
if [ $? -eq 0 ]
then
   fieldType=$field_Type
fi

echo $field_Type | grep -q "double"
if [ $? -eq 0 ]
then
   fieldType="decimal(18,4)"
fi

echo $field_Type | grep -q "double("
if [ $? -eq 0 ]
then
   fieldType=${field_Type//double/decimal}
fi

echo $field_Type | grep -q "double(" |grep -q ",0)"
if [ $? -eq 0 ]
then
   fieldType=${fieldType//,0/,4}
fi

echo $field_Type | grep -q "bit"
if [ $? -eq 0 ]
then
   fieldType="boolean"
fi

create_Table_Str=${create_Table_Str}'`'$field'` '$fieldType','
done
create_Table_Str=${create_Table_Str}" ROW format delimited fields terminated BY '\001' stored AS textfile LOCATION '${CREATELOCATION}/${CREATEHIVETABLE}';"
echo ${create_Table_Str//, ROW format delimited fields/)ROW format delimited fields}
impala-shell --quiet -B -q "${create_Table_Str//, ROW format delimited fields/)ROW format delimited fields}"

if [ "$?" == "0" ]; then
flag=0;
TIME=`date +%H:%M:%S`
echo " $DATE $TIME $HIVE_TABLE  createTable is successful"

else
flag=1;
TIME=`date +%H:%M:%S`
result="$DATE $TIME $HIVE_TABLE createTable is failed"
echo $result
updateFlag
fi
impala-shell --quiet -B -q " invalidate metadata ${CREATEHIVEDB}.${CREATEHIVETABLE} ";
}

function insertNum() { 
echo "-----------------insert result-------------------"
TIME=`date +%H:%M:%S`
YESTERDAY=`date -d "1 day ago" +%Y-%m-%d`
echo "#impala-shell -q \" REFRESH $HIVE_DB.$HIVE_TABLE ; \";";
impala-shell -q  " REFRESH $HIVE_DB.$HIVE_TABLE ; ";

echo "#impala-shell -q ' select count(1) from $HIVE_DB.$HIVE_TABLE ' | cat|grep -v count |awk '{print $2}'";
hive_num=`impala-shell -q " select count(1) from $HIVE_DB.$HIVE_TABLE " | cat|grep -v count |awk '{print $2}' |tail -n -2|head -n 1`

if [ $hive_num -ge 0 ]; then
echo $hive_num;
else
hive_num=-1
fi

TIME=`date +%H:%M:%S`
echo "#impala-shell -q \"insert into data_market.last_process_time(hivedb,mysqldb,tablename,status,synctime,hive_count_num,datatime) values('$HIVE_DB','$MYSQL_DB','$HIVE_TABLE','$flag','$DATE$TIME',$hive_num,'$YESTERDAY');\";";
impala-shell -q " insert into  data_market.last_process_time(hivedb,mysqldb,tablename,status,synctime,hive_count_num,datatime) values('$HIVE_DB','$MYSQL_DB','$HIVE_TABLE','$flag','$DATE $TIME',$hive_num,'$YESTERDAY') ; ";

if [ "$?" == "0" ]; then
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set hive_count_num=$hive_num  where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"
TIME=`date +%H:%M:%S`
echo " $DATE $TIME MYSQL_DB=$MYSQL_DB MYSQL_TABLE=$MYSQL_TABLE insert into  data_market.last_process_time is successful";

else
flag=4;
TIME=`date +%H:%M:%S`
result="$DATE $TIME MYSQL_DB=$MYSQL_DB MYSQL_TABLE=$MYSQL_TABLE insert into  data_market.last_process_time is failed"
echo $result
updateFlag
fi

}

function appendData() {
echo "-----------------append table-------------------"
appendpgid=${pgid}1

echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;select mysqldb,check_column,last_value,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers from $table_conf where is_enabled=0 and tablename='$MYSQL_TABLE' and mysqldb='$MYSQL_DB'  and pgid=${appendpgid}   limit 1;\" |cat |grep -v mysqldb"
appendinfo=`mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;select mysqldb,check_column,last_value,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers from $table_conf where is_enabled=0 and tablename='$MYSQL_TABLE' and mysqldb='$MYSQL_DB' and pgid=${appendpgid} limit 1 ;" |cat |grep -v mysqldb`;

if [ "$?" == "0" ]; then

echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};\""
mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and  pgid=${appendpgid};"

appendinfosplit=(${appendinfo//,,/ })
appendlen=${#appendinfosplit[@]}

APPENDHOST=${appendinfosplit[3]}
APPENDPORT=${appendinfosplit[4]}
APPENDUSER=${appendinfosplit[5]}
APPENDPASSWD=${appendinfosplit[6]}
APPENDCOLUMN=${appendinfosplit[1]}
APPENDMYSQL_DB=${appendinfosplit[0]}
APPENDLASTVALUE=${appendinfosplit[2]}
APPENDTEMPDIR=${appendinfosplit[7]}
APPENDHDFS_DIR=${appendinfosplit[8]}
APPENDNUMMAPPERS=${appendinfosplit[9]}
echo $APPENDHOST;
echo $APPENDPORT;
echo $APPENDUSER;
echo $APPENDPASSWD;
echo $APPENDCOLUMN;
echo $APPENDMYSQL_DB;
echo $APPENDLASTVALUE;
echo $APPENDTEMPDIR;
echo $APPENDHDFS_DIR;
echo $APPENDNUMMAPPERS;
APPENDTEMPDIR=$APPENDTEMPDIR/$HIVE_TABLE

#APPENDLASTVALUE=`impala-shell --quiet -B -q "SELECT max(systime) FROM  $HIVE_DB.$HIVE_TABLE LIMIT 1"`
echo "mysql -h ${APPENDHOST} -u$APPENDUSER -p$APPENDPASSWD -e \"use $APPENDMYSQL_DB;select * from $HIVE_TABLE  limit 1\"";
mysql -h ${APPENDHOST} -u$APPENDUSER -p$APPENDPASSWD -e "use $APPENDMYSQL_DB;select * from $HIVE_TABLE  limit 1"
  if [ "$?" == "0" ]; then
  echo "#sqoop  import --connect jdbc:mysql://${APPENDHOST}:$APPENDPORT/$APPENDMYSQL_DB --username $APPENDUSER --password '$APPENDPASSWD' \
   --table $HIVE_TABLE --incremental append --check-column $APPENDCOLUMN --outdir /root/tanj/sh/java \
   --hive-import --hive-database $HIVE_DB --hive-table $HIVE_TABLE \
   --target-dir $APPENDTEMPDIR --delete-target-dir --fields-terminated-by '\001'  \
   --hive-drop-import-delims --null-string '\\\\N' --null-non-string '\0' -m $APPENDNUMMAPPERS --last-value '${APPENDLASTVALUE};';";
  sqoop  import --connect jdbc:mysql://${APPENDHOST}:$APPENDPORT/$APPENDMYSQL_DB --username $APPENDUSER --password ''${APPENDPASSWD}'' \
   --table $HIVE_TABLE --incremental append --check-column $APPENDCOLUMN --outdir /root/tanj/sh/java \
   --hive-import --hive-database $HIVE_DB --hive-table $HIVE_TABLE \
   --target-dir $APPENDTEMPDIR --delete-target-dir --fields-terminated-by "\001"  \
   --hive-drop-import-delims --null-string '\\N' --null-non-string "\0" -m $APPENDNUMMAPPERS --last-value "${APPENDLASTVALUE};";

     if [ "$?" == "0" -a "$flag" == "0" ]; then
     flag=0;
     TIME=`date +%H:%M:%S`
     echo " $DATE $TIME $HIVE_TABLE  append is successful"
     else
     flag=3;
     TIME=`date +%H:%M:%S`
     result="$DATE $TIME $HIVE_TABLE append is failed"
     echo $result
     updateFlag
     fi
  impala-shell -q " REFRESH $HIVE_DB.$HIVE_TABLE ";
  fi
impala-shell --quiet -B -q " invalidate metadata $HIVE_DB.$HIVE_TABLE ";
TIME=`date +%H:%M:%S`  
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; update $table_conf set updatetime='$DATE $TIME',end_time='$DATE $TIME',status='end',run_time=TIMEDIFF($TIME,$startTime)  where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};\";";
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set updatetime='$DATE $TIME',end_time='$DATE $TIME',status='end',run_time=TIMEDIFF($TIME,$startTime)  where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};"
  
else
echo " $DATE $TIME $HIVE_TABLE not exists append "
fi

}

function runSqoop() {
#-------------------- create datebases -------------------------
echo "impala-shell -q \" CREATE DATABASE IF NOT EXISTS ${HIVE_DB} \";";
impala-shell -q " CREATE DATABASE IF NOT EXISTS ${HIVE_DB} ";

echo "-----------------create table-------------------"
impala-shell -q " drop table $HIVE_DB.$HIVE_TABLE "
createTable;
impala-shell --quiet -B -q " REFRESH $HIVE_DB.$HIVE_TABLE "

echo "#mysql -h $HOST -u$USER -p$PASSWD -e\" use $MYSQL_DB ; select * from $MYSQL_TABLE limit 1 \";";
mysql -h $HOST -u$USER -p$PASSWD -e" use $MYSQL_DB ; select * from $MYSQL_TABLE limit 1 "

if [ "$?" == "0" -a  "$flag" == "0" ]; then
echo "-----------------import table-------------------";

#echo $TEMPDIR | grep -q "$sqoop_temp_dir";
#   if [ $? -eq 0 ]
#   then
#   echo "hdfs dfs -rm -R $TEMPDIR";
#   hdfs dfs -rm -R $TEMPDIR
#   fi

echo "#sqoop import --connect jdbc:mysql://$HOST:$PORT/$MYSQL_DB  --username $USER --password '$PASSWD' --table $MYSQL_TABLE \
       --hive-import --hive-overwrite  --hive-database $HIVE_DB --fields-terminated-by '\001' --target-dir $TEMPDIR --delete-target-dir  --outdir /root/tanj/sh/java \
       --hive-drop-import-delims --null-string '\\\\N' --null-non-string '\0'  -m $NUMMAPPERS";
PASSWD1="'$PASSWD'"
sqoop import --connect jdbc:mysql://$HOST:$PORT/$MYSQL_DB  --username $USER --password ''${PASSWD}'' --table $MYSQL_TABLE \
--hive-import --hive-overwrite  --hive-database $HIVE_DB --fields-terminated-by '\001' --target-dir $TEMPDIR --delete-target-dir --outdir /root/tanj/sh/java \
--hive-drop-import-delims --null-string '\\N' --null-non-string '\0'  -m $NUMMAPPERS

   if [ "$?" == "0" ]; then
   TIME=`date +%H:%M:%S`
   echo " $DATE $TIME sqoop import $MYSQL_DB.$MYSQL_TABLE is successful";
   else
   flag=2;
   TIME=`date +%H:%M:%S`
   result="$DATE $TIME sqoop import $MYSQL_DB.$MYSQL_TABLE  is failed";
   echo $result
   updateFlag
   fi

#"-----------------append table-------------------"   
   if [ "$?" == "0" -a "$flag" == "0" ]; then
      appendData
   fi
   
elif [ "$?" != "0" -a  "$flag" == "0" ]; then
   flag=5;
   TIME=`date +%H:%M:%S`
   result="mysql -h $HOST -u$USER -p$PASSWD -e\" use $MYSQL_DB ; select * from $MYSQL_TABLE limit 1 \"  is failed"
   echo $result
   updateFlag
fi

impala-shell --quiet -B -q " invalidate metadata $HIVE_DB.$HIVE_TABLE ";
}

host_conf='127.0.0.1'
user_conf='root'
passwd_conf='123456'
db_conf='bigdata_shell'
table_conf='sqoop_append'
sqoop_temp_dir='/user/root/temp_sqoop'

TIME=`date +%H:%M:%S`
startTime=$TIME
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;select mysqldb,tablename,hivedb,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers,tuomindb from $table_conf where is_enabled=0 and pgid = $pgid ;\" |cat "
arr=`mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;select mysqldb,tablename,hivedb,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers,tuomindb from $table_conf where is_enabled=0 and pgid = $pgid ;" |cat `;
split=(${arr//,,,/ })
len=${#split[@]}

for (( i=0; i < $len-1; i=i+11 ))
do
HOST=${split[$i+3]}
PORT=${split[$i+4]}
USER=${split[$i+5]}
PASSWD=${split[$i+6]}
MYSQL_TABLE=${split[$i+1]}
MYSQL_DB=${split[$i]}
HIVE_DB=${split[$i+2]}
TEMPDIR=${split[$i+7]}
HDFS_DIR=${split[$i+8]}
NUMMAPPERS=${split[$i+9]}
TUOMIN_DB=${split[$i+10]}
echo "HOST="$HOST;
echo "PORT="$PORT;
echo "USER="$USER;
echo "PASSWD="$PASSWD;
echo "MYSQL_TABLE="$MYSQL_TABLE;
echo "MYSQL_DB="$MYSQL_DB;
echo "HIVE_DB="$HIVE_DB;
echo "TEMPDIR="$TEMPDIR;
echo "HDFS_DIR="$HDFS_DIR;
echo "NUMMAPPERS="$NUMMAPPERS;
echo "TUOMIN_DB="${TUOMIN_DB};
HIVE_TABLE=$MYSQL_TABLE
TEMPDIR=${TEMPDIR}/$HIVE_TABLE

echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid ;\""
mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"

if [ "$?" == "0" ]; then
   echo "#mysql -h $host_conf -uroot -p$passwd_conf -N -e ""\"use $db_conf;select is_tuomin from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' limit 1 ;\"";
   is_tuomin=`mysql -h $host_conf -uroot -p$passwd_conf -N -e "use $db_conf;select is_tuomin from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' limit 1 ;"`
   
   if [ "$is_tuomin" == "0" ]; then
   #-------------------- create tuomin datebases -------------------------
   echo "impala-shell -q \" CREATE DATABASE IF NOT EXISTS ${TUOMIN_DB} \";";
   impala-shell -q " CREATE DATABASE IF NOT EXISTS ${TUOMIN_DB}";
   
   runSqoop
   echo "#mysql -h $host_conf -uroot -p$passwd_conf -N -e \"use $db_conf;select tuominsql from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and is_tuomin = 0 limit 1 ;\"";
   tuominsql=`mysql -h $host_conf -uroot -p$passwd_conf -N -e "use $db_conf;select tuominsql from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and is_tuomin = 0 limit 1 ;"`
   echo "impala-shell --quiet -B -q \" invalidate metadata $HIVE_DB.$HIVE_TABLE \";" 
   impala-shell --quiet -B -q " invalidate metadata $HIVE_DB.$HIVE_TABLE ";
   echo "#impala-shell -q \"DROP TABLE IF EXISTS  ${TUOMIN_DB}.$MYSQL_TABLE;\"";
   impala-shell -q "DROP TABLE IF EXISTS  ${TUOMIN_DB}.$MYSQL_TABLE;"
   echo "-----------------run tuominsql-------------------" 
   echo "#impala-shell -q  \"$tuominsql\"";
   impala-shell -q "$tuominsql"
      if  [ "$?" == "0" ] ;then
      echo "impala-shell -q \"DROP TABLE IF EXISTS  ${HIVE_DB}.$MYSQL_TABLE;\";"
#      impala-shell -q "DROP TABLE IF EXISTS  ${HIVE_DB}.$MYSQL_TABLE;"
       
      else
      TIME=`date +%H:%M:%S`
      flag=7;
      result="$DATE $TIME $HIVE_TABLE tuominsql is failed"
      echo $result
      updateFlag
      fi
      
   elif  [ "$is_tuomin" == "1" ]; then

   #----------dev,prod------------- 
#   HDFS_DIR=${HDFS_DIR//$HIVE_DB/$TUOMIN_DB}
#   TEMPDIR=${TEMPDIR//$HIVE_DB/$TUOMIN_DB}
#   HIVE_DB=${TUOMIN_DB}

   runSqoop
   fi
   
   if [ "$?" != "0" ]; then
      TIME=`date +%H:%M:%S`
      flag=6;
      result="$DATE $TIME $HIVE_TABLE tuomin is failed"
      echo $result
      updateFlag    
   fi

   hive_num=-1
#   HIVE_DB=${TUOMIN_DB}
   insertNum
   
   if [ "$?" == "0" -a "$flag" == "0" ]; then
   TIME=`date +%H:%M:%S`
   result="$HIVE_DB.$HIVE_TABLE is successful"
   echo $result
   updateFlag
   fi
   
else
flag=5;
TIME=`date +%H:%M:%S`
result="mysql -h $HOST -u$USER -p$PASSWD -e\" use $MYSQL_DB ; UPDATE  is failed"
echo $result
updateFlag   
fi
 
TIME=`date +%H:%M:%S`
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; update $table_conf set end_time='$DATE $TIME',resultcode='$flag',status='end',run_time=TIMEDIFF(end_time,start_time)  where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;\";";
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set end_time='$DATE $TIME',resultcode='$flag',status='end',run_time=TIMEDIFF(end_time,start_time)  where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"

done

if  [ "$?" == "0" ] ;then
echo "-----------------check data-------------------"
pid=`ps -ef |grep  "$0 ${param1} 1000" |grep -v grep |awk '{print $2}'`
pids=(${pid//,,,/ })
pidl=${#pids[@]}
echo $pidl
if [ $pidl -gt 0 ]; then
exit 0;

else

echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; select id,pgid from $table_conf where mysqldb='$MYSQL_DB' and is_enabled=0 and status='end' and resultcode != 0; \" |cat ;";
errorpgid=`mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; select id,pgid from $table_conf where mysqldb='$MYSQL_DB' and is_enabled=0 and status='end' and resultcode != 0;" |cat`
errorpgids=(${errorpgid//,,,/ })
errorpgidl=${#errorpgids[@]}
   if [ $errorpgidl > 1 ]; then
      echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; update $table_conf set pgid=1000 where mysqldb='$MYSQL_DB' and is_enabled=0 and resultcode != 0; \" ;";
      mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set pgid=1000 where mysqldb='$MYSQL_DB' and is_enabled=0 and resultcode != 0;"
      echo "sh $0 ${param1} 1000"
      sh $0 ${param1} 1000
      for (( i=0; i < $errorpgidl-1; i=i+2 ))
      do
          ID=${errorpgids[$i]}
          PGID=${errorpgids[$i+1]}
          mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set pgid=$PGID where id=$ID ;"
      done
   fi
fi

echo "-----------------END-------------------"
fi

if  [ "$?" == "0" -a "zhifu" == "${param1}" ] ;then
zhifupid=`ps -ef |grep  "$0 ${param1}" |grep -v grep |awk '{print $2}'`
pidsplit=(${zhifupid//,,,/ })
pidlen=${#pidsplit[@]}

   if [ 3 -eq $pidlen ]; then
   processid=`yarn application -list|grep 'com.yanxin.javaspark.application.NewImportHDFSHiveData'|awk '{print $1}'`

      if [ $? == 0 ]; then
      echo "applicationId="$processid
      yarn application -kill $processid
      hdfs dfs -rm -R /user/hive/warehouse/real_time_zhifu.db/*
      hdfs dfs -cp /user/hive/warehouse/transfer_zhifu.db/* /user/hive/warehouse/real_time_zhifu.db
      sh /root/harry/NewImportHDFSHiveData_submit.sh
      fi

   else
      echo "#the zhifupid is $zhifuid";
      exit 0;
   fi
fi

if  [ "$?" == "0" -a "sjdk" == "${param1}" ] ;then
sjdkpid=`ps -ef |grep  "$0 ${param1}" |grep -v grep |awk '{print $2}'`
pidsplit=(${sjdkpid//,,,/ })
pidlen=${#pidsplit[@]}
echo $pidlen
   if [ 3 -eq $pidlen ]; then
   DATE=`date +%Y-%m-%d`
   TIME=`date +%H:%M:%S`
   YESTERDAY=`date -d "1 day ago" +%Y-%m-%d`
   sql="insert into data_market.last_process_time SELECT 'NULL' AS syncid,AA.hivedb,AA.tablename,'0' as status,'$DATE $TIME' as  synctime , AA.hive_count_num,'sjdk' mysqldb,'$YESTERDAY' datatime from
   (SELECT hivedb,tablename,hive_count_num FROM data_market.last_process_time WHERE mysqldb='sjdk' and substr(synctime,1,10)='$YESTERDAY' AND tablename NOT IN
   (SELECT tablename from data_market.last_process_time WHERE mysqldb='sjdk' and substr(synctime,1,10)='$DATE')) as AA;"
   echo "impala-shell -q \"$sql\";"
   impala-shell -q "$sql";
   
   else
      echo "#the sjdkpid is $sjdkpid";
      exit 0;

   fi
fi

点赞
收藏
评论区
推荐文章
blmius blmius
2年前
MySQL:[Err] 1292 - Incorrect datetime value: ‘0000-00-00 00:00:00‘ for column ‘CREATE_TIME‘ at row 1
文章目录问题用navicat导入数据时,报错:原因这是因为当前的MySQL不支持datetime为0的情况。解决修改sql\mode:sql\mode:SQLMode定义了MySQL应支持的SQL语法、数据校验等,这样可以更容易地在不同的环境中使用MySQL。全局s
Jacquelyn38 Jacquelyn38
2年前
2020年前端实用代码段,为你的工作保驾护航
有空的时候,自己总结了几个代码段,在开发中也经常使用,谢谢。1、使用解构获取json数据let jsonData  id: 1,status: "OK",data: 'a', 'b';let  id, status, data: number   jsonData;console.log(id, status, number )
皕杰报表之UUID
​在我们用皕杰报表工具设计填报报表时,如何在新增行里自动增加id呢?能新增整数排序id吗?目前可以在新增行里自动增加id,但只能用uuid函数增加UUID编码,不能新增整数排序id。uuid函数说明:获取一个UUID,可以在填报表中用来创建数据ID语法:uuid()或uuid(sep)参数说明:sep布尔值,生成的uuid中是否包含分隔符'',缺省为
Stella981 Stella981
2年前
AndroidStudio封装SDK的那些事
<divclass"markdown\_views"<!flowchart箭头图标勿删<svgxmlns"http://www.w3.org/2000/svg"style"display:none;"<pathstrokelinecap"round"d"M5,00,2.55,5z"id"raphael
Stella981 Stella981
2年前
KVM调整cpu和内存
一.修改kvm虚拟机的配置1、virsheditcentos7找到“memory”和“vcpu”标签,将<namecentos7</name<uuid2220a6d1a36a4fbb8523e078b3dfe795</uuid
Stella981 Stella981
2年前
Python之time模块的时间戳、时间字符串格式化与转换
Python处理时间和时间戳的内置模块就有time,和datetime两个,本文先说time模块。关于时间戳的几个概念时间戳,根据1970年1月1日00:00:00开始按秒计算的偏移量。时间元组(struct_time),包含9个元素。 time.struct_time(tm_y
Wesley13 Wesley13
2年前
mysql设置时区
mysql设置时区mysql\_query("SETtime\_zone'8:00'")ordie('时区设置失败,请联系管理员!');中国在东8区所以加8方法二:selectcount(user\_id)asdevice,CONVERT\_TZ(FROM\_UNIXTIME(reg\_time),'08:00','0
Wesley13 Wesley13
2年前
00:Java简单了解
浅谈Java之概述Java是SUN(StanfordUniversityNetwork),斯坦福大学网络公司)1995年推出的一门高级编程语言。Java是一种面向Internet的编程语言。随着Java技术在web方面的不断成熟,已经成为Web应用程序的首选开发语言。Java是简单易学,完全面向对象,安全可靠,与平台无关的编程语言。
Stella981 Stella981
2年前
Django中Admin中的一些参数配置
设置在列表中显示的字段,id为django模型默认的主键list_display('id','name','sex','profession','email','qq','phone','status','create_time')设置在列表可编辑字段list_editable
Wesley13 Wesley13
2年前
MySQL部分从库上面因为大量的临时表tmp_table造成慢查询
背景描述Time:20190124T00:08:14.70572408:00User@Host:@Id:Schema:sentrymetaLast_errno:0Killed:0Query_time:0.315758Lock_
Python进阶者 Python进阶者
2个月前
Excel中这日期老是出来00:00:00,怎么用Pandas把这个去除
大家好,我是皮皮。一、前言前几天在Python白银交流群【上海新年人】问了一个Pandas数据筛选的问题。问题如下:这日期老是出来00:00:00,怎么把这个去除。二、实现过程后来【论草莓如何成为冻干莓】给了一个思路和代码如下:pd.toexcel之前把这