数据仓库-shell脚本实现hive hdfs文件上传到目标ftp

总的来说就是一行一行的linux命令，组装起来就变成一个脚本了！

 

#!/bin/bash

cd /web/data/file/tmpjf

rm -rf bm_jingfeng_export_zte_login_d_{Date_No}.txt
rm -rf bm_jingfeng_export_cms_content_d_{Date_No}.txt
rm -rf bm_jingfeng_export_channel_info_d_{Date_No}.txt
rm -rf bm_jingfeng_export_stb_info_d_{Date_No}.txt



#导出hdfs上面的文件到指定目录,date_no为批次
hadoop fs -getmerge hdfs://SH1/user/hive/warehouse/nginxlogdb.db/bm_jingfeng_export_zte_login_d/date_no={Date_No}/ /web/data/file/tmpjf/bm_jingfeng_export_zte_login_d_{Date_No}.txt
hadoop fs -getmerge hdfs://SH1/user/hive/warehouse/nginxlogdb.db/bm_jingfeng_export_cms_content_d/date_no={Date_No}/ /web/data/file/tmpjf/bm_jingfeng_export_cms_content_d_{Date_No}.txt
hadoop fs -getmerge hdfs://SH1/user/hive/warehouse/nginxlogdb.db/bm_jingfeng_export_channel_info_d/date_no={Date_No}/ /web/data/file/tmpjf/bm_jingfeng_export_channel_info_d_{Date_No}.txt
hadoop fs -getmerge hdfs://SH1/user/hive/warehouse/nginxlogdb.db/bm_jingfeng_export_stb_info_d/date_no={Date_No}/ /web/data/file/tmpjf/bm_jingfeng_export_stb_info_d_{Date_No}.txt


#以下代码为文件名操作，可忽略####################
function rand(){
  min=$1
  max=$(($2-$min+1))
  num=$(($RANDOM+1000000000)) #增加一个10位的数再求余
  echo $(($num%$max+$min))
}

#生成三位随机数
rnd=$(rand 100 999)
#生成两位随机数
rnd2=$(rand 10 99)
#echo $rnd
  
mv bm_jingfeng_export_zte_login_d_{Date_No}.txt "s_DD_BAS_21048_{Date_No}_"$rnd2"_"$rnd.dat

mv bm_jingfeng_export_cms_content_d_{Date_No}.txt "i_DDcontinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat

mv bm_jingfeng_export_channel_info_d_{Date_No}.txt "i_DDchannelinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat

mv bm_jingfeng_export_stb_info_d_{Date_No}.txt "i_DDstbinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat

#获取文件行数和大小(字节)生成校验文件
line=$(cat "s_DD_BAS_21048_{Date_No}_"$rnd2"_"$rnd.dat |wc -l);
size=$(cat "s_DD_BAS_21048_{Date_No}_"$rnd2"_"$rnd.dat |wc -c);
echo "s_DD_BAS_21048_{Date_No}_"$rnd2"_"$rnd".dat|"$line"|"$size > "s_DD_BAS_21048_{Date_No}_"$rnd2.verf

line=$(cat "i_DDcontinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat |wc -l);
size=$(cat "i_DDcontinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat |wc -c);
echo "i_DDcontinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd".dat|"$line"|"$size > "i_DDcontinfo_BAS_02172_{Date_No}_"$rnd2.verf

line=$(cat "i_DDchannelinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat |wc -l);
size=$(cat "i_DDchannelinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat |wc -c);
echo "i_DDchannelinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd".dat|"$line"|"$size > "i_DDchannelinfo_BAS_02172_{Date_No}_"$rnd2.verf

line=$(cat "i_DDstbinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat |wc -l);
size=$(cat "i_DDstbinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat |wc -c);
echo "i_DDstbinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd".dat|"$line"|"$size > "i_DDstbinfo_BAS_02172_{Date_No}_"$rnd2.verf

gzip "s_DD_BAS_21048_{Date_No}_"$rnd2"_"$rnd.dat
gzip "i_DDcontinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat
gzip "i_DDchannelinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat
gzip "i_DDstbinfo_BAS_02172_{Date_No}_"$rnd2"_"$rnd.dat

#以上代码为文件名操作，可忽略####################


rm -rf ./tempjf
mkdir ./tempjf
#split -100000 dwd_itv_cdn_mbh_d_{Date_No}.txt stats_pkg_mbh_iptv_
mv *.* ./tempjf/
cd ./tempjf

# index=0
# stats_pkg_mbh_iptv_20191013.txt
# for i in `ls | grep stats_pkg_mbh_iptv_`
# do
	# index=`expr $index + 1`;
	# if [ "$index" -eq "1" ];then 
		# filename=stats_pkg_mbh_iptv_{Date_No}.txt
	# else 
		# filename=stats_pkg_mbh_iptv_{Date_No}_$index.txt
	# fi
	# echo $filename
	# mv $i $filename
	#tar -zcvf $filename".tar.gz" $filename".txt"
# done

#这里上传文件
ftp -n<<! 
open 11.111.11.111
user ftp_account pwd 
binary
prompt 
cd /login
mkdir {Date_No}
cd {Date_No}
mdel *.*
mput s_DD_BAS_21048_*

cd ../../continfo
mkdir {Date_No}
cd {Date_No}
mdel *.*
mput i_DDcontinfo_BAS_02172_*

cd ../../channelinfo
mkdir {Date_No}
cd {Date_No}
mdel *.*
mput i_DDchannelinfo_BAS_02172_*

cd ../../stbinfo
mkdir {Date_No}
cd {Date_No}
mdel *.*
mput i_DDstbinfo_BAS_02172_*
close 
bye 
!

 

来源：CSDN

作者：iayay222

链接：https://blog.csdn.net/jingjingaibiancheng/article/details/103480428