MapReduce工作笔记——Hadoop MR Streaming通用模板

#!/bin/sh

#  #-----------------------------------#
#   _   _           _
#  | | | | __ _  __| | ___   ___  _ __
#  | |_| |/ _` |/ _` |/ _  / _ | '_ 
#  |  _  | (_| | (_| | (_) | (_) | |_) |
#  |_| |_|__,_|__,_|___/ ___/| .__/
#                                 |_|
#  #-----------------------------------#
#  Created on 2018.11.13
#  Latest modified on 2018.11.13
#  @author: wangcongying
#  #-----------------------------------#
#  MAPPER:   ${CURDIR}/mapper.py
#  REDUCER:  ${CURDIR}/reducer.py
#  #-----------------------------------#

if [ $# != 1 ] ; then
    echo "***********************"
    echo "Parameter error !!!"
    echo "USAGE: ./TestHadoopJob.sh "
    echo " e.g.:"
    echo "***********************"
exit 1;
fi

#  ${HADOOP_HOME}: HADOOP 路径
HADOOP_HOME=""
HDP="$HADOOP_HOME/bin/hadoop fs"

MY_PATH=$(dirname $0)
CUR_DIR=`dirname $(readlink -f $0)`

Today=`date +%Y%m%d`

#  ${INPUT}: HDFS 输入路径
#  ${OUTDIR}: HDFS 输出路径
#  ${JOB_NAME}: MR JOB 命名
INPUT=""
OUTDIR=""
JOB_NAME="wangcongying_${Today}"

echo '===============================================' >> $MY_PATH/hadoop_screen.ans
$HDP -rmr $OUTDIR
$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-streaming.jar 
    -D mapred.job.max.map.running=500 
    -D mapred.job.max.reduce.running=500 
    -D mapred.ignore.badcompress="true" 
    -D mapred.use.multimembergzip="true" 
    -D mapred.max.map.failures.percent=3 
    -D stream.num.map.output.key.fields=1 
    -D mapred.text.key.comparator.options="-k1,1" 
    -D mapred.job.name=$JOB_NAME 
    -D mapred.job.priority="VERY_HIGH" 
    -jobconf mapred.reduce.tasks=100 
    -jobconf mapred.child.env="LANG=en_US.UTF-8,LC_ALL=en_US.UTF-8" 
    -mapper "mapper.py" 
    -reducer "reducer.py"  
    -input ${INPUT} 
    -output ${OUTDIR} 
    -file "$MY_PATH/reducer.py" 
    -file "$MY_PATH/mapper.py" 
2>&1 | tee -a $MY_PATH/hadoop_screen.ans

$HDP -test -e ${OUTDIR}/_SUCCESS
if [ $? -ne 0 ]; then
    echo "ERROR: Hadoop job Hubble Launcher Merge failed, quit!"
    exit 1;
fi
    exit 0;