shell执行spark-submit

来源:互联网 发布:网络写手挣钱么 编辑:程序博客网 时间:2024/06/07 13:45
#!/bin/bash# chmod +x ./test.sh  修改权限num=0while(($num<2181))do     spark-submit \--master yarn \--deploy-mode cluster \--queue default \--num-executors 45 \--executor-memory 2G \--driver-memory 24G \--archives  \--py-files , \--conf spark.dynamicAllocation.enabled=false \--conf spark.yarn.maxAppAttempts=1 \--conf spark.yarn.executor.memoryOverhead=12288 \myfloder.py  \--num $num \--output predictions    #num=$[num+6]    #echo $num    let "num+=6" # 自加6done