fix adn_sdk_daily.sh,update thread 20

azkaban/adn_sdk/adn_sdk_daily.sh

@@ -26,8 +26,8 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkDaily \
  --conf spark.app.loadTime=${LOG_TIME} \
  --conf spark.app.input_path=${INPUT_PATH} \
  --conf spark.app.output_path=${OUTPUT_PATH} \
- --conf spark.sql.shuffle.partitions=2000 \
- --conf spark.default.parallelism=2000 \
+ --conf spark.sql.shuffle.partitions=5000 \
+ --conf spark.default.parallelism=5000 \
  --conf spark.shuffle.memoryFraction=0.4 \
  --conf spark.storage.memoryFraction=0.4 \
  --conf spark.driver.maxResultSize=8g \
@@ -35,8 +35,9 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkDaily \
  --conf spark.app.coalesce=60000 \
  --files ${HIVE_SITE_PATH} \
  --jars ${JARS} \
- --master yarn --deploy-mode cluster --name adn_sdk_daily --executor-memory 6g --driver-memory 6g  --executor-cores 4 --num-executors 200 \
+ --master yarn --deploy-mode cluster --name adn_sdk_daily --executor-memory 18g --driver-memory 6g  --executor-cores 5 --num-executors 80 \
  ../${JAR}
+
 if [[ $? -ne 0 ]];then
   exit 255
 fi

src/main/java/mobvista/dmp/datasource/rtdmp/RTDmpFetch.java

@@ -86,7 +86,7 @@ public class RTDmpFetch {
         ThreadFactory threadFactory = new ThreadFactoryBuilder()
                 .setNameFormat("RTDmpFetch-%d").build();
 
-        ExecutorService pool = new ThreadPoolExecutor(10, 20,
+        ExecutorService pool = new ThreadPoolExecutor(20, 30,
                 120L, TimeUnit.MILLISECONDS,
                 new LinkedBlockingQueue<>(64), threadFactory, new ThreadPoolExecutor.AbortPolicy());