fix tracking_daily.sh,dm_realtime_service.sh

azkaban/output/reyun/tracking_daily.sh

@@ -25,15 +25,18 @@ for hour in ${hours}; do
 done
 
 if [[ ${log_type} = 'impression' ]]; then
+  EXECUTORS=80
   OUTPUT_PATH="${OUTPUT_REYUN_IMPRESSION_PATH}/${date_path}"
 elif [[ ${log_type} = 'click' ]]; then
+  EXECUTORS=10
   OUTPUT_PATH="${OUTPUT_REYUN_CLICK_PATH}/${date_path}"
 elif [[ ${log_type} = 'install' ]]; then
+  EXECUTORS=10
   OUTPUT_PATH="${OUTPUT_REYUN_INSTALL_PATH}/${date_path}"
 fi
 
 spark-submit --class mobvista.dmp.output.reyun.TrackingLog \
-  --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 10 \
+  --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors ${EXECUTORS} \
   ../../${JAR} -date ${dt} -log_type ${log_type} -output ${OUTPUT_PATH} -coalesce 100
 
 if [ $? -ne 0 ]; then

azkaban/realtime/dm_realtime_service.sh

@@ -28,7 +28,6 @@ unmount_output_path="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dm_us
 
 spark-submit --class mobvista.dmp.datasource.retargeting.DeviceInfoJob \
      --name "DeviceInfoJob.wangjf.${date}" \
-     --conf spark.sql.broadcastTimeout=1200 \
      --conf spark.sql.shuffle.partitions=5000 \
      --conf spark.default.parallelism=5000 \
      --conf spark.kryoserializer.buffer.max=512m \
@@ -37,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.retargeting.DeviceInfoJob \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 8g --executor-cores 3 --num-executors 100 \
+     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 100 \
      ../${JAR} \
      -date ${date} -output ${output_path} -coalesce 2000
 

src/main/scala/mobvista/dmp/output/reyun/Constant.scala

@@ -36,8 +36,8 @@ object Constant {
 
   val user_info =
     """
-      |SELECT dev_id, dev_id_md5, LOWER(dev_type) dev_type, LOWER(platform) platform, UPPER(country) country, install, interest, update_date
+      |SELECT dev_id, CASE WHEN dev_type = 'android_id' THEN 'androidid' ELSE LOWER(dev_type) END dev_type, LOWER(platform) platform, UPPER(country) country, install, interest, update_date
       |  FROM dwh.ods_dmp_user_info_daily
-      |  WHERE dt = '@date' AND UPPER(country) = 'CN'
+      |  WHERE dt = '@date' AND UPPER(country) = 'CN' AND dev_type NOT IN ('sysid', 'ruid')
       |""".stripMargin
 }