update RTDmpRequest

azkaban/datatory/user_info/user_info_ck.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.clickhouse.feature.UserInfo \
      --conf spark.kryoserializer.buffer=64m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --files ${HIVE_SITE_PATH} \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 16 \
+     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 12 \
      ../../${JAR} -date ${date} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 
 if [[ $? -ne 0 ]]; then

azkaban/rtdmp/rtdmp_request.sh

@@ -16,25 +16,25 @@ elif [[ ${business} = 'dsp_req' ]]; then
   date=$(date +"%Y%m%d" -d "-1 day $today")
   date_path=$(date +%Y/%m/%d -d "-1 day $today")
   partition=2000
-  executors=20
+  executors=100
   coalesce=100
 elif [[ ${business} = 'btop' ]]; then
   date=$(date +"%Y%m%d" -d "-1 day $today")
   date_path=$(date +%Y/%m/%d -d "-1 day $today")
   partition=20
-  executors=5
+  executors=20
   coalesce=40
 elif [[ ${business} = 'uc_activation' ]]; then
   date=$(date +"%Y%m%d" -d "0 day $today")
   date_path=$(date +%Y/%m/%d -d "0 day $today")
   partition=100
-  executors=5
+  executors=20
   coalesce=40
 elif [[ ${business} = 'iqiyi_activation' ]]; then
   date=$(date +"%Y%m%d" -d "0 day $today")
   date_path=$(date +%Y/%m/%d -d "0 day $today")
   partition=100
-  executors=5
+  executors=10
   coalesce=40
 elif [[ ${business} = 'alipay_activation' ]]; then
   date=$(date +"%Y%m%d" -d "1 day $today")
@@ -42,7 +42,7 @@ elif [[ ${business} = 'alipay_activation' ]]; then
   hh=$(expr $hour / 6 + 1)
   date_path=$(date +%Y/%m/%d/0${hh} -d "1 day $today")
   partition=100
-  executors=5
+  executors=10
   coalesce=40
 elif [[ ${business} = 'alipay_acquisition' ]]; then
   date=$(date +"%Y%m%d" -d "1 day $today")
@@ -50,13 +50,13 @@ elif [[ ${business} = 'alipay_acquisition' ]]; then
   hh=$(expr $hour / 6 + 1)
   date_path=$(date +%Y/%m/%d/0${hh} -d "1 day $today")
   partition=100
-  executors=5
+  executors=10
   coalesce=40
 elif [[ ${business} = 'youku_acquisition' ]]; then
   date=$(date +"%Y%m%d" -d "0 day $today")
   date_path=$(date +%Y/%m/%d -d "0 day $today")
   partition=100
-  executors=5
+  executors=10
   coalesce=40
 elif [[ ${business} = 'tencent' ]]; then
   date=$(date +"%Y%m%d" -d "-1 day $today")
@@ -68,7 +68,7 @@ elif [[ ${business} = 'other' ]]; then
   date=$(date +"%Y%m%d" -d "-2 day $today")
   date_path=$(date +%Y/%m/%d -d "-2 day $today")
   partition=1000
-  executors=10
+  executors=20
   coalesce=40
 fi
 

src/main/resources/config.properties

@@ -105,7 +105,8 @@ dsp_req.package_name=com.taobao.taobao_oppo,com.eg.android.AlipayGphone_oppo,com
   com.eg.android.AlipayGphone_bes,com.youku.phone_notinstall_oppo,com.sankuai.meituan_oppo,com.meituan.itakeaway_oppo,com.taobao.idlefish_bes,\
   com.taobao.idlefish_oppo,com.UCMobile_bes,com.taobao.taobao_bes,com.tencent.news_fromtencent,com.taobao.taobao_iqiyi,com.taobao.taobao,com.UCMobile_iqiyi,\
   com.UCMobile,com.eg.android.AlipayGphone_iqiyi,com.eg.android.AlipayGphone,com.taobao.idlefish_iqiyi,com.taobao.idlefish,com.sankuai.meituan_iqiyi,com.sankuai.meituan,\
-  com.tencent.news_iqiyi,com.tencent.news,134037632320210617,44094811020210617,147250281920210617,com.kuaishou.nebula_fromkuaishou,com.smile.gifmaker_fromkuaishou
+  com.tencent.news_iqiyi,com.tencent.news,134037632320210617,44094811020210617,147250281920210617,com.kuaishou.nebula_fromkuaishou,com.smile.gifmaker_fromkuaishou,\
+  com.sankuai.meituan_bes
 
 btop.package_name=com.taobao.taobao,com.taobao.taobao_btop7,com.taobao.taobao_btop15,com.eg.android.AlipayGphone_btop,com.taobao.taobao_btop7_imei,\
   com.taobao.taobao_btop15_imei,com.taobao.taobao_btop7_oaid,com.taobao.taobao_btop15_oaid,com.taobao.idlefish_btop,com.UCMobile_btop,com.qiyi.video_btop,\
@@ -131,4 +132,4 @@ youku_acquisition.package_name=com.youku.foracquisition_imei,com.youku.foracquis
 
 rtdmp.stop.audience=433,406,405,407,390,395,389,123,238,388,1133,1134,1135
 
-define.package_name.update_date=com.sankuai.meituan_oppo:0,com.sankuai.meituan_bes:0,com.sankuai.meituan_iqiyi:0
\ No newline at end of file
+define.package_name.update_date=com.sankuai.meituan_oppo:365,com.sankuai.meituan_bes:365,com.sankuai.meituan_iqiyi:365
\ No newline at end of file

src/main/scala/mobvista/dmp/datasource/rtdmp/Constant.scala

@@ -87,12 +87,11 @@ object Constant {
     """
       |SELECT device_id, device_type, platform, package_name, update_date
       |   FROM dwh.@table WHERE dt = '@dt' AND business = '@business' @check_package @check_hr
-      |   GROUP BY device_id, device_type, platform, package_name
       |""".stripMargin
 
   val tencent_device_sql =
     """
-      |SELECT device_id, device_type, platform, package_name
+      |SELECT device_id, device_type, platform, package_name, '@update_date' update_date
       |   FROM dwh.@table WHERE dt = '@dt' @check_package @check_hr
       |   GROUP BY device_id, device_type, platform, package_name
       |""".stripMargin

src/main/scala/mobvista/dmp/datasource/rtdmp/RTDmpRequest.scala

@@ -86,6 +86,7 @@ class RTDmpRequest extends CommonSparkJob with Serializable {
           .replace("@dt", date)
           .replace("@check_package", "AND check_package(package_name)")
           .replace("@table", table)
+          .replace("@update_date",update_date)
       } else {
         PropertyUtil.getProperty("config.properties", s"$business.package_name").split(",", -1).foreach(p => {
           package_name_set.add(p)