init id_mapping

azkaban/id_mapping/id_mapping.sh

@@ -24,8 +24,8 @@ spark-submit --class mobvista.dmp.datasource.id_mapping.IDMappingGraphx \
   --name "IDMappingGraphx.${LOG_TIME}.${country}.${platform}" \
   --conf spark.yarn.executor.memoryOverhead=2048 \
   --conf spark.network.timeout=720s \
-  --conf spark.sql.shuffle.partitions=5000 \
-  --conf spark.default.parallelism=5000 \
+  --conf spark.sql.shuffle.partitions=10000 \
+  --conf spark.default.parallelism=10000 \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
   --master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 8g --executor-cores 5 --num-executors 100 \
   ../${JAR} -date ${LOG_TIME} -country ${country} -platform ${platform} -output ${OUTPUT_PATH} -coalesce 500

src/main/scala/mobvista/dmp/datasource/id_mapping/Constant.scala

@@ -267,9 +267,6 @@ object Constant {
   val iosIDScoreMap: Map[String, Double] = Map("idfa" -> 1000, "sysid" -> 1, "xwho" -> 0.9, "user_id" -> 0.9, "bmosv_osv_upt" -> 0.9,
     "idfv_bundle" -> 0.8, "bmosv_upt" -> 0.7, "bmosv_ipua_bundle" -> 0.6, "bkupid" -> 0.3)
 
-  val androidCNIDScoreMap: Map[String, Double] = Map("imei" -> 1000, "oaid" -> 1000, "gaid" -> 1000, "sysid" -> 1, "xwho" -> 0.9, "user_id" -> 0.9,
-    "android_pkg" -> 0.8, "bmosv_upt" -> 0.7, "bmosv_ipua_pkg" -> 0.6, "bkupid" -> 0.3)
-
   val androidIDScoreMap: Map[String, Double] = Map("gaid" -> 1000, "imei" -> 1000, "oaid" -> 1000, "sysid" -> 1, "xwho" -> 0.9, "user_id" -> 0.9,
     "android_pkg" -> 0.8, "bmosv_upt" -> 0.7, "bmosv_ipua_pkg" -> 0.6, "bkupid" -> 0.3)
 

src/main/scala/mobvista/dmp/datasource/id_mapping/IDMappingGraphx.scala

@@ -42,12 +42,13 @@ class IDMappingGraphx extends CommonSparkJob with Serializable {
     val platform = commandLine.getOptionValue("platform")
     val date = commandLine.getOptionValue("date")
     val output = commandLine.getOptionValue("output")
+    val result_output = commandLine.getOptionValue("result_output")
     val coalesce = Integer.parseInt(commandLine.getOptionValue("coalesce"))
 
     val spark = MobvistaConstant.createSparkSession(s"IDMappingGraphx.$date.$country.$platform")
 
     try {
-      oldAndTodayIdMapping(country, platform, date, spark, output, output, coalesce)
+      oldAndTodayIdMapping(country, platform, date, spark, output, result_output, coalesce)
     } finally {
       if (spark != null) {
         spark.stop()