update InstallList

src/main/scala/mobvista/dmp/common/CommonInstallListOrc.scala

@@ -141,11 +141,6 @@ abstract class CommonInstallListOrc extends CommonSparkJob with Serializable {
               val old_ext_data_json = JSON.parseObject(installListDate(1))
               val region_list = if (old_ext_data_json.containsKey("region")) {
                 JSON.parseArray(old_ext_data_json.getString("region"), classOf[String])
-                  /*
-                  .filter(r => {
-                    MobvistaConstant.regionSet.contains(r)
-                  }).asJava
-                  */
               } else {
                 new util.ArrayList[String]()
               }
@@ -217,11 +212,6 @@ abstract class CommonInstallListOrc extends CommonSparkJob with Serializable {
               }
               val region_list = if (old_ext_data_json.containsKey("region")) {
                 JSON.parseArray(old_ext_data_json.getString("region"), classOf[String])
-                /*
-                .filter(r => {
-                  MobvistaConstant.regionSet.contains(r)
-                }).asJava
-                */
               } else {
                 new util.ArrayList[String]()
               }
@@ -231,7 +221,6 @@ abstract class CommonInstallListOrc extends CommonSparkJob with Serializable {
               }
               ext_data = old_ext_data_json.toJSONString
 
-              //  ext_data = dailyOpt.get._2
               country = if (StringUtils.isNotBlank(dailyOpt.get._3)) {
                 dailyOpt.get._3
               } else {

src/main/scala/mobvista/dmp/common/InstallListDailyV2.scala

@@ -64,7 +64,6 @@ class InstallListDailyV2 extends CommonSparkJob with Serializable {
       activeDev.createOrReplaceTempView("active_dev")
 
       spark.udf.register("merge", merge _)
-      //  spark.udf.register("filterInstall", filterInstall _)
       spark.udf.register("udf_mergeExtData", mobvista.dmp.datasource.dm.Constant.mergeExtData _)
       sql = sql.replace("@date", date).replace("@before_date", before_date)
       val df = spark.sql(sql)
@@ -84,8 +83,8 @@ class InstallListDailyV2 extends CommonSparkJob with Serializable {
   }
 
   /**
-    * 先筛选近1天的活跃设备，再进行 LEFT SEMI JOIN 获取近1天设备的安装列表数据
-    */
+   * 先筛选近14天的活跃设备，再进行 LEFT SEMI JOIN 获取近14天设备的安装列表数据
+   */
   val active_sql: String =
     """
       |SELECT device_id
@@ -126,8 +125,8 @@ class InstallListDailyV2 extends CommonSparkJob with Serializable {
     val installJSONObject = new JSONObject
     installList.iterator.foreach(install => {
       val installMap = MobvistaConstant.String2JSONObject(install).asInstanceOf[java.util.Map[String, String]].asScala
-      installMap.retain((k, v) => !installJSONObject.containsKey(k) || installJSONObject.getString(k).compareTo(v) < 0).foreach(kv => {
-        //  && !k.equalsIgnoreCase("0000000000") && !k.equalsIgnoreCase("com.nonepkg.nonepkg")
+      installMap.retain((k, v) => (!installJSONObject.containsKey(k) || installJSONObject.getString(k).compareTo(v) < 0) &&
+        !k.equalsIgnoreCase("0000000000") && !k.equalsIgnoreCase("com.nonepkg.nonepkg")).foreach(kv => {
         installJSONObject.put(kv._1, kv._2)
       })
     })

src/main/scala/mobvista/dmp/datasource/dm/Constant.scala

@@ -776,7 +776,6 @@ object Constant {
       mergeExtData.put("region", regionSet.asJava)
     }
     val strategySet = new mutable.HashSet[String]()
-    import scala.collection.JavaConversions._
     if (data_1.containsKey("strategy")) {
       data_1.getJSONArray("strategy").foreach(v => {
         strategySet.add(v.toString)