fix craw package tag bug

src/main/scala/mobvista/dmp/datasource/apptag/CrawPkgsSpark.scala

 package mobvista.dmp.datasource.apptag
 
-import java.net.URI
-
 import mobvista.dmp.common.{CommonSparkJob, MobvistaConstant}
 import org.apache.commons.cli.Options
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.sql.SparkSession
 
+import java.net.URI
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
@@ -74,14 +73,14 @@ class CrawPkgsSpark extends CommonSparkJob with Serializable {
         s"""
            |SELECT b.package_name, getPlatform(b.package_name) platform
            |  FROM
-           |    (SELECT package_name
-           |      FROM dwh.dmp_install_list LATERAL VIEW explode(getPkgs(install_list)) dmp_table AS package_name WHERE dt = '$yesday' AND business = 'day'
+           |    (SELECT package_name, COUNT(1) counts
+           |      FROM dwh.dmp_install_list LATERAL VIEW explode(getPkgs(install_list)) dmp_table AS package_name WHERE dt = '$yesday' AND business = '14days'
            |      GROUP BY package_name
            |    ) b
            |    LEFT JOIN
            |    (SELECT package_name FROM dev.dm_package_black_list WHERE dt = '$yesday') a
-           |  ON a.package_name = b.package_name
-           |  WHERE a.package_name IS NULL
+           |  ON LOWER(a.package_name) = LOWER(b.package_name)
+           |  WHERE a.package_name IS NULL ORDER BY counts DESC LIMIT 1000000
            |""".stripMargin
 
       spark.sql(sql).rdd

src/main/scala/mobvista/dmp/main/PackageBlackList.scala

 package mobvista.dmp.main
 
-import java.net.URI
-
 import mobvista.dmp.common.CommonSparkJob
 import org.apache.commons.cli.Options
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.spark.sql.{SaveMode, SparkSession}
 
+import java.net.URI
+
 /**
-  * 将抓包程序抓不到的包，加入到黑名单中
-  */
+ * 将抓包程序抓不到的包，加入到黑名单中
+ */
 class PackageBlackList extends CommonSparkJob with Serializable {
   override protected def run(args: Array[String]): Int = {
     val commandLine = commParser.parse(options, args)
@@ -47,8 +47,16 @@ class PackageBlackList extends CommonSparkJob with Serializable {
       val adrDailyRDD = sc.textFile(adrDailyPath)
         .map(splitFun(_)(0))
 
-      iosDailyRDD.union(adrDailyRDD)
-        .map(PackageVO(_))
+      val packageRDD = spark.sql(
+        """
+          |SELECT LOWER(package_name) package_name FROM dwh.dim_package_tag GROUP BY LOWER(package_name)
+          |""".stripMargin)
+        .rdd.map(r => {
+        r.getAs[String]("package_name")
+      })
+
+      iosDailyRDD.union(adrDailyRDD).union(packageRDD)
+        .map(PackageVO)
         .toDF()
         .createOrReplaceTempView("t_package_daily")