package mobvista.dmp.datasource.apptag

import java.net.URI

import mobvista.dmp.common.CommonSparkJob
import mobvista.prd.datasource.util.GsonUtil
import org.apache.commons.cli.{BasicParser, Options}
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.{Row, SaveMode, SparkSession}

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer

/**
  * @package: mobvista.dmp.datasource.apptag
  * @author: wangjf
  * @date: 2019-11-06
  * @time: 11:06
  * @email: jinfeng.wang@mobvista.com
  * @phone: 152-1062-7698
  */
class MergeAppTagID extends CommonSparkJob with java.io.Serializable {
  def commandOptions(): Options = {
    val options = new Options()
    options.addOption("output", true, "output")
    options.addOption("date", true, "date")
    options
  }

  var bMap: Broadcast[scala.collection.Map[String, String]] = null
  var packageMap: Broadcast[scala.collection.Map[String, Int]] = null

  override protected def run(args: Array[String]): Int = {
    val parser = new BasicParser()
    val options = commandOptions()
    val commandLine = parser.parse(options, args)
    val output = commandLine.getOptionValue("output")
    val date = commandLine.getOptionValue("date")

    val spark = SparkSession
      .builder()
      .appName(s"MergeAppTagID")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "snappy")
      .config("spark.sql.autoBroadcastJoinThreshold", "268435456")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()
    try {

      val sc = spark.sparkContext
      FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output), true)

      val code_sql = Constant.old_new_sql
      bMap = sc.broadcast(spark.sql(code_sql).rdd.map(r => {
        (r.getAs("tag_code").toString, r.getAs("new_second_id").toString)
      }).collectAsMap())

      var package_sql =
        """
          |SHOW PARTITIONS dwh.package_mapping
        """.stripMargin
      val partDF = spark.sql(package_sql)
      val package_dt = partDF.orderBy(partDF("partition").desc).first.getString(0).split("=")(1)
      package_sql =
        s"""
           |SELECT id, package_name FROM dwh.package_mapping WHERE dt = '${package_dt}'
        """.stripMargin
      packageMap = spark.sparkContext.broadcast(spark.sql(package_sql).rdd.map(r => {
        var packageName = r.getAs("package_name").toString.toLowerCase
        if (packageName.matches("^id[0-9]+$")) {
          packageName = packageName.replace("id", "")
        }
        (packageName, Integer.parseInt(r.getAs("id").toString))
      }).collectAsMap())

      val year = date.substring(0, 4)
      val month = date.substring(4, 6)
      val day = date.substring(6, 8)

      var sql =
        s"""
           |SELECT
           |   package_name,
           |   CASE WHEN platform = 'ios' THEN 'ios' ELSE 'android' END AS platform,
           |   tag
           |   FROM dwh.dim_app_tag
           |   WHERE year = '$year' and month = '$month' and day = '$day'
        """.stripMargin

      import spark.implicits._

      spark.sql(sql)
        .rdd
        .flatMap(flatInterest)
        .toDF
        .createOrReplaceTempView("store_tag")

      spark.udf.register("getPackageId", getPackageId _)
      spark.udf.register("getId", getId _)

      sql =
        s"""
           |SELECT
           |   getPackageId(lower(package_name)) package_id,
           |   getId(CONCAT(tag_type,'-',first_tag,'-',second_tag)) interest
           |   FROM dwh.dim_package_tags
        """.stripMargin

      spark.sql(sql).createOrReplaceTempView("manual_tag")

      sql =
        """
          |SELECT package_id,COLLECT_SET(interest) interest
          |   FROM (
          |     SELECT package_id,interest FROM store_tag
          |     UNION ALL
          |     SELECT package_id,interest FROM manual_tag
          |     ) t
          |   WHERE package_id > 0 AND interest IS NOT NULL AND interest != ''
          |   GROUP BY package_id
        """.stripMargin

      spark.sql(sql).write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .orc(output)

    } finally {
      if (spark != null) {
        spark.stop()
      }
    }

    0
  }

  def flatInterest(row: Row): Array[PackageTag] = {
    val buffer = new ArrayBuffer[PackageTag]()
    val packageName = row.getString(0)
    val pacakgeId = getPackageId(packageName)
    GsonUtil.String2JsonArray(row.getString(2))
      .foreach(element => {
        val obj = element.getAsJsonObject
        val firstTag = obj.get("1").getAsString
        val secondTag = if (obj.get("2") != null) obj.get("2").getAsString else ""
        buffer += PackageTag(pacakgeId, getId("category" + "-" + firstTag + "-" + secondTag))
      })
    buffer.toArray
  }

  //  process inter_id
  def getId(tag_code: String): String = {
    val id = if (bMap.value.keySet.contains(tag_code.toUpperCase)) {
      bMap.value(tag_code.toUpperCase)
    } else {
      bMap.value.getOrElse(tag_code.toUpperCase + "OTHER", "")
    }
    id
  }

  //  process package_id
  def getPackageId(package_name: String): Int = {
    var packageId = 0
    var pkgName = package_name
    if (pkgName.matches("^id[0-9]+$")) {
      pkgName = pkgName.replace("id", "")
    }
    if (pkgName.nonEmpty && StringUtils.isNotBlank(pkgName) && packageMap.value.contains(pkgName.toLowerCase)) {
      packageId = packageMap.value(pkgName.toLowerCase)
    }
    packageId
  }
}

object MergeAppTagID {
  def main(args: Array[String]): Unit = {
    new MergeAppTagID().run(args)
  }
}