package mobvista.dmp.datasource.dm

import com.alibaba.fastjson.{JSON, JSONArray, JSONObject}
import mobvista.dmp.common.CommonSparkJob
import mobvista.dmp.datasource.dm.Constant.DmInterestTagV2
import org.apache.commons.cli.Options
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{SaveMode, SparkSession}

import java.net.URI

/**
 * @package: mobvista.dmp.datasource.dm
 * @author: wangjf
 * @date: 2018/12/13
 * @time: 下午6:42
 * @email: jinfeng.wang@mobvista.com
 * @phone: 152-1062-7698
 */
class DmpDeviceInterest extends CommonSparkJob with Serializable {
  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      printOptions(commandLine)
      return 1
    } else {
      printOptions(commandLine)
    }

    val date = commandLine.getOptionValue("date")
    val coalesce = Integer.parseInt(commandLine.getOptionValue("coalesce"))
    val output = commandLine.getOptionValue("output")

    val spark = SparkSession.builder()
      .appName(name = s"DmpDeviceInterest.$date")
      .config("spark.rdd.compress", "true")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.io.compression.codec", "lz4")
      .config("spark.io.compression.lz4.blockSize", "64k")
      .config("spark.broadcast.compress", "true")
      .config("spark.sql.autoBroadcastJoinThreshold", "524288000")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()
    val sc = spark.sparkContext

    FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output), true)
    try {
      /*
      val tag_sql = Constant.dim_app_tag_sql.replace("@date", date)
      val oldMap = sc.broadcast(spark.sql(tag_sql).rdd.map(r => {
        (r.getAs("package_name").toString.toLowerCase, r.getAs("tag").toString)
      }).collectAsMap())
      */

      import spark.implicits._
      val app_two_tags_sql = InterestTagConstant.app_tag_except_sql.replace("@date", date)
      spark.sql(app_two_tags_sql)
        .flatMap(InterestTagConstant.flatInterest)
        .toDF()
        .createOrReplaceTempView("t_app_two_tags")

      spark.udf.register("str2Json", str2Json _)

      val new_tag_sql = InterestTagConstant.tag_sql_v2

      //  import scala.collection.JavaConversions._
      val newMap = sc.broadcast(spark.sql(new_tag_sql).rdd.map(r => {

        var packageName = r.getAs("package_name").toString.toLowerCase()
        if (packageName.matches("^id\\d+$")) {
          packageName = packageName.replace("id", "")
        }
        (packageName, r.getAs("tags").toString)
      }).collectAsMap())

      val rdd = spark.sql(Constant.install_sql.replace("@date", date)).rdd
        .map(row => {
          val device_id = row.getAs("device_id").toString
          val device_type = row.getAs("device_type").toString
          val platform = if (StringUtils.isNotBlank(row.getAs("platform").toString) && "ios".equals(row.getAs("platform").toString)) {
            "ios"
          } else {
            "android"
          }
          val ext_data = row.getAs[String]("ext_data")
          val update_date = row.getAs[String]("update_date")
          import scala.collection.JavaConverters._
          val install_list = JSON.parse(row.getAs("install_list").toString).asInstanceOf[java.util.Map[String, String]].asScala
          val jsonArray = new JSONArray()
          install_list.foreach(install => {
            val json = new JSONObject()
            var package_name = install._1
            val install_date = install._2
            if (package_name.matches("^id\\d+$")) {
              package_name = package_name.replace("id", "")
            }
            json.put("package_name", package_name)
            json.put("date", install_date)
            if (newMap.value.keySet.contains(package_name.toLowerCase)) {
              val newTags = newMap.value(package_name.toLowerCase)
              json.put("tag_new", JSON.parseArray(newTags))
            }
            if (json.containsKey("tag_new") && !package_name.toLowerCase.equals("0000000000") && !package_name.equals("com.nonepkg.nonepkg")) {
              jsonArray.add(json)
            }
          })
          if (install_list.size > 0) {
            DmInterestTagV2(device_id = device_id, device_type = device_type, platform = platform, install = install_list.asJava.toString,
              tags = jsonArray.toString, ext_data = ext_data, update_date = update_date)
          } else {
            null
          }
        })

      rdd.filter(dm => {
        dm != null
      }).repartition(coalesce).toDF()
        .write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "snappy")
        .orc(output)

    } finally {
      if (spark != null) {
        sc.stop()
        spark.stop()
      }
    }
    0
  }

  def str2Json(tag_id: String, first_tag: String, second_tag: String): String = {
    val jsonObject = new JSONObject()
    jsonObject.put("id", tag_id)
    jsonObject.put("1", first_tag)
    if (StringUtils.isNotBlank(second_tag)) {
      jsonObject.put("2", second_tag)
    }
    jsonObject.toString
  }

  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("date", true, "[must] date")
    options.addOption("coalesce", true, "[must] coalesce")
    options.addOption("output", true, "[must] output")
    options
  }
}

object DmpDeviceInterest {
  def main(args: Array[String]): Unit = {
    new DmpDeviceInterest().run(args)
  }
}