package mobvista.prd.main

import java.util

import com.google.gson.JsonElement
import mobvista.dmp.common.CommonSparkJob
import mobvista.prd.datasource.util.GsonUtil
import org.apache.commons.cli.Options
import org.apache.commons.lang.StringUtils
import org.apache.spark.sql.SparkSession
import java.math.BigDecimal
import java.net.URI

import org.apache.hadoop.fs.{FileSystem, Path}

/**
  * dsp、m系统统计
  */
class GenderAgeCnt extends CommonSparkJob with Serializable {
  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      return 1
    } else {
      printOptions(commandLine)
    }

    val output = commandLine.getOptionValue("output")
    val agePath = commandLine.getOptionValue("agePath")
    val parallelism = commandLine.getOptionValue("parallelism")
    val genderPath = commandLine.getOptionValue("genderPath")
    val dspDailyPath = commandLine.getOptionValue("dspDailyPath")
    val adnDailyPath = commandLine.getOptionValue("adnDailyPath")

    val spark = SparkSession.builder()
      .appName("GenderAgeCnt")
      .config("spark.default.parallelism", parallelism)
      .config("spark.rdd.compress", "true")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .getOrCreate()
    import spark.implicits._
    val sc = spark.sparkContext

    try {
      val dspDailyRDD = sc.textFile(dspDailyPath)
          .map(splitFun(_))
          .map(array => DailyVO(array(0), array(1), "dsp"))
      val adnDailyRDD = sc.textFile(adnDailyPath)
        .map(splitFun(_))
        .map(array => DailyVO(array(0), array(1), "adn"))

      dspDailyRDD.union(adnDailyRDD)
        .toDF()
        .persist()
        .createOrReplaceTempView("t_daily_data")

      sc.textFile(agePath)
        .map(splitFun(_))
        .map(array => AgeVO(array(0), array(1), getAgeRange(array(3))))
        .toDF()
        .createOrReplaceTempView("dm_device_age")

      sc.textFile(genderPath)
        .map(splitFun(_))
        .map(array => GenderVO(array(0), array(1), getGender(array(4))))
        .toDF()
        .createOrReplaceTempView("dm_device_gender")

      var hql =
        """
          |select '' as dimension, t.business, count(1) as cnt, 'dau'
          |from (
          |  select t.device_id, t.device_type, t.business
          |  from t_daily_data t
          |  group by t.device_id, t.device_type, t.business
          |) t
          |group by t.business
          |
        """.stripMargin
      val dauDF = spark.sql(hql)


      val ageSql =
        """
          |
          |select t.age, t.business, count(1) as cnt, 'age'
          |from (
          |select a.device_id,a.device_type, nvl(b.age, 'unknow')as age, a.business
          |from t_daily_data a
          |left outer join dm_device_age b
          |  on a.device_id = b.device_id and a.device_type=b.device_type
          |group by a.device_id,a.device_type, nvl(b.age, 'unknow'), a.business
          |) t
          |group by t.age, t.business
          |
        """.stripMargin
      val ageDF = spark.sql(ageSql)


      var genderSql =
        """
          |
          |select t.gender, t.business, count(1) as cnt, 'gender'
          |from (
          |select a.device_id,a.device_type, nvl(b.gender, 'unknow')as gender, a.business
          |from t_daily_data a
          |left outer join dm_device_gender b
          |  on a.device_id = b.device_id and a.device_type=b.device_type
          |group by a.device_id,a.device_type, nvl(b.gender, 'unknow'), a.business
          |) t
          |group by t.gender, t.business
          |
        """.stripMargin
      val genderDF = spark.sql(genderSql)
        .cache()

      val result = dauDF.union(ageDF).union(genderDF)
        .map(_.mkString("\t"))
        .rdd
        .collect()

      // 进行列转行操作，最终数据title
      // Data source、	DMP DAU、	Male、Female、	Gender unknown、	Age 0-18、	Age 19-25、Age 26-45、	Age 46-60、Age above60	、Age unknown
      val map = new util.HashMap[String, Integer]()
      val totalMap = result.map(line => {
          val splits = line.split("\t", -1)
          val dname = splits(0)
          val business = splits(1)
          val cvalue = splits(2)
          val dtype = splits(3)
          map.put(s"${business}|${dtype}|${dname}", cvalue.toInt)
          (s"${business}|${dtype}", cvalue.toInt)
        })
        .groupBy(_._1)
        .mapValues(_.map(_._2).sum)  /// 计算每个类别总设备量

      val dspDAU = map.get("dsp|dau|")
      val dspGenderM = map.get("dsp|gender|m")
      val dspGenderF = map.get("dsp|gender|f")
      val dspGenderU = map.get("dsp|gender|unknow")
      val dspGenderTotal = totalMap.get("dsp|gender").get
      val dspAge17 = map.get("dsp|age|0-17")
      val dspAge24 = map.get("dsp|age|18-24")
      val dspAge44 = map.get("dsp|age|25-44")
      val dspAge59 = map.get("dsp|age|45-59")
      val dspAge60: Int = if (map.get("dsp|age|60+") != null) map.get("dsp|age|60+") else 0
      val dspAgeTotal = totalMap.get("dsp|age").get

      val builder = new StringBuilder
      builder
        .append("dsp").append("\t")
        .append(dspDAU).append("\t")
        .append(dspGenderM).append("(").append(calcPercen(dspGenderM, dspGenderTotal)).append("%)\t")
        .append(dspGenderF).append("(").append(calcPercen(dspGenderF, dspGenderTotal)).append("%)\t")
        .append(dspGenderU).append("(").append(calcPercen(dspGenderU, dspGenderTotal)).append("%)\t")
        .append(dspAge17).append("(").append(calcPercen(dspAge17, dspAgeTotal)).append("%)\t")
        .append(dspAge24).append("(").append(calcPercen(dspAge24, dspAgeTotal)).append("%)\t")
        .append(dspAge44).append("(").append(calcPercen(dspAge44, dspAgeTotal)).append("%)\t")
        .append(dspAge59).append("(").append(calcPercen(dspAge59, dspAgeTotal)).append("%)\t")
        .append(dspAge60).append("(").append(calcPercen(dspAge60, dspAgeTotal)).append("%)\t")

      builder.append("\n")

      val adnDAU = map.get("adn|dau|")
      val adnGenderM = map.get("adn|gender|m")
      val adnGenderF = map.get("adn|gender|f")
      val adnGenderU = map.get("adn|gender|unknow")
      val adnGenderTotal = totalMap.get("adn|gender").get
      val adnAge17 = map.get("adn|age|0-17")
      val adnAge24 = map.get("adn|age|18-24")
      val adnAge44 = map.get("adn|age|25-44")
      val adnAge59 = map.get("adn|age|45-59")
      val adnAge60: Int = if (map.get("adn|age|60+") != null) map.get("adn|age|60+") else 0
      val adnAgeTotal = totalMap.get("adn|age").get

      builder
        .append("adn").append("\t")
        .append(adnDAU).append("\t")
        .append(adnGenderM).append("(").append(calcPercen(adnGenderM, adnGenderTotal)).append("%)\t")
        .append(adnGenderF).append("(").append(calcPercen(adnGenderF, adnGenderTotal)).append("%)\t")
        .append(adnGenderU).append("(").append(calcPercen(adnGenderU, adnGenderTotal)).append("%)\t")
        .append(adnAge17).append("(").append(calcPercen(adnAge17, adnAgeTotal)).append("%)\t")
        .append(adnAge24).append("(").append(calcPercen(adnAge24, adnAgeTotal)).append("%)\t")
        .append(adnAge44).append("(").append(calcPercen(adnAge44, adnAgeTotal)).append("%)\t")
        .append(adnAge59).append("(").append(calcPercen(adnAge59, adnAgeTotal)).append("%)\t")
        .append(adnAge60).append("(").append(calcPercen(adnAge60, adnAgeTotal)).append("%)\t")

      FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output),true)


      sc.parallelize(Array(builder.toString()))
        .coalesce(1, true)
        .saveAsTextFile(output)
    } finally {
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }

  def calcPercen(i: Int, total: Int): Double = {
    val a = new BigDecimal(i)
    val b = new BigDecimal(total)
    a.divide(b, 4, BigDecimal.ROUND_HALF_UP).multiply(new BigDecimal(100)).doubleValue()
  }

  def getGender(str: String): String = {
    if (StringUtils.isEmpty(str) || "o".equals(str)) {
      return "unknow"
    } else {
      return str
    }
  }

  def getAgeRange(str: String): String = {
    val json = GsonUtil.String2JsonObject(str)
    val ageJson = json.get("age_and_proportion").getAsJsonObject
    val itr = ageJson.entrySet.iterator

    var max = 0.0
    var age: String = null
    var entry: util.Map.Entry[String, JsonElement] = null
    while ({itr.hasNext}) {
      entry = itr.next
      val temp = entry.getValue.getAsDouble
      if (temp > max) {
        max = temp
        age = entry.getKey
      }
    }
    if (max < 0.5d) age = "unknow"
    return age
  }

  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("dspDailyPath", true, "[must]")
    options.addOption("adnDailyPath", true, "[must]")
    options.addOption("agePath", true, "[must]")
    options.addOption("genderPath", true, "[must]")
    options.addOption("output", true, "[must]")
    options.addOption("parallelism", true, "parallelism")
    options
  }
}

object GenderAgeCnt {
  def main(args: Array[String]): Unit = {
    new GenderAgeCnt().run(args)
  }
}

case class DailyVO(device_id: String, device_type: String, business: String)
case class AgeVO(device_id: String, device_type: String, age: String)
case class GenderVO(device_id: String, device_type: String, gender: String)
