package mobvista.dmp.datasource.rtdmp

import com.alibaba.fastjson.JSON
import mobvista.dmp.common.{CommonSparkJob, MobvistaConstant}
import mobvista.dmp.util.DateUtil
import mobvista.dmp.utils.clickhouse.ClickHouseConnectionFactory
import mobvista.dmp.utils.clickhouse.ClickHouseSparkExt._
import org.apache.commons.cli.{BasicParser, Options}
import ru.yandex.clickhouse.ClickHouseDataSource

import scala.collection.JavaConversions._
import scala.collection.mutable


/**
 * @package: mobvista.dmp.datasource.rtdmp
 * @author: wangjf
 * @date: 2020/8/17
 * @time: 10:57 上午
 * @email: jinfeng.wang@mobvista.com
 * @phone: 152-1062-7698
 */
class RTDmpMergeCK extends CommonSparkJob with Serializable {
  var expire_time = ""

  def commandOptions(): Options = {
    val options = new Options()
    options.addOption("date_time", true, "date_time")
    options.addOption("host", true, "host")
    options.addOption("cluster", true, "cluster")
    options.addOption("database", true, "database")
    options.addOption("table", true, "table")
    options
  }

  override protected def run(args: Array[String]): Int = {
    val parser = new BasicParser()
    val options = commandOptions()
    val commandLine = parser.parse(options, args)
    val date_time = commandLine.getOptionValue("date_time")
    val cluster = commandLine.getOptionValue("cluster")
    val host = commandLine.getOptionValue("host")
    val database = commandLine.getOptionValue("database")
    val table = commandLine.getOptionValue("table")

    val spark = MobvistaConstant.createSparkSession(s"RTDmpMergeCK.$date_time")

    try {
      expire_time = DateUtil.getDayByString(date_time, "yyyyMMddHH", -1)

      spark.udf.register("process", process _)

      val df = spark.sql(sql.replace("@dt", date_time))

      implicit val clickhouseDataSource: ClickHouseDataSource = ClickHouseConnectionFactory.get(host)

      val clusterName = Some(cluster): Option[String]

      val date = date_time.substring(0, 8)

      val dt = MobvistaConstant.sdf1.format(MobvistaConstant.sdf2.parse(date))
      val hour = date_time.substring(8, 10)

      val tdf = spark.emptyDataFrame
      //  drop curr partition
      tdf.dropPartition(database, table, s"($date,'$hour')", clusterName)

      Thread.sleep(120000)

      df.saveToClickHouse(database, table, Seq(dt, hour), Seq("dt", "hour"), clusterName, batchSize = 200000)

    } finally {
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }

  val sql =
    """
      |SELECT devid, process(audience_map) audience_id
      |  FROM dwh.audience_merge
      |  WHERE dt = '@dt'
      |""".stripMargin

  def process(audience_map: String): mutable.WrappedArray[Int] = {
    val set = new mutable.HashSet[Int]()
    val audienceMap = JSON.parseObject(audience_map).asInstanceOf[java.util.Map[String, String]]
    audienceMap.retain((_, v) => v.compareTo(expire_time) > 0).keys.foreach(k => {
      set.add(Integer.parseInt(k))
    })
    mutable.WrappedArray.make(set.toArray)
  }
}

object RTDmpMergeCK {
  def main(args: Array[String]): Unit = {
    new RTDmpMergeCK().run(args)
  }
}