package mobvista.dmp.datasource.rtdmp

import java.net.URI
import java.util

import com.alibaba.fastjson.{JSONArray, JSONObject}
import com.datastax.spark.connector._
import com.datastax.spark.connector.cql.CassandraConnector
import mobvista.dmp.common.{CommonSparkJob, MobvistaConstant}
import mobvista.dmp.util.{DateUtil, MD5Util, PropertyUtil}
import org.apache.commons.cli.{BasicParser, Options}
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{Row, SaveMode, SparkSession}
import org.apache.spark.storage.StorageLevel

import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer

/**
 * @package: mobvista.dmp.datasource.rtdmp
 * @author: wangjf
 * @date: 2020/7/13
 * @time: 11:25 上午
 * @email: jinfeng.wang@mobvista.com
 * @phone: 152-1062-7698
 */
class RTDmpMain extends CommonSparkJob with Serializable {

  def commandOptions(): Options = {
    val options = new Options()
    options.addOption("time", true, "time")
    options.addOption("output", true, "output")
    options.addOption("coalesce", true, "coalesce")
    options.addOption("partition", true, "partition")
    //  options.addOption("update_time_start", true, "update_time_start")
    //  options.addOption("update_time_end", true, "update_time_end")
    options.addOption("data_utime", true, "data_utime")
    options
  }

  override protected def run(args: Array[String]): Int = {

    val parser = new BasicParser()
    val options = commandOptions()
    val commandLine = parser.parse(options, args)
    val time = commandLine.getOptionValue("time")
    val output = commandLine.getOptionValue("output")
    val coalesce = commandLine.getOptionValue("coalesce")
    val partition = commandLine.getOptionValue("partition")
    //  var update_time_start = commandLine.getOptionValue("update_time_start")
    //  var update_time_end = commandLine.getOptionValue("update_time_end")
    val data_utime = commandLine.getOptionValue("data_utime")

    val spark: SparkSession = SparkSession.builder()
      .appName(s"RTDmpMain.$time")
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "snappy")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .config("spark.cassandra.connection.host", PropertyUtil.getProperty("ip.properties", "aws_host"))
      .config("spark.cassandra.connection.port", "9042")
      .config("spark.cassandra.connection.factory", s"mobvista.dmp.utils.cassandra.AWSFactory")
      .config("spark.cassandra.connection.connections_per_executor_max", "512")
      .config("spark.cassandra.output.concurrent.writes", "32")
      .config("spark.cassandra.concurrent.reads", "1024")
      .config("spark.cassandra.output.batch.grouping.buffer.size", "1024")
      .config("spark.cassandra.connection.keep_alive_ms", "600000")
      .getOrCreate()
    //  .config("spark.cassandra.input.consistency.level", "LOCAL_QUORUM")
    //  .config("spark.cassandra.output.consistency.level", "LOCAL_QUORUM")
    //  .config("spark.cassandra.connection.connections_per_executor_max", "8")

    val sc = spark.sparkContext
    try {
      var mergeRDD = sc.emptyRDD[(String, (Int, String))]

      val expire_time = DateUtil.getDayByString(time, "yyyy-MM-dd HH", -7)

      //  默认计算上个小时的数据
      val update_time_start = DateUtil.format(time + ":00:00", "yyyy-MM-dd HH:mm:ss")
      val update_time_end = DateUtil.format(time + ":59:59", "yyyy-MM-dd HH:mm:ss")

      val audience_date_utime_start = DateUtil.parse(data_utime, "yyyy-MM-dd HH").getTime / 1000 - 28800
      val audience_date_utime_end = DateUtil.parse(data_utime, "yyyy-MM-dd HH").getTime / 1000 - 25200

      val map: util.Map[Integer, (JSONArray, Integer, Integer, JSONObject)] =
        ServerUtil.request(update_time_start, update_time_end, audience_date_utime_start, audience_date_utime_end, 0, 0, 2)

      println(s"map -->> $map")

      map.foreach(t => {
        val audienceId = Integer2int(t._1)
        val audienceOp = t._2._2
        val dmap = new mutable.HashMap[String, String]()
        t._2._1.foreach(json => {
          val jsonObject = json.asInstanceOf[JSONObject]
          if (jsonObject.containsKey("s3_path") && StringUtils.isNotBlank(jsonObject.getString("s3_path"))) {
            //  (s3_path, update_date)
            dmap.put(jsonObject.getString("s3_path"), jsonObject.getString("update_time"))
          }
        })

        //  判断所有 s3_path 对应的 update_date 中是否有 当前 update_date，进行有效过滤
        //  if (dmap.values.contains(time)) {
        /**
         * audienceOp == 0 and dmap.size >= 2 即做差集计算，表示在上个分区出现，这个分区不出现的设备对应的安装包置为 -1 * audienceId，
         * 用于下游删除；否则与前一个人群包进行合并操作
         */
        val updateRDD = if (audienceOp == 1 && dmap.size >= 2) {
          val list = dmap.toList.sortWith(_._2 > _._2).take(1) //  按 update_date 进行降序排序，提取最前面的两个人群包
          val newAudience = sc.textFile(list.get(0)._1).map(r => { //  First 为最新的人群包
            val device_id =
              if (r.matches(MobvistaConstant.md5Ptn)) {
                r
              } else {
                MD5Util.getMD5Str(r)
              }
            (device_id, (audienceId, list.get(0)._2))
          })

          /*
          val oldAudience = sc.textFile(list.get(1)._1).map(r => { //  Second 为旧的人群包，表示上一版本人群包
            val device_id =
              if (r.matches(MobvistaConstant.md5Ptn)) {
                r
              } else {
                MD5Util.getMD5Str(r)
              }
            (device_id, (audienceId, list.get(1)._2))
          })
          oldAudience.subtractByKey(newAudience).map(t => {
            //  对差集 audienceId * -1，用于下游计算删除 audienceId
            val device_id =
              if (t._1.matches(MobvistaConstant.md5Ptn)) {
                t._1
              } else {
                MD5Util.getMD5Str(t._1)
              }
            (device_id, ((-1) * audienceId, t._2._2))
            //  (devId, ((-1) * audienceId, update_date))
          }).union(newAudience) //  与最新的人群包进行合并操作
          */
          newAudience
        } else {
          val audData = dmap.toList.sortWith(_._2 > _._2)
          if (audData.nonEmpty) {
            sc.textFile(audData.get(0)._1).map(r => { //  取出最新的人群包
              val device_id =
                if (r.matches(MobvistaConstant.md5Ptn)) {
                  r
                } else {
                  MD5Util.getMD5Str(r)
                }
              (device_id, (audienceId, audData.get(0)._2))
            })
          } else { //  如果没有，则创建 空RDD
            sc.emptyRDD[(String, (Int, String))]
          }
        }
        //  所有人群包进行合并操作
        mergeRDD = mergeRDD.union(updateRDD)
        //  }
      })

      val keyspace = "rtdmp"
      val tableName = "audience_info"
      val columns = SomeColumns("devid", "audience_data", "update_time")
      val cassandraConnector = CassandraConnector(sc.getConf)

      val set = ServerUtil.request(update_time_start, update_time_end, audience_date_utime_start, audience_date_utime_end, 0, 0, 4)
        .retain((_, v) => v._2 == 1)
        .keySet

      println("audienceIds -->> " + set.mkString(","))
      val df = mergeRDD.groupByKey().map(r => {
        val devId = r._1
        val jsonObject = new JSONObject()
        //  生成 audienceId -> update_date JSONObject
        r._2.foreach(t => {
          jsonObject.put(t._1.toString, t._2)
        })
        Row(devId, jsonObject.toJSONString, time)
      }).repartition(partition.toInt)
        .mapPartitions(it => cassandraConnector.withSessionDo(session => {
          Logic.getResultFeature(session, it)
        }))
        .mapPartitions(new CustomIteratorAudienceInfo(_, time, expire_time, set))

      //  .mapPartitions(Logic.parseAudienceInfo(_, expire_time))

      df.persist(StorageLevel.MEMORY_AND_DISK_SER)

      df.saveToCassandra(keyspace, tableName, columns)

      val audienceSum = df.map(r => {
        val array = new ArrayBuffer[(Int, Int)]()
        MobvistaConstant.String2JSONObject(r.audience_data).keySet().foreach(k => {
          if (map.keySet().contains(Integer.parseInt(k))) {
            array.add((Integer.parseInt(k), 1))
          }
        })
        array.iterator
      }).flatMap(l => l)
        .countByKey()

      val audience_output = output + "/audience"
      FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(audience_output), true)

      sc.parallelize(audienceSum.toList).coalesce(1).saveAsTextFile(audience_output)

      val data_output = output + "/data"
      FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(data_output), true)

      import spark.implicits._
      df.mapPartitions(Logic.writeResult(cassandraConnector, _))
        .repartition(coalesce.toInt)
        .toDF
        .write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .orc(data_output)

      //  .mapPartitions(Logic.writeResult(cassandraConnector, _))
      /*
      .mapPartitions(Logic.parseResult(data_output, _))
      .repartition(coalesce.toInt)
      .saveAsNewAPIHadoopFile(data_output, classOf[Text], classOf[Text], classOf[RDDMultipleOutputFormat[_, _]])

      val jsonArray = new JSONArray()
      audienceSum.foreach(m => {
        val jsonObject = new JSONObject()
        jsonObject.put("id", m._1)
        jsonObject.put("audience_data_status", 2)
        jsonObject.put("audience_count", m._2)
        jsonArray.add(jsonObject)
      })

      val jsonObject = ServerUtil.update(jsonArray)
      if (jsonObject.getInteger("code") == 200) {
        println("Audience Update OK!")
      }
      */

    } finally {
      if (sc != null) {
        sc.stop()
      }
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }
}

object RTDmpMain {
  def main(args: Array[String]): Unit = {
    new RTDmpMain().run(args)
  }
}