package mobvista.dmp.datasource.event_tag

import mobvista.dmp.common.{CommonSparkJob, MobvistaConstant}
import org.apache.commons.cli.Options
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SaveMode, SparkSession}

import java.net.URI
import scala.collection.mutable.ArrayBuffer

class DmpSsEventTag extends CommonSparkJob with Serializable {


  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      return 1
    } else {
      printOptions(commandLine)
    }


    val today = commandLine.getOptionValue("today")
    val input_path = commandLine.getOptionValue("input")
    val coalesce = commandLine.getOptionValue("coalesce")
    val output = commandLine.getOptionValue("output")

    val spark = SparkSession
      .builder()
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "lz4")
      .config("spark.io.compression.lz4.blockSize", "64k")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.sql.autoBroadcastJoinThreshold", "209715200")
      .config("spark.sql.broadcastTimeout", "1200")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()

    import spark.implicits._
    FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output), true)

    try {

      val input_LOGRDD = spark.
        sparkContext.
        textFile(input_path)
        .map(splitFun(_))
        .filter(filterData(_))
        .mapPartitions(
          elements => {
            val linesArr = new ArrayBuffer[Row]()
            if (!elements.isEmpty) {
              elements.foreach(array => {
                val androidid = array(3)
                val imei = array(4)
                val gaid = array(6)
                val idfa = array(7)
                val event_time = array(9)
                val event_name = array(10)
                val event_value = array(11)
                val campaign_id = array(14)
                val platform = array(15)
                val country = array(16)
                val `type` = array(18)


                if (StringUtils.isNotBlank(imei) && imei.matches(imeiPtn)) {
                  linesArr += Row(`type`, imei, "imei", platform, campaign_id, country, event_name, event_value, event_time)
                }

                if (StringUtils.isNotBlank(androidid) && androidid.matches(andriodIdPtn)) {
                  linesArr += Row(`type`, androidid, "androidid", platform, campaign_id, country, event_name, event_value, event_time)
                }

                if (StringUtils.isNotBlank(gaid) && gaid.matches(didPtn) && !allZero.equals(gaid)) {
                  linesArr += Row(`type`, gaid, "gaid", platform, campaign_id, country, event_name, event_value, event_time)
                }

                if (StringUtils.isNotBlank(idfa) && idfa.matches(didPtn) && !allZero.equals(idfa)) {
                  linesArr += Row(`type`, idfa, "idfa", platform, campaign_id, country, event_name, event_value, event_time)
                }

              })
            }

            linesArr.toIterator
          }
        )

      val schema = StructType(
        StructField("type", StringType) ::
          StructField("device_id", StringType) ::
          StructField("device_type", StringType) ::
          StructField("platform", StringType) ::
          StructField("campaign_id", StringType) ::
          StructField("country", StringType) ::
          StructField("event_name", StringType) ::
          StructField("event_value", StringType) ::
          StructField("event_time", StringType) ::
          Nil)

      spark.sqlContext.createDataFrame(input_LOGRDD, schema).createOrReplaceTempView("etl_ss_event_org_tab")


      /* val sql1=
         s"""
            |select
            |'' type,
            |t2.uuid,
            |t1.device_id,
            |t1.device_type,
            |t1.platform,
            |case when instr(t2.package_name,'id') = 1 then substr(t2.package_name,3) else package_name end as package_name,
            |t1.country,
            |t1.event_name,
            |t1.event_value,
            |t1.event_time,
            |''  install_time,
            |'${today}' update_date
            |from etl_ss_event_org_tab t1 join ( select id,uuid,package_name from dwh.ods_adn_campaign_list where dt like '20%' group by id,uuid,package_name) t2
            |on (t1.campaign_id = t2.id)
            |group by
            |t2.uuid,
            |t1.device_id,
            |t1.device_type,
            |t1.platform,
            |t2.package_name,
            |t1.country,
            |t1.event_name,
            |t1.event_value,
            |t1.event_time
         """.stripMargin */

      var sql1 =
        """
          |select
          |t1.`type`,
          |t2.uuid,
          |t1.device_id,
          |t1.device_type,
          |t1.platform,
          |t2.package_name,
          |t1.country,
          |t1.event_name,
          |t1.event_value,
          |t1.event_time,
          |t2.first_tag,
          |t2.second_tag,
          |case when d.event_name is null then '0'  else '1' end as pursub,
          |''  install_time,
          |'${today}' update_date
          | from etl_ss_event_org_tab t1 join (select t0.id,t0.package_name,t0.uuid,t.first_tag,t.second_tag from (select id,uuid,case when instr(package_name,'id') = 1 then substr(package_name,3) else package_name end as package_name from dwh.ods_adn_campaign_list where dt like '2%') t0 join  dwh.dim_package_tags_combine t on(t0.package_name = t.package_name) group by  t0.id,t0.package_name,t0.uuid,t.first_tag,t.second_tag ) t2
          | on (t1.campaign_id = t2.id)
          |	LEFT  JOIN dwh.adv_event_name_tab d on (lower(t1.event_name) = lower(d.event_name))
          | group by
          | t1.`type`,
          | t2.uuid,
          | t1.device_id,
          | t1.device_type,
          | t1.platform,
          | t2.package_name,
          | t1.country,
          | t1.event_name,
          | t1.event_value,
          | t1.event_time,
          |	t2.first_tag,
          |t2.second_tag,
          |case when d.event_name is null then '0'  else '1' end
        """.stripMargin
      spark.sql(sql1).map(row => {
        val `type` = row.getAs[String]("type")
        val uuid = row.getAs[String]("uuid")
        val device_id = row.getAs[String]("device_id")
        val device_type = row.getAs[String]("device_type")
        val platform = row.getAs[String]("platform")
        val package_name = row.getAs[String]("package_name")
        val country = row.getAs[String]("country")
        val event_name = row.getAs[String]("event_name")
        val event_value = row.getAs[String]("event_value")
        val event_time = row.getAs[String]("event_time")
        val first_tag = row.getAs[String]("first_tag")
        val second_tag = row.getAs[String]("second_tag")
        var pursub = row.getAs[String]("pursub")
        val install_time = row.getAs[String]("install_time")
        val update_date = row.getAs[String]("update_date")

        var thirdparty = ""
        if (StringUtils.isNotBlank(`type`) && `type`.startsWith("{")) {
          val ext_data = MobvistaConstant.String2JSONObject(`type`)
          if (ext_data.containsKey("third_party")) {
            thirdparty = ext_data.getString("third_party")
          }
        }


        if ("appsflyer".equalsIgnoreCase(thirdparty) && "1".equalsIgnoreCase(pursub)) {
          if (StringUtils.isNotBlank(event_value) && event_value.startsWith("{")) {
            val value = MobvistaConstant.String2JSONObject(event_value)
            var af_revenue = 1.0
            try {
              if (value.containsKey("af_revenue")) {
                af_revenue = value.getDoubleValue("af_revenue")
                if (af_revenue < 0.9) {
                  pursub = "0"
                }
              }
            } catch {
              case e: Exception => Some(0)
            }

            if (value.containsKey("af_content_type") && "advertising".equalsIgnoreCase(value.getString("af_content_type"))) {
              pursub = "0"
            }
          }
        }
        (thirdparty, uuid, device_id, device_type, platform, package_name, country, event_name, event_value, event_time, first_tag, second_tag, pursub, install_time, update_date)
      }).toDF("type", "uuid", "device_id", "device_type", "platform", "package_name", "country", "event_name", "event_value", "event_time", "first_tag", "second_tag", "pursub", "install_time", "update_date").repartition(coalesce.toInt)
        .coalesce(coalesce.toInt)
        .write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .orc(output)

    } finally {
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }

  def filterData(array: Array[String]): Boolean = {
    if (array.length <= 21) {
      return false
    }

    val androidid = array(3)
    val imei = array(4)
    val gaid = array(6)
    val idfa = array(7)
    val event_name = array(10)
    val platform = array(15)

    if ("ios".equalsIgnoreCase(platform) && StringUtils.isBlank(idfa)) {
      return false
    }

    if ("android".equalsIgnoreCase(platform) && StringUtils.isBlank(imei) && StringUtils.isBlank(androidid) && StringUtils.isBlank(gaid)) {
      return false
    }

    if (StringUtils.isBlank(event_name)) {
      return false
    }

    true
  }

  def initConfig(conf: Configuration): Configuration = {
    import org.apache.hadoop.io.SequenceFile
    conf.set("mapreduce.output.compress", "true")
    conf.set("mapreduce.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec")
    conf.setBoolean("mapreduce.output.fileoutputformat.compress", true)
    conf.set("mapreduce.output.fileoutputformat.compress.type", SequenceFile.CompressionType.BLOCK.toString)
    conf
  }


  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("today", true, "[must] today")
    options.addOption("coalesce", true, "[must] coalesce")
    options.addOption("input", true, "[must] input")
    options.addOption("output", true, "[must] output path")
    options
  }


}


object DmpSsEventTag {
  def main(args: Array[String]): Unit = {
    new DmpSsEventTag().run(args)
  }
}