package mobvista.dmp.datasource.dm

import mobvista.dmp.common.CommonSparkJob
import org.apache.commons.cli.Options
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.functions.{concat_ws, lit}
import org.apache.spark.storage.StorageLevel

import java.net.URI

/**
 * @author jiangfan
 * @date 2021/11/12 14:36
 */

class  ComLazadaNoregister70p30p  extends CommonSparkJob with Serializable {
  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("coalesce", true, "[must] coalesce")
    options.addOption("output01", true, "[must] output01")
    options.addOption("output02", true, "[must] output02")
    options.addOption("first_day", true, "[must] first_day")
    options.addOption("second_day", true, "[must] second_day")
    options
  }

  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      return -1
    } else printOptions(commandLine)

    val coalesce = commandLine.getOptionValue("coalesce")
    val output01 = commandLine.getOptionValue("output01")
    val output02 = commandLine.getOptionValue("output02")
    val first_day = commandLine.getOptionValue("first_day")
    val second_day = commandLine.getOptionValue("second_day")

    val spark = SparkSession.builder()
      .appName("ComLazadaNoregister70p30p")
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "snappy")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()

    val sc = spark.sparkContext
    import spark.implicits._

    FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output01), true)
    FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output02), true)

    val yyyy01 = first_day.substring(0, 4)
    val mm01 = first_day.substring(4, 6)
    val dd01 = first_day.substring(6, 8)

    val yyyy02 = second_day.substring(0, 4)
    val mm02 = second_day.substring(4, 6)
    val dd02 = second_day.substring(6, 8)

    var time_limit=""
// 用下列方式取代sql中的concat(yyyy,mm,dd) = 20211111 避免扫描整个hive表的元数据
    if(yyyy01==yyyy02 && mm01==mm02)
    { time_limit=s"""yyyy = '${yyyy01}' and mm= '${mm01}'  and dd>='${dd01}'  and dd<='${dd02}'  """}
    else
    { time_limit=s"""( (yyyy = '${yyyy01}' and mm= '${mm01}'  and dd='${dd01}')  or (yyyy = '${yyyy02}' and mm= '${mm02}'  and dd='${dd02}') )  """}

    println(time_limit)

    try {

      val sql01 =
        s"""
           |select
           |t1.gaid device_id,cast(ceiling(rand() * 100) as int) num
           |from
           |(select
           |campaign_id,gaid
           |from
           |dwh.ods_adn_trackingnew_install
           |where
           |yyyy = '${yyyy01}' and mm= '${mm01}'  and dd='${dd01}'
           |and
           |country_code in ('ID')
           |and ext_campaignpackagename = 'com.lazada.android'
           |group by campaign_id,gaid) t1
           |left join
           |(select
           |campaign_id,gaid
           |from
           |dwh.ods_adn_tracking_ss_event
           |where
           |  ${time_limit}
           |and
           |  country in ('ID')
           |and
           |  event_name = 'REGISTRATION'
           |group by campaign_id,gaid
           |) t2
           |on t1.campaign_id = t2.campaign_id and t1.gaid = t2.gaid
           |where
           |t2.gaid is null
           |group by t1.gaid
           |""".stripMargin
      println("sql01===="+sql01)

      val all_df: DataFrame = spark.sql(sql01).persist(StorageLevel.MEMORY_AND_DISK_SER)
      val df01_with_package_name = all_df.filter(all_df.col("num")<=30).select(concat_ws("\t", all_df.col("device_id"),  lit("gaid"),  lit("android"),lit("[\"com.lazada_noregister_30p\"]")))
      val df01_with_country = all_df.filter(all_df.col("num")<=30).select(concat_ws("\t", all_df.col("device_id"),  lit("gaid"),  lit("android"),lit("ID")))

      val df02_with_package_name = all_df.filter(all_df.col("num")>30).select(concat_ws("\t", all_df.col("device_id"),  lit("gaid"),  lit("android"),lit("[\"com.lazada_noregister_70p\"]")))
      val df02_with_country = all_df.filter(all_df.col("num")>30).select(concat_ws("\t", all_df.col("device_id"),  lit("gaid"),  lit("android"),lit("ID")))

      df01_with_package_name.union(df02_with_package_name).coalesce(coalesce.toInt).write.format("text").mode("overwrite").save(output01)
      df01_with_country.union(df02_with_country).coalesce(coalesce.toInt).write.format("text").mode("overwrite").save(output02)

    } finally {
      spark.stop()
    }
    0
  }
}

object ComLazadaNoregister70p30p {
  def main(args: Array[String]): Unit = {
    new  ComLazadaNoregister70p30p().run(args)
  }
}
