package mobvista.dmp.datasource.btop

import mobvista.dmp.common.CommonSparkJob
import org.apache.commons.cli.Options
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{SaveMode, SparkSession}

import java.net.URI

class BtopDaily  extends CommonSparkJob with Serializable {
  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("coalesce", true, "[must] coalesce")
    options.addOption("output", true, "[must] output")
    options.addOption("dt_today", true, "[must] dt_today")
    options.addOption("dt_dash_tow_days", true, "[must] dt_dash_tow_days")
    options.addOption("seven_days_ago", true, "[must] seven_days_ago")
    options.addOption("fifteen_days_ago", true, "[must] fifteen_days_ago")
    options.addOption("dt_two_days_ago", true, "[must] dt_two_days_ago")
    options.addOption("dt_dash_one_days", true, "[must] dt_dash_one_days")
    options
  }

  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      return -1
    } else printOptions(commandLine)

    val coalesce = commandLine.getOptionValue("coalesce")
    val output = commandLine.getOptionValue("output")
    val dt_today = commandLine.getOptionValue("dt_today")
    val dt_dash_tow_days = commandLine.getOptionValue("dt_dash_tow_days")
    val seven_days_ago = commandLine.getOptionValue("seven_days_ago")
    val fifteen_days_ago = commandLine.getOptionValue("fifteen_days_ago")
    val dt_two_days_ago = commandLine.getOptionValue("dt_two_days_ago")
    val dt_dash_one_days = commandLine.getOptionValue("dt_dash_one_days")

    val spark = SparkSession.builder()
      .appName("BtopDaily")
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "snappy")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()

    val sc = spark.sparkContext

    FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output), true)
//    idfv,imei,gaid,idfa,android_id
//    case when name='11' then 'android' when name='aa' then 'ios' end as os_platform,
    try {
      val year = dt_today.substring(0, 4)
      val month = dt_today.substring(4, 6)
      val day = dt_today.substring(6, 8)

//      当日新增设备
      val append_sql=
        s"""
           |select t1.device_id,
           |t1.device_type,
           |case when t1.os_platform='1' then 'android' when t1.os_platform='2' then 'ios' end as platform,
           |t2.package_name,
           |t1.country_code country
           |from
           |(select imei device_id,
           | 'imei' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and imei!='' and qcc_package_install_status='1'
           | union
           | select gaid device_id,
           | 'gaid' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and gaid!='' and qcc_package_install_status='1'
           | union
           | select idfa device_id,
           | 'idfa' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and idfa!='' and qcc_package_install_status='1'
           | union
           | select android_id device_id,
           | 'android_id' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and android_id!='' and qcc_package_install_status='1'
           | union
           | select idfv device_id,
           | 'idfv' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and idfv!='' and qcc_package_install_status='1'
           | union
           | select oaid device_id,
           | 'oaid' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and oaid!='' and qcc_package_install_status='1'
           | ) t1
           |join
           |(select id,package_name from  uparpu_main.uparpu_qcc_package ) t2
           |on
           |t1.qcc_package=t2.id
           |""".stripMargin

      spark.sql(append_sql).createOrReplaceTempView("btop_daily_append")

//      当日删除设备
      val delete_sql=
        s"""
           |select t1.device_id,
           |t1.device_type,
           |case when t1.os_platform='1' then 'android' when t1.os_platform='2' then 'ios' end as platform,
           |t2.package_name,
           |t1.country_code country
           |from
           |(select imei device_id,
           | 'imei' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and imei!='' and qcc_package_install_status='2'
           | union
           | select gaid device_id,
           | 'gaid' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and gaid!='' and qcc_package_install_status='2'
           | union
           | select idfa device_id,
           | 'idfa' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and idfa!='' and qcc_package_install_status='2'
           | union
           | select android_id device_id,
           | 'android_id' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and android_id!='' and qcc_package_install_status='2'
           | union
           | select idfv device_id,
           | 'idfv' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and idfv!='' and qcc_package_install_status='2'
           | union
           | select oaid device_id,
           | 'oaid' device_type,
           | os_platform,
           | qcc_package,
           | country_code
           | from  uparpu_main.uparpu_plugin_qcc_package where yyyy = '$year' and mm = '$month' and dd = '$day' and oaid!='' and qcc_package_install_status='2'
           | ) t1
           |join
           |(select id,package_name from  uparpu_main.uparpu_qcc_package ) t2
           |on
           |t1.qcc_package=t2.id
           |""".stripMargin

      spark.sql(delete_sql).createOrReplaceTempView("btop_daily_delete")

//      昨日设备
//      val yesterday_sql=
//        s"""
//           |select t1.device_id, t1.device_type, t1.platform, t1.package_name, t2.country
//           |from
//           |(select *
//           |from dwh.dm_install_list_v2 where dt='${dt_two_days_ago}' and business='btop' and update_date='${dt_dash_tow_days}') t1
//           |join
//           |(select * from
//           |dwh.ods_dmp_user_info where dt='${dt_two_days_ago}' and business='btop' and last_req_day='${dt_dash_tow_days}') t2
//           |on t1.device_id=t2.device_id
//           |""".stripMargin

            val yesterday_sql=
              s"""
                 |select device device_id, device_type, os_platform platform, package_name, country_code country
                 |from
                 |uparpu_main.uparpu_qcc_package_install_list where dt='${dt_dash_one_days}'
                 |""".stripMargin

      spark.sql(yesterday_sql).createOrReplaceTempView("btop_yesterday")

//     每天入库数据为：  增量+昨日全量-昨日删除
      val daily_sql =
        s"""
           |select device_id,
           |device_type,
           |platform,
           |package_name,
           |country
           |from btop_daily_append
           |union
           |select t1.device_id device_id,t1.device_type device_type,t1.platform  platform,t1.package_name  package_name,t1.country country
           |from btop_yesterday t1
           |left join
           |btop_daily_delete t2
           |on t1.device_id=t2.device_id and t1.package_name=t2.package_name and t1.device_type=t2.device_type and t1.country=t2.country
           |where  t2.device_id is null and t2.package_name is null and t2.device_type is null and t2.country is null
           |""".stripMargin

      spark.sql(daily_sql).createOrReplaceTempView("btop_daily")

     // 2021.05.11   com.eg.android.AlipayGphone_btop 的imei  oaid类型设备md5化，后入库
      val sql1=
        s"""
           |select md5(device_id) device_id,
           |concat(device_type,"md5") device_type,
           |platform,
           |concat(package_name,"_btop") package_name,
           |country
           |from btop_daily where package_name="com.eg.android.AlipayGphone"  and  device_type in ("imei","oaid")
           |union
           |select device_id,
           |device_type,
           |platform,
           |package_name,
           |country
           |from btop_daily
           |union
           |select device_id,
           |device_type,
           |platform,
           |concat(package_name,"_btop") package_name,
           |country
           |from btop_daily where package_name in ("com.eg.android.AlipayGphone","com.taobao.idlefish","com.UCMobile","com.qiyi.video","com.sankuai.meituan","com.youku.phone","com.quark.browser")
           |union
           |select t1.device_id device_id,t1.device_type device_type,'android' platform,'com.taobao.taobao_btop7' as package_name,t2.country_code country
           |from
           |(select device_id,device_type
           |from btop_daily where
           |package_name='com.taobao.taobao'
           |and device_type='imei') t1
           |join
           |(
           |select imei,country_code
           |from uparpu_main.uparpu_strategy_app_v2
           |where Concat(yyyy,mm,dd) >= '${seven_days_ago}'
           |and Concat(yyyy,mm,dd) <= '${dt_today}'
           |group by imei,country_code
           |) t2
           |on t1.device_id=t2.imei
           |group by t1.device_id,t1.device_type,t2.country_code
           |union
           |select t1.device_id device_id,t1.device_type device_type,'android' platform,'com.taobao.taobao_btop15' as package_name,t2.country_code country
           |from
           |(select device_id,device_type
           |from btop_daily where
           |package_name='com.taobao.taobao'
           |and device_type='imei') t1
           |join
           |(
           |select imei,country_code
           |from uparpu_main.uparpu_strategy_app_v2
           |where Concat(yyyy,mm,dd) >= '${fifteen_days_ago}'
           |and Concat(yyyy,mm,dd) <= '${dt_today}'
           |group by imei,country_code
           |) t2
           |on t1.device_id=t2.imei
           |group by t1.device_id,t1.device_type,t2.country_code
           |union
           |select t1.device_id device_id,t1.device_type device_type,'android' platform,'com.taobao.taobao_btop7_oaid' as package_name,t2.country_code country
           |from
           |(select device_id,device_type
           |from btop_daily where
           |package_name='com.taobao.taobao'
           |and device_type='oaid') t1
           |join
           |(
           |select oaid,country_code
           |from uparpu_main.uparpu_strategy_app_v2
           |where Concat(yyyy,mm,dd) >= '${seven_days_ago}'
           |and Concat(yyyy,mm,dd) <= '${dt_today}'
           |group by oaid,country_code
           |) t2
           |on t1.device_id=t2.oaid
           |group by t1.device_id,t1.device_type,t2.country_code
           |union
           |select t1.device_id device_id,t1.device_type device_type,'android' platform,'com.taobao.taobao_btop15_oaid' as package_name,t2.country_code country
           |from
           |(select device_id,device_type
           |from btop_daily where
           |package_name='com.taobao.taobao'
           |and device_type='oaid') t1
           |join
           |(
           |select oaid,country_code
           |from uparpu_main.uparpu_strategy_app_v2
           |where Concat(yyyy,mm,dd) >= '${fifteen_days_ago}'
           |and Concat(yyyy,mm,dd) <= '${dt_today}'
           |group by oaid,country_code
           |) t2
           |on t1.device_id=t2.oaid
           |group by t1.device_id,t1.device_type,t2.country_code
        """.stripMargin
      spark.sql(sql1).coalesce(coalesce.toInt)
        .write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .option("mapreduce.fileoutputcommitter.marksuccessfuljobs", false)
        .orc(output)

    } finally {
      spark.stop()
    }
    0
  }
}


object BtopDaily {
  def main(args: Array[String]): Unit = {
    new BtopDaily().run(args)
  }
}
