package mobvista.dmp.datasource.dsp

import com.fasterxml.jackson.databind.ObjectMapper
import mobvista.dmp.common.{CommonSparkJob, MobvistaConstant}
import mobvista.dmp.datasource.dsp.DspConstant.Value
import mobvista.dmp.datasource.dsp.mapreduce.SegmentVO
import mobvista.dmp.util.MD5Util
import mobvista.prd.datasource.util.GsonUtil
import org.apache.commons.cli.{BasicParser, Options}
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.sql.functions.{collect_set, max, udf}
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Row, SparkSession}

import java.net.URI
import java.util
import java.util.regex.Pattern
import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer


class DspOrgEtlDailys extends CommonSparkJob with Serializable {

  private val iosPkgPtn = Pattern.compile("^\\d+$")
  private val adrPkgPtn = Pattern.compile("^[0-9a-zA-Z\\.]+$")

  override protected def run(args: Array[String]): Int = {
    val parser = new BasicParser()
    val options = commandOptions()
    val commandLine = parser.parse(options, args)

    val input = commandLine.getOptionValue("input")
    val output_etl_dsp_request_hour = commandLine.getOptionValue("output")
    val parallelism = commandLine.getOptionValue("parallelism").toInt
    val coalesce = commandLine.getOptionValue("coalesce").toInt


    val spark = SparkSession
      .builder()
      .config("spark.rdd.compress", "true")
      .config("spark.default.parallelism", s"${parallelism}")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()
    val sc = spark.sparkContext


    FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output_etl_dsp_request_hour), true)


    import spark.implicits._

    try {

      val mergePkgUdf = udf((pkgsArrays: mutable.WrappedArray[String]) => {
        var res = "[]"
        val pkgSet: util.Set[String] = new util.HashSet[String]()
        if (pkgsArrays != null && pkgsArrays.size != 0) {
          for (pkgs <- pkgsArrays) {
            if (StringUtils.isNotBlank(pkgs)) {
              val pkgsname = pkgs.split("#", -1)
              for (pkgname <- pkgsname) {
                pkgSet.add(pkgname)
              }
            }
          }
        }

        if (pkgSet.size() != 0) {
          res = new ObjectMapper().writeValueAsString(pkgSet)
        }
        res
      })

      val mergeAdridIdUdf = udf((adrsArrays: mutable.WrappedArray[String]) => {
        var res = ""
        val adrSet: util.Set[String] = new util.HashSet[String]()
        if (adrsArrays != null && adrsArrays.size != 0) {
          for (adrs <- adrsArrays) {
            if (StringUtils.isNotBlank(adrs)) {
              adrSet.add(adrs)
            }
          }
        }
        if (adrSet.size() != 0) {
          res = adrSet.mkString(",")
        }
        res
      })


      val mergeUdf = udf((segmentArrays: mutable.WrappedArray[String]) => {
        var ret = ""
        val segmentMap: mutable.HashMap[String, SegmentVO] = new mutable.HashMap[String, SegmentVO]()

        if (segmentArrays != null && segmentArrays.size != 0) {
          for (segment <- segmentArrays) {
            if (StringUtils.isNotBlank(segment) && segment.startsWith("[") && segment.contains("{")) {
              val jsonArray = GsonUtil.String2JsonArray(segment.toString)
              for (json <- jsonArray) { //mergeUdf:segmentArray2>[{"name":"carrier","value":"verified"}];{"name":"carrier","value":"verified"}
                // println("mergeUdf:segmentArray2>" + segmentArray.toString + ";" +json.toString)
                import mobvista.dmp.datasource.dsp.mapreduce.SegmentVO
                import mobvista.prd.datasource.util.GsonUtil
                val segmentVO = GsonUtil.fromJson(json, classOf[SegmentVO])
                segmentMap.put(segmentVO.getId, segmentVO)
              }
            }

          }
        }
        if (segmentMap.size != 0) {
          ret = new ObjectMapper().writeValueAsString(segmentMap.values())
        }
        ret
      })


      val mergeRegionUdf = udf((regsArrays: mutable.WrappedArray[String]) => {
        var res = ""
        val regSet: util.Set[String] = new util.HashSet[String]()
        if (regsArrays != null && regsArrays.size != 0) {
          for (regs <- regsArrays) {
            if (StringUtils.isNotBlank(regs)) {
              val regsname = regs.split("#", -1)
              for (regname <- regsname) {
                regSet.add(regname)
              }
            }
          }
        }

        if (regSet.size() != 0) {
          res = new ObjectMapper().writeValueAsString(regSet)
        }
        res
      })


      val midData = spark.read.schema(dspEtlSchema).orc(input).flatMap(parseMapData(_)).toDF("device_id", "device_type", "platform", "country_code", "ip", "gender", "birthday", "maker",
        "model", "os_version", "package_list", "androidids", "datetime", "segment", "region")


      midData
        .groupBy("device_id", "device_type")
        .agg(max("platform"),
          max("country_code"),
          max("ip"),
          max("gender"),
          max("birthday"),
          max("maker"),
          max("model"),
          max("os_version"),
          mergePkgUdf(collect_set("package_list")),
          mergeAdridIdUdf(collect_set("androidids")),
          max("datetime"),
          mergeUdf(collect_set("segment")),
          mergeRegionUdf(collect_set("region"))
        ).repartition(coalesce)
        .rdd.map(_.mkString("\t")).saveAsTextFile(output_etl_dsp_request_hour, classOf[GzipCodec])


    } finally {
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }


  def parseMapData(row: Row): Array[DspReqVOoptimization] = {
    var arrayBuffer = new ArrayBuffer[DspReqVOoptimization]()

    //    val idfa = row.getAs[String]("idfa")
    var idfa = ""
    val idfaTmp = row.getAs[String]("idfa")
    if (StringUtils.isNotBlank(idfaTmp) && idfaTmp.matches(didPtn)) idfa = idfaTmp

    //    val gaid = row.getAs[String]("gaid")
    var gaid = ""
    val gaidTmp = row.getAs[String]("gaid")
    if (StringUtils.isNotBlank(gaidTmp) && gaidTmp.matches(didPtn)) gaid = gaidTmp


    val platform = row.getAs[String]("platform")
    val country = row.getAs[String]("country")
    val ip = row.getAs[String]("ip")
    val gender = row.getAs[String]("gender")
    val birthday = row.getAs[String]("birthday")
    val maker = row.getAs[String]("maker")
    val model = row.getAs[String]("model")
    val osVersion = row.getAs[String]("osVersion")
    var packageName = ""
    if (StringUtils.isNotBlank(row.getAs[String]("packageName"))) {
      packageName = row.getAs[String]("packageName")
    }

    val exitId = row.getAs[String]("exitId")
    val time = row.getAs[String]("datetime")
    var segment = ""
    if (StringUtils.isNotBlank(row.getAs[String]("segment"))) {
      segment = row.getAs[String]("segment")
    }
    val region = row.getAs[String]("region")

    var androidId = ""
    var imei = ""
    var imeimd5 = ""
    // 添加oaid  oaidmd5 解析
    var oaid = ""
    var oaidmd5 = ""

    //  新增
    var idfv = ""
    var gaidmd5 = ""

    if (StringUtils.isNotBlank(exitId)) {
      val devIds = splitFun(exitId, ",")
      if (devIds.length >= 17) {
        if ("ios".equalsIgnoreCase(platform)) {
          if (StringUtils.isBlank(idfa) && StringUtils.isNotBlank(devIds(1)) && devIds(1).matches(MobvistaConstant.didPtn)) {
            idfa = devIds(1)
          }
          if (StringUtils.isNotBlank(devIds(16)) && devIds(16).matches(MobvistaConstant.didPtn)) {
            idfv = devIds(16)
          }
        } else {
          if (StringUtils.isBlank(gaid) && StringUtils.isNotBlank(devIds(0)) && devIds(0).matches(MobvistaConstant.didPtn)) {
            gaid = devIds(0)
          }
          if (StringUtils.isNotBlank(devIds(2)) && devIds(2).matches(MobvistaConstant.md5Ptn)) {
            gaidmd5 = devIds(2)
          }
          if (StringUtils.isNotBlank(devIds(12))) {
            oaid = devIds(12)
          }
          if (StringUtils.isNotBlank(devIds(13)) && devIds(13).matches(MobvistaConstant.md5Ptn)) {
            oaidmd5 = devIds(13)
          }
          if (StringUtils.isNotBlank(devIds(4)) && devIds(4).matches(MobvistaConstant.imeiPtn)) {
            imei = devIds(4)
          }
          if (StringUtils.isNotBlank(devIds(5)) && devIds(5).matches(MobvistaConstant.md5Ptn)) {
            imeimd5 = devIds(5)
          }
          if (StringUtils.isNotBlank(devIds(7)) && devIds(7).matches(MobvistaConstant.andriodIdPtn)) {
            androidId = devIds(7)
          }
        }
      }

      /*
      if (devIds.length >= 14) {
        if (StringUtils.isNotBlank(devIds(7)) && devIds(7).matches(CommonMapReduce.andriodIdPtn)) {
          androidId = devIds(7)
        }

        if (StringUtils.isNotBlank(devIds(4)) && devIds(4).matches(CommonMapReduce.imeiPtn) && "android".equals(platform) && StringUtils.isNotBlank(country) && "CN".equalsIgnoreCase(country)) {
          imei = devIds(4)
        }

        if (StringUtils.isNotBlank(devIds(5)) && devIds(5).matches(CommonMapReduce.imeiMd5Ptn) && "android".equals(platform) && StringUtils.isNotBlank(country) && "CN".equalsIgnoreCase(country)) {
          imeimd5 = devIds(5)
        }

        if (StringUtils.isNotBlank(devIds(12)) && "android".equals(platform) && StringUtils.isNotBlank(country) && "CN".equalsIgnoreCase(country)) {
          oaid = devIds(12)
        }

        if (StringUtils.isNotBlank(devIds(13)) && devIds(13).matches(CommonMapReduce.imeiMd5Ptn) && "android".equals(platform) && StringUtils.isNotBlank(country) && "CN".equalsIgnoreCase(country)) {
          oaidmd5 = devIds(13)
        }
      }
      */
    }

    val value = Value(country, ip, gender, birthday, maker, model, osVersion, packageName, androidId, time, segment, region)

    var deviceType = ""
    var deviceId = ""


    if ("ios".equals(platform) && idfa.length > 4 && !allZero.equals(idfa)) {
      deviceId = idfa
      deviceType = "idfa"
    } else if ("android".equals(platform) && gaid.length > 4 && !allZero.equals(gaid)) {
      deviceId = gaid
      deviceType = "gaid"
    }

    arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)

    /*
    if (!deviceId.isEmpty) {
      arrayBuffer += DspReqVOoptimization(deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }

    if (!androidId.isEmpty) {
      arrayBuffer += DspReqVOoptimization(androidId, "androidid", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }

    if (StringUtils.isNotBlank(imei)) {
      arrayBuffer += DspReqVOoptimization(imei, "imei", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)

      val imeimd5 = MD5Util.getMD5Str(imei)
      arrayBuffer += DspReqVOoptimization(imeimd5, "imeimd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)

    }

    if (StringUtils.isNotBlank(imeimd5)) {
      arrayBuffer += DspReqVOoptimization(imeimd5, "imeimd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }

    if (StringUtils.isNotBlank(oaid)) {
      arrayBuffer += DspReqVOoptimization(oaid, "oaid", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)

      val oaidmd5 = MD5Util.getMD5Str(oaid)
      arrayBuffer += DspReqVOoptimization(oaidmd5, "oaidmd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }

    if (StringUtils.isNotBlank(oaidmd5)) {
      arrayBuffer += DspReqVOoptimization(oaidmd5, "oaidmd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }
    */

    //  添加dealid入库逻辑
    val exchanges = row.getAs[String]("exchanges")
    val dealerid = row.getAs[String]("dealerid")
    val dealeridArray: Array[String] = dealerid.split(",")
    if ("oppocn".equals(exchanges)) {
      if("android".equalsIgnoreCase(platform)) {
          if ( !(dealeridArray.contains("2532") || dealeridArray.contains("2533")) ){
          // 2020.11.26 需求 若adx（exchanges字段）为oppo且dealid不等于2532或2533，则伪包名为com.taobao.taobao_notinstall_oppo
          value.packageName = "com.taobao.taobao_notinstall_oppo"
          arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
        }

        val mapData_oppo= Map(
          "2532" -> Array("com.taobao.taobao","com.taobao.taobao_oppo"),
          "2533" -> Array("com.taobao.taobao","com.taobao.taobao_oppo"),
          "4053" -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_oppolianmeng"),
          "4143" -> Array("com.xunmeng.pinduoduo","com.xunmeng.pinduoduo_oppolianmeng"),
          "2906" -> Array("com.youku.phone","com.youku.phone_oppolianmeng"),
          "4182" -> Array("me.ele","me.ele_oppolianmeng"),
          "4059" -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_oppoziyou"),
          "4140" -> Array("com.xunmeng.pinduoduo","com.xunmeng.pinduoduo_oppoziyou"),
          "4183" -> Array("me.ele","me.ele_oppoziyou"),
          "2728" -> Array("com.UCMobile","com.ucmobile_oppo"),   //  添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
          "4060" -> Array("com.ss.android.ugc.aweme_oppoziyou_notinstall"),
          "4061" -> Array("com.ss.android.ugc.aweme_oppoziyou_hist_notinstall"),
          "4054" -> Array("com.ss.android.ugc.aweme_oppolianmeng_hist1year_notinstall"),
          "4055" -> Array("com.ss.android.ugc.aweme_oppolianmeng_histhalfyear_notinstall"),
          "4141" -> Array("com.xunmeng.pinduoduo_oppoziyou_notinstall"),
          "4142" -> Array("com.xunmeng.pinduoduo_oppoziyou_hist_notinstall"),
          "4144" -> Array("com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall"),
          "4145" -> Array("com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall"),
          "4184" -> Array("me.ele_oppoziyou_hist_notinstall"),
          "4185" -> Array("me.ele_oppoziyou_notinstall"),
          "4180" -> Array("me.ele_oppolianmeng_notinstall"),
          "2840" -> Array("com.youku.phone_oppocn_notinstall"),
          "4181" -> Array("me.ele_oppolianmeng_hist_notinstall"),
          "2774" -> Array("com.smile.gifmaker_notinstall_oppo"),   //2020.11.11添加快手、京东入库
          "2773" -> Array("com.jingdong.app.mallr_notinstall_oppo"),
          "2716" -> Array("com.jingdong.app.mall","com.jingdong.app.mall_oppo"),
          "2717" -> Array("com.eg.android.AlipayGphone","com.eg.android.AlipayGphone_oppo"),  //补充说明： com.eg.android.AlipayGphone_oppo伪包名大写，但已经入库，修改困难。以后入库伪包名统一定为小写，类似上面的 com.ucmobile_oppo）
          "2718" -> Array("com.qiyi.video","com.qiyi.video_oppo"),
          "2783" -> Array("com.taobao.idlefish","com.taobao.idlefish_oppo"),
          "2840" -> Array("com.youku.phone_notinstall","com.youku.phone_notinstall_oppo"),
          "2889" -> Array("com.sankuai.meituan","com.sankuai.meituan_oppo"),
          "2890" -> Array("com.meituan.itakeaway","com.meituan.itakeaway_oppo"),
          "2904" -> Array("com.smile.gifmaker","com.smile.gifmaker_oppo"),
          "2905" -> Array("com.kuaishou.nebula","com.kuaishou.nebula_oppo"),
          "2906" -> Array("com.youku.phone","com.youku.phone_oppo"),
          "3160" -> Array("com.tencent.news","com.tencent.news_oppo")
        )
        for (item <- mapData_oppo) {
          if (dealeridArray.contains(item._1)) {
            for( packageName <-  item._2 ){
              value.packageName = packageName
              arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
            }
          }
        }
      }
    }
    if ("inmobi".equals(exchanges)) {
      if("android".equalsIgnoreCase(platform)) {
        if (dealeridArray.contains("1594807676568")) {
          value.packageName = "com.taobao.taobao_inmobi"
          arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
        }
      }
    }
    //2020.11.26 增加bes入库需求
    // wiki   https://confluence.mobvista.com/pages/viewpage.action?pageId=47976499
    //2021.04.22 添加adx（exchanges字段）为bes且dealid=100188，则伪包名为com.tencent.news_bes和com.tencent.news
    //2021.06.17 添加adx（exchanges字段）为bes且dealid=100310，则伪包名为com.taobao.litetao_bes和com.taobao.litetao
    val mapData_bes = Map("100193" -> Array("com.taobao.taobao","com.taobao.taobao_bes"),
      "100189" -> Array("com.eg.android.AlipayGphone","com.eg.android.AlipayGphone_bes"),
      "100191" -> Array("com.jingdong.app.mall","com.jingdong.app.mall_bes"),
      "100187" -> Array("com.UCMobile","com.UCMobile_bes"),
      "100194" -> Array("com.taobao.idlefish","com.taobao.idlefish_bes"),
      "100195" -> Array("com.qiyi.video","com.qiyi.video_bes"),
      "100196" -> Array("com.smile.gifmaker","com.smile.gifmaker_bes"),
      "100188" -> Array("com.tencent.news","com.tencent.news_bes"),
      "100310" -> Array("com.taobao.litetao","com.taobao.litetao_bes"),
      "100203" -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_bes"),
      "00251" ->  Array("com.xunmeng.pinduoduo","com.xunmeng.pinduoduo_bes"),
      "100400" -> Array("com.youku.phone","com.youku.phone_bes"),
      "100219" -> Array("com.youku.phone_bes_notinstall"),
      "100304" -> Array("me.ele","me.ele_bes") )

    val mapData_ios_bes = Map("100197" -> Array("id387682726","id3876827262021090301"),
      "100345" -> Array("id333206289","id3332062892021090301"),
      "100344" -> Array("id1340376323","id13403763232021090301"),
      "100353" -> Array("id1044283059","id10442830592021090301"),
      "100220" -> Array("id3361414752022010401"),
      "100368" -> Array("id507161324","id5071613242021091701") )

    if ("bes".equals(exchanges)) {
      if("android".equalsIgnoreCase(platform)) {
        for (item <- mapData_bes) {
          if (dealeridArray.contains(item._1)) {
            for( packageName <-  item._2 ){
              value.packageName = packageName
              arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
            }
          }
        }
      }
      if("ios".equalsIgnoreCase(platform)) {
        for (item <- mapData_ios_bes) {
          if (dealeridArray.contains(item._1)) {
            for( packageName <-  item._2 ){
              value.packageName = packageName
              arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
            }
          }
        }
      }
    }
    //2021.04.28  天级别从dsp请求日志（adn_dsp.log_adn_dsp_request_orc_hour）中拉取exchanges='tencent' 并且appid = 'com.tencent.news',入库伪包名为com.tencent.news_fromtencent
    if ("tencent".equals(exchanges)) {
      if("android".equalsIgnoreCase(platform)) {
        if (packageName.split("#", -1).contains("com.tencent.news")) {
          value.packageName = "com.tencent.news_fromtencent"
          arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
        }
      }
    }
    //2021.06.17  天级别从dsp请求日志（adn_dsp.log_adn_dsp_request_orc_hour）中拉取exchanges='kuaishou' 并且appid 符合下列名称的入库
    val mapData_kuaishou = Map("com.smile.gifmaker" -> "com.smile.gifmaker_fromkuaishou",
      "com.kuaishou.nebula" -> "com.kuaishou.nebula_fromkuaishou",
      "440948110" -> "44094811020210617",
      "1472502819" -> "147250281920210617")
    if ("kuaishou".equals(exchanges)) {
      for (item <- mapData_kuaishou) {
        if (packageName.split("#", -1).contains(item._1)) {
          value.packageName = item._2
          arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
        }
      }
    }

    val mapData_iqiyi = Map(5260 -> Array("com.taobao.taobao","com.taobao.taobao_iqiyi"),
      1301 ->   Array("com.UCMobile","com.UCMobile_iqiyi"),
      10949 ->  Array("com.eg.android.AlipayGphone","com.eg.android.AlipayGphone_iqiyi"),
      6242 ->   Array("com.taobao.idlefish","com.taobao.idlefish_iqiyi"),
      3996 ->   Array("com.sankuai.meituan","com.sankuai.meituan_iqiyi"),
      3334 ->   Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_iqiyi"),
      112799 -> Array("com.ss.android.ugc.aweme","com.ss.android.ugc.aweme_iqiyi"),
      7156 ->   Array("com.tencent.news","com.tencent.news_iqiyi"),
      112644 ->   Array("com.taobao.litetao","com.taobao.litetao_iqiyi"),
      1649 ->   Array("me.ele","me.ele_iqiyi") )
    val mapData_ios_iqiyi = Map(
      3334 ->   Array("id1142110895","id11421108952021090302"),
      112799 -> Array("id1142110895","id11421108952021090302"),
      112644 -> Array("1340376323","134037632320210617"),
      1649 ->   Array("id507161324","id5071613242021091702"))
    if("iqiyi".equals(exchanges)){
      if("android".equalsIgnoreCase(platform)) {
           for (item <- mapData_iqiyi) {
             if (dealeridArray.contains(item._1.toString)) {
               for( packageName <-  item._2 ){
                 value.packageName = packageName
                 arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
             }
           }
        }
      }
      if("ios".equalsIgnoreCase(platform)) {
           for (item <- mapData_ios_iqiyi) {
             if (dealeridArray.contains(item._1.toString)) {
               for( packageName <-  item._2 ){
                 value.packageName = packageName
                 arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
             }
           }
         }
      }
    }


    val mapData_vivo = Map("ba6e4f5323603561086a985b6fc5ffb4" -> Array("me.ele","me.ele_vivo"),
      "d326b636626b0d927be609d317b88cae" -> Array("com.youku.phone","com.youku.phone_vivo"))
    val mapData_ios_vivo = Map(
      "d326b636626b0d927be609d317b88cae" -> Array("id336141475","id3361414752022010402") )
    if ("vivo".equals(exchanges)) {
      if("android".equalsIgnoreCase(platform)) {
        for (item <- mapData_vivo) {
          if (dealeridArray.contains(item._1)) {
            for( packageName <-  item._2 ){
              value.packageName = packageName
              arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
            }
          }
        }
      }
      if("ios".equalsIgnoreCase(platform)) {
        for (item <- mapData_ios_vivo) {
          if (dealeridArray.contains(item._1)) {
            for( packageName <-  item._2 ){
              value.packageName = packageName
              arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
            }
          }
        }
      }
    }

    //adx=iqiyi 且os=android的imeiMD5和oaidMD5去重设备， 分别和com.taobao.foractivation.227229和com.taobao.foractivation.227229_oaid做差集   2020.12.15
    //2021.04.22下掉,该需求不需要了
    //    if("iqiyi".equals(exchanges)){
    //      arrayBuffer = addDatas(arrayBuffer,imei,imeimd5,oaid,oaidmd5,deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
    //        "com.iqiyi_android.subtract_with_227229", androidId, time, segment,region)
    //    }
    arrayBuffer.toArray
  }


  def addDatas(arrayBuffer: ArrayBuffer[DspReqVOoptimization], imei: String, imeimd5: String, oaid: String, oaidmd5: String, deviceId: String, deviceType: String, platform: String, country: String, ip: String, gender: String, birthday: String, maker: String,
               model: String, osVersion: String, packageName: String, androidId: String, time: String, segment: String, region: String): ArrayBuffer[DspReqVOoptimization] = {
    if (!deviceId.isEmpty) {
      arrayBuffer += DspReqVOoptimization(deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }
    if (!androidId.isEmpty) {
      arrayBuffer += DspReqVOoptimization(androidId, "androidid", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }
    if (StringUtils.isNotBlank(imei)) {
      arrayBuffer += DspReqVOoptimization(imei, "imei", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
      val imeimd5 = MD5Util.getMD5Str(imei)
      arrayBuffer += DspReqVOoptimization(imeimd5, "imeimd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }
    if (StringUtils.isNotBlank(imeimd5)) {
      arrayBuffer += DspReqVOoptimization(imeimd5, "imeimd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }
    if (StringUtils.isNotBlank(oaid)) {
      arrayBuffer += DspReqVOoptimization(oaid, "oaid", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)

      val oaidmd5 = MD5Util.getMD5Str(oaid)
      arrayBuffer += DspReqVOoptimization(oaidmd5, "oaidmd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)

    }
    if (StringUtils.isNotBlank(oaidmd5)) {
      arrayBuffer += DspReqVOoptimization(oaidmd5, "oaidmd5", "android", country, ip, gender, birthday, maker, model, osVersion,
        packageName, androidId, time, segment, region)
    }
    arrayBuffer
  }

  def addDatasV2(arrayBuffer: ArrayBuffer[DspReqVOoptimization], deviceId: String, gaidmd5: String, imei: String, imeimd5: String,
                 oaid: String, oaidmd5: String, androidId: String, idfv: String, deviceType: String, platform: String, value: Value
                ): ArrayBuffer[DspReqVOoptimization] = {
    var gaidFlag = true
    if (StringUtils.isNotBlank(deviceId)) {
      arrayBuffer += DspReqVOoptimization(deviceId, deviceType, platform,
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
      if ("android".equalsIgnoreCase(platform) && "gaid".equalsIgnoreCase(deviceType)) {
        val gaidmd5 = MD5Util.getMD5Str(deviceId)
        gaidFlag = false
        arrayBuffer += DspReqVOoptimization(gaidmd5, "gaidmd5", "android",
          value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
      }
    }
    if (StringUtils.isNotBlank(gaidmd5) && gaidFlag) {
      arrayBuffer += DspReqVOoptimization(gaidmd5, "gaidmd5", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    if (StringUtils.isNotBlank(deviceId)) {
      arrayBuffer += DspReqVOoptimization(deviceId, deviceType, platform,
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    if (StringUtils.isNotBlank(androidId)) {
      arrayBuffer += DspReqVOoptimization(androidId, "androidid", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    if (StringUtils.isNotBlank(idfv)) {
      arrayBuffer += DspReqVOoptimization(idfv, "idfv", "ios",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    var imeiFlag = true
    if (StringUtils.isNotBlank(imei)) {
      arrayBuffer += DspReqVOoptimization(imei, "imei", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
      val imeimd5 = MD5Util.getMD5Str(imei)
      imeiFlag = false
      arrayBuffer += DspReqVOoptimization(imeimd5, "imeimd5", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    if (StringUtils.isNotBlank(imeimd5) && imeiFlag) {
      arrayBuffer += DspReqVOoptimization(imeimd5, "imeimd5", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    var oaidFlag = true
    if (StringUtils.isNotBlank(oaid)) {
      arrayBuffer += DspReqVOoptimization(oaid, "oaid", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
      val oaidmd5 = MD5Util.getMD5Str(oaid)
      oaidFlag = false
      arrayBuffer += DspReqVOoptimization(oaidmd5, "oaidmd5", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)

    }
    if (StringUtils.isNotBlank(oaidmd5) && oaidFlag) {
      arrayBuffer += DspReqVOoptimization(oaidmd5, "oaidmd5", "android",
        value.country, value.ip, value.gender, value.birthday, value.maker, value.model, value.osVersion, value.packageName, value.androidId, value.time, value.segment, value.region)
    }
    arrayBuffer
  }


  def dspEtlSchema: StructType = {
    StructType(StructField("idfa", StringType) ::
      StructField("gaid", StringType) ::
      StructField("platform", StringType) ::
      StructField("country", StringType) ::
      StructField("ip", StringType) ::
      StructField("gender", StringType) ::
      StructField("birthday", StringType) ::
      StructField("maker", StringType) ::
      StructField("model", StringType) ::
      StructField("osVersion", StringType) ::
      StructField("packageName", StringType) ::
      StructField("exitId", StringType) ::
      StructField("datetime", StringType) ::
      StructField("segment", StringType) ::
      StructField("dealerid", StringType) ::
      StructField("exchanges", StringType) ::
      StructField("region", StringType)
      :: Nil)
  }


  def commandOptions(): Options = {
    val options = new Options()
    options.addOption("input", true, "input")
    options.addOption("output", true, "output")
    options.addOption("parallelism", true, "parallelism")
    options.addOption("coalesce", true, "coalesce")
    options
  }
}

object DspOrgEtlDailys {
  def main(args: Array[String]): Unit = {
    new DspOrgEtlDailys().run(args)
  }
}
