package mobvista.dmp.datasource.event_tag

import com.alibaba.fastjson.JSON
import org.apache.spark.sql._
import org.apache.spark.sql.types._

import java.text.SimpleDateFormat
import java.util.Date

/**
 * 刘凯      2017-12-18 15:20
 * 计算 3s event source 日志抽取事件
 */
object Dmp_event_source {
  def main(args: Array[String]) {
    val spark = SparkSession.builder()
      .enableHiveSupport()
      .getOrCreate()
    //yyyyMMdd
    val loadTime = spark.conf.get("spark.app.loadTime")
    var year = loadTime.substring(0, 4)
    var month = loadTime.substring(4, 6)
    val day = loadTime.substring(6, 8)
    val coalesce_num_str = spark.conf.get("spark.app.coalesce_num")
    var coalesce_num = 500
    if (coalesce_num_str.nonEmpty) {
      coalesce_num = Integer.parseInt(coalesce_num_str)
    }
    //����orc����
    //OrcConfigUtil.configSettings(sparkContext)
    //spark.sparkContext.hadoopConfiguration.set("mapreduce.fileoutputcommitter.algorithm.version", "2")
    //spark.sparkContext.hadoopConfiguration.set("yarn.nodemanager.pmem-check-enabled", "false")
    //spark.sparkContext.hadoopConfiguration.set("yarn.nodemanager.vmem-check-enabled", "false")
    val jobContext = spark.sqlContext
    
    val normal_path="s3://mob-ad/3s/trackinglogs/event/"+year+"/"+month+"/"+day+"/singapore/*/info/*"
    val warning_path="s3://mob-ad/3s/trackinglogs/event/"+year+"/"+month+"/"+day+"/singapore/*/warning/*"
    val input_path=normal_path+","+warning_path
      println("input_path=="+input_path)
    val input_LOGRDD = spark.sparkContext
      .textFile(input_path).coalesce(coalesce_num).filter { x => x.split("\t").length>2 }.map(p => parseRow(p,loadTime))
          
  val schema = StructType(Array(
      StructField("device_id", StringType, true),
      StructField("event_day", StringType, true),
      StructField("event_name", StringType, true),
      StructField("mobvista_clickid", StringType, true),
      StructField("campaign_id", StringType, true),
      StructField("uuid", StringType, true),
      StructField("event_time", LongType, true),
      StructField("platform", StringType, true),
      StructField("country", StringType, true),
      StructField("data_Info", StringType, true),
      StructField("day", StringType, true)))
    var df = jobContext.createDataFrame(input_LOGRDD, schema)    
    val result_df=df.na.fill(Map("platform"->"-","campaign_id"->"-","event_name"->"-"))
    val out_tempPath = "s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dmp_3s_event_source/day="+loadTime
   // val partitions = "day"
    result_df.write.mode(SaveMode.Overwrite).format("parquet").save(out_tempPath)
    if(loadTime.nonEmpty){
      val sql = "alter table dwh.dmp_3s_event_source add IF NOT EXISTS partition(day='" + loadTime + "')"
      jobContext.sql(sql)
    }
   spark.sparkContext.stop()
  }
  

  def parseRow(log: String,log_day: String) = {
    val p = log.split("\t")
    val uuid = p(1).split(" ")(0)
    val t_json = p(2)
    val json_data = JSON.parseObject(t_json)
    var mobvista_clickid = "-"
    var event_name = "-"
    var event_time = 0l
    var country = "-"
    var data_Info = "-"
    var device_id = "-"
    var event_day = "-"
    var campaign_id = json_data.getString("campaign_id")
    var platform = "-"
     val time_format = new SimpleDateFormat("yyyyMMdd")
    try {
      val query_json = json_data.getJSONObject("query")
      if (query_json != null && query_json.size() > 0) {
        mobvista_clickid = query_json.getString("mobvista_clickid")
        event_name = query_json.getString("event_name")
        event_time = query_json.getLong("event_time")
        platform = query_json.getString("mobvista_pl")
        var d = new Date()
        d.setTime(event_time * 1000)
        event_day = time_format.format(d)
        country = query_json.getString("mobvista_country")
        if (country == null || country.isEmpty()) {
          try {
            val ip_detail = json_data.getJSONObject("ip_detail")
            val i_country = ip_detail.getString("edge-two-letter-country")
            if (i_country.nonEmpty) {
              country = i_country
            } else {
              country = "-"
            }
          } catch {
            case e: Exception =>
             // println(e.getMessage)
              country = "-"
          } finally {
          }
        }
        country = country.toUpperCase()
       
        if (query_json.get("mobvista_gaid") != null&&query_json.getString("mobvista_gaid").nonEmpty&&query_json.getString("mobvista_gaid").split("-").length>3) {
          device_id = query_json.getString("mobvista_gaid")
        } else if (query_json.get("mobvista_idfa") != null&&query_json.getString("mobvista_idfa").nonEmpty&&query_json.getString("mobvista_idfa").split("-").length>3) {
          device_id = query_json.getString("mobvista_idfa")
        } else if (query_json.get("mobvista_adid") != null&&query_json.getString("mobvista_adid").nonEmpty&&query_json.getString("mobvista_adid").split("-").length>3) {
          device_id = query_json.getString("mobvista_adid")
        } else if (query_json.get("mobvista_devid") != null&&query_json.getString("mobvista_devid").nonEmpty&&query_json.getString("mobvista_devid").split("-").length>3) {
          device_id = query_json.getString("mobvista_devid")
        } else if (query_json.get("mobvista_imei") != null&&query_json.getString("mobvista_imei").nonEmpty&&query_json.getString("mobvista_imei").split("-").length>3) {
          device_id = query_json.getString("mobvista_imei")
        } else {
          try {
            val query_json_qd = json_data.getJSONObject("match_result").getJSONObject("data").getJSONObject("query")
            /**
             * mb_ifa
             * ios_idfa
             * mb_idfa
             * mb_gid
             * mb_gaid
             * ios_ifa
             * ifa
             * idfa
             * mb_adid
             * mb_devid
             *
             */
            if (query_json_qd.size() > 0) {
              if (query_json_qd.get("mb_ifa") != null&&query_json_qd.getString("mb_ifa").nonEmpty&&query_json_qd.getString("mb_ifa").split("-").length>3) {
                device_id = query_json_qd.getString("mb_ifa")
              } else if (query_json_qd.get("ios_idfa") != null&&query_json_qd.getString("ios_idfa").nonEmpty&&query_json_qd.getString("ios_idfa").split("-").length>3) {
                device_id = query_json_qd.getString("ios_idfa")
              } else if (query_json_qd.get("mb_idfa") != null&&query_json_qd.getString("mb_idfa").nonEmpty&&query_json_qd.getString("mb_idfa").split("-").length>3) {
                device_id = query_json_qd.getString("mb_idfa")
              } else if (query_json_qd.get("mb_gid") != null&&query_json_qd.getString("mb_gid").nonEmpty&&query_json_qd.getString("mb_gid").split("-").length>3) {
                device_id = query_json_qd.getString("mb_gid")
              } else if (query_json_qd.get("mb_gaid") != null&&query_json_qd.getString("mb_gaid").nonEmpty&&query_json_qd.getString("mb_gaid").split("-").length>3) {
                device_id = query_json_qd.getString("mb_gaid")
              } else if (query_json_qd.get("ios_ifa") != null&&query_json_qd.getString("ios_ifa").nonEmpty&&query_json_qd.getString("ios_ifa").split("-").length>3) {
                device_id = query_json_qd.getString("ios_ifa")
              } else if (query_json_qd.get("ifa") != null&&query_json_qd.getString("ifa").nonEmpty&&query_json_qd.getString("ifa").split("-").length>3) {
                device_id = query_json_qd.getString("ifa")
              } else if (query_json_qd.get("idfa") != null&&query_json_qd.getString("idfa").nonEmpty&&query_json_qd.getString("idfa").split("-").length>3) {
                device_id = query_json_qd.getString("idfa")
              } else if (query_json_qd.get("mb_adid") != null&&query_json_qd.getString("mb_adid").nonEmpty&&query_json_qd.getString("mb_adid").split("-").length>3) {
                device_id = query_json_qd.getString("mb_adid")
              } else if (query_json_qd.get("mb_devid") != null&&query_json_qd.getString("mb_devid").nonEmpty&&query_json_qd.getString("mb_devid").split("-").length>3) {
                device_id = query_json_qd.getString("mb_devid")
              }
            }
          } catch {
            case e: Exception =>
             // println(e.getMessage)
              country = "-"
          } finally {
          }
        }

      } else {
        data_Info = t_json
      }
      
      if(device_id.isEmpty()){
        data_Info = t_json
      }

    } catch {
      case e: Exception =>
       // println(e.getMessage)
        country = "-"
    } finally {
    }
    
   // val day=time_format.format(new Date())
    val day=log_day
    Row( //五字段一行
      device_id,
      event_day,
      event_name,
      mobvista_clickid,
      campaign_id,
      uuid,
      event_time,
      platform,
      country,
      data_Info,
      day)
  }

}