package mobvista.dmp.datasource.app_info_tag
import java.util.Properties

import com.alibaba.fastjson.{JSONArray, JSONObject}
import org.apache.spark.SparkConf
import org.apache.spark.sql.{SparkSession, _}
import org.apache.spark.sql.types._

object Publish_ios {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    val spark = SparkSession.builder().appName("JdbcOperation").config(conf)
      .getOrCreate()
    val loadTime = spark.conf.get("spark.app.loadTime")
    var year = loadTime.substring(0, 4)
    var month = loadTime.substring(4, 6)
    val day = loadTime.substring(6, 8)
    val output_path = spark.conf.get("spark.app.output_path")
    try {
      /**
       * ios 特殊，第一步先获得primary category 及 ID
       *
       *
       */
      val primary_category_rdd = spark.sparkContext
        .textFile("s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dim_category/dim_category.txt")
      val primary_category_dict = primary_category_rdd.map { line =>
        {
          val p = line.split("\t")
          val id = p(0)
          val store = p(1)
          val tag_code = p(5)
          var mv_first_tag_1 = ""
          try {
            mv_first_tag_1 = p(6)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_second_tag_1 = ""
          try {
            mv_second_tag_1 = p(7)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_first_tag_2 = ""
          try {
            mv_first_tag_2 = p(8)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_second_tag_2 = ""
          try {
            mv_second_tag_2 = p(9)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }

          Row(
            tag_code,
            mv_first_tag_1,
            mv_second_tag_1,
            mv_first_tag_2,
            mv_second_tag_2,
            store)
        }
      }.filter { x =>
        {
          var mv_first_tag_1 = x.getString(1)
          var mv_first_tag_2 = x.getString(3)

          var mv_second_tag_1 = x.getString(2)
          if (mv_second_tag_1 == null) {
            mv_second_tag_1 = ""
          }
          var mv_second_tag_2 = x.getString(4)
          if (mv_second_tag_2 == null) {
            mv_second_tag_2 = ""
          }
          if (mv_second_tag_1.isEmpty() && mv_second_tag_2.isEmpty()) {
            if (mv_first_tag_1.nonEmpty && mv_first_tag_2.isEmpty()) {
              true
            } else if (mv_first_tag_2.nonEmpty && mv_first_tag_1.isEmpty()) {
              true
            } else {
              false
            }
          } else {
            false
          }
        }
      }.map { p =>
        {
          var tag_code = p.getString(0)
          var mv_first_tag_1 = p.getString(1)
          var mv_first_tag_2 = p.getString(3)
          var store = p.getString(5)
          var primary_category = ""
          if (mv_first_tag_1.nonEmpty) {
            primary_category = mv_first_tag_1
          } else if (mv_first_tag_2.nonEmpty) {
            primary_category = mv_first_tag_2
          }
          Row(
            tag_code,
            primary_category,
            store)
        }
      }.distinct()

      var primary_dict_json = new JSONObject()
      primary_category_dict.collect().foreach { x =>
        {
          val code = x.getString(0)
          val primary_category = x.getString(1)
          val store = x.getString(2)
          primary_dict_json.put(primary_category + "_" + store, code)
        }
      }

      val primary_dict_json_brocast = spark.sparkContext.broadcast(primary_dict_json)

      /**
       * *
       *
       * 1:  第一步获得字典， 广播字典
       *
       *
       */
      val dim_category_rdd = spark.sparkContext
        .textFile("s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dim_category/dim_category.txt")
      val dict_rdd = dim_category_rdd.map { line =>
        {
          val primary_dict_json = primary_dict_json_brocast.value
          val p = line.split("\t")
          val id = p(0)
          val store = p(1)
          val tag_code = p(5)
          val primary_category = new scala.collection.mutable.ArrayBuffer[String]()
          val primary_category_id = new scala.collection.mutable.ArrayBuffer[String]()
          val category_list = new scala.collection.mutable.ArrayBuffer[String]()
          val category_list_id = new scala.collection.mutable.ArrayBuffer[String]()
          var mv_first_tag_1 = ""
          try {
            mv_first_tag_1 = p(6)
            var prim_id = primary_dict_json.getString(mv_first_tag_1.toString() + "_" + store)
            if (prim_id != null || prim_id.nonEmpty) {
              primary_category_id += "\"" + prim_id + "\""
              primary_category += "\"" + mv_first_tag_1.toString() + "\""
            }
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_second_tag_1 = ""
          try {
            mv_second_tag_1 = p(7)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_first_tag_2 = ""
          try {
            mv_first_tag_2 = p(8)
            var prim_id = primary_dict_json.getString(mv_first_tag_2.toString() + "_" + store)
            if (prim_id != null || prim_id.nonEmpty) {
              primary_category_id += "\"" + prim_id + "\""
              primary_category += "\"" + mv_first_tag_2.toString() + "\""
            }
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_second_tag_2 = ""
          try {
            mv_second_tag_2 = p(9)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }

          val tag_array = new JSONArray()
          if (mv_second_tag_1.nonEmpty || mv_first_tag_1.nonEmpty) {
            var tag = ""
            if (mv_second_tag_1.nonEmpty) {
              tag = mv_second_tag_1
            } else if (mv_first_tag_1.nonEmpty) {
              tag = mv_first_tag_1
            }
            /**
             * val tag_json = new JSONObject()
             * tag_json.put("code", tag_code)
             * tag_json.put("name", tag)
             * tag_array.add(tag_json)
             * *
             */
            category_list += "\"" + tag + "\""
            category_list_id += "\"" + tag_code + "\""
          }
          var tag = ""
          if (mv_second_tag_2.nonEmpty || mv_first_tag_2.nonEmpty) {
            var tag = ""
            if (mv_second_tag_2.nonEmpty) {
              tag = mv_second_tag_2
            } else if (mv_first_tag_2.nonEmpty) {
              tag = mv_first_tag_1
            }
            /**
             * val tag_json = new JSONObject()
             * tag_json.put("code", tag_code)
             * tag_json.put("name", tag)
             * tag_array.add(tag_json)
             * *
             */
            category_list += "\"" + tag + "\""
            category_list_id += "\"" + tag_code + "\""
          }
          Row(
            id,
            category_list_id.toSet.mkString(","),
            category_list.toSet.mkString(","),
            primary_category.toSet.mkString(","),
            primary_category_id.toSet.mkString(","))
        }
      }
      var dict_json = new JSONObject()
      dict_rdd.collect().foreach { x =>
        {
          val id = x.getString(0)
          val category_list_id = x.getString(1)
          val category_list = x.getString(2)
          val primary_category = x.getString(3)
          val primary_category_id = x.getString(4)
          var category_tag = "" + category_list_id + "##" + category_list + "##" + primary_category + "##" + primary_category_id
          dict_json.put(id, category_tag)
        }
      }
      val dict_json_brocast = spark.sparkContext.broadcast(dict_json)

      /**
       *
       * 2: 第二步， 处理publisher数据
       *
       */

      // val conf = new SparkConf().setAppName("JdbcOperation")
      val sqlContext = spark.sqlContext
      // val sqlContext = new SQLContext(sc)
      val properties = new Properties()
      properties.put("user", "adnro")
      properties.put("password", "YcM123glh")
      val url = "jdbc:mysql://adn-data-foronlinetest.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/mob_adn"
      val table_DF = sqlContext.read.jdbc(url, "publisher_channel", properties).select("channel_name", "platform", "confirmed_url", "sub_category_v2").filter("platform='2' and sub_category_v2!=''")
        .toDF("app_name", "platform", "confirmed_url", "category_list")
      val result_df = table_DF.rdd.map { p =>
        {
          val dict_json = dict_json_brocast.value
          val app_name = p.getString(0)
          var platform = p.getLong(1).toString()
          val confirmed_url = p.getString(2)
          val category_array = p.getString(3).split(",")
          var package_name = ""
          if (platform.equals("1")) {
            try {
              platform = "android"
              package_name = confirmed_url.substring(confirmed_url.indexOf("id=") + 3, confirmed_url.length())
              package_name = package_name.split("&")(0).split(";")(0)
            } catch {
              case e: Exception =>
                package_name = confirmed_url
                e.printStackTrace()
            } finally {
            }
          } else if (platform.equals("2")) {
            try {
              platform = "ios"
              if (confirmed_url.lastIndexOf("?") > 0) {
                package_name = confirmed_url.substring(0, confirmed_url.lastIndexOf("?"))
              } else {
                package_name = confirmed_url.substring(0, confirmed_url.length())
              }
              package_name = package_name.substring(package_name.lastIndexOf("/") + 1, package_name.length())
              
            } catch {
              case e: Exception =>
                package_name = confirmed_url
                e.printStackTrace()
            } finally {
            }
          }
          val primary_category = new scala.collection.mutable.ArrayBuffer[String]()
          val primary_category_id = new scala.collection.mutable.ArrayBuffer[String]()
          val category_list = new scala.collection.mutable.ArrayBuffer[String]()
          val category_list_id = new scala.collection.mutable.ArrayBuffer[String]()

          for (a <- 0 to category_array.length - 1) {
            val catrgory_id = category_array(a)
            if (dict_json.get(catrgory_id) != null) {
              val category_data = dict_json.getString(catrgory_id)
              val category_list_id_strs = category_data.split("##")(0).split(",")
              val category_list_strs = category_data.split("##")(1).split(",")
              val primary_category_strs = category_data.split("##")(2).split(",")
              val primary_category_id_strs = category_data.split("##")(3).split(",")

              for (i <- 0 until category_list_id_strs.length) {
                category_list_id += category_list_id_strs(i)
              }
              for (i <- 0 until category_list_strs.length) {
                category_list += category_list_strs(i)
              }

              for (i <- 0 until primary_category_id_strs.length) {
                primary_category_id += primary_category_id_strs(i)
                category_list_id += primary_category_id_strs(i)

              }

              for (i <- 0 until primary_category_strs.length) {
                primary_category += primary_category_strs(i)
                category_list += primary_category_strs(i)
              }
            }
          }
          if (package_name.startsWith("id")) {
            package_name = package_name.substring(2)
          }

          Row(
            package_name,
            app_name,
            "",
            "",
            "",
            "",
            "[" + primary_category.toSet.mkString(",") + "]",
            "[" + primary_category_id.toSet.mkString(",") + "]",
            "[" + category_list.toSet.mkString(",") + "]",
            "[" + category_list_id.toSet.mkString(",") + "]",
            "",
            confirmed_url,
            "")
        }
      }
      val schema = StructType(Array(
        StructField("package_name", StringType, true),
        StructField("app_name", StringType, true),
        StructField("file_size_bytes", StringType, true),
        StructField("support_language", StringType, true),
        StructField("bundle_id", StringType, true),
        StructField("release_date", StringType, true),
        StructField("primary_category", StringType, true),
        StructField("primary_category_id", StringType, true),
        StructField("category_list", StringType, true),
        StructField("category_id_list", StringType, true),
        StructField("user_rating_count", StringType, true),
        StructField("track_view_url", StringType, true),
        StructField("update_time", StringType, true)))
      var df = spark.createDataFrame(result_df, schema)
      df.coalesce(20).write.format("orc").mode(SaveMode.Overwrite).save(output_path)
    } catch {
      case e: Exception =>
        e.printStackTrace()
    } finally {
    }
    spark.sparkContext.stop()
  }

}