package mobvista.dmp.datasource.app_info_tag
import java.util.Properties

import com.alibaba.fastjson.{JSONArray, JSONObject}
import org.apache.spark.SparkConf
import org.apache.spark.sql.{SparkSession, _}
import org.apache.spark.sql.types._
/**
 *
 * mysql -h adn-data-foronlinetest.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -u adnro  -pYcM123glh
 *
 */

object Publish_adr {
  def main2(args: Array[String]): Unit = {
    var platform = "2"
    val confirmed_url = "https://itunes.apple.com/us/app/photo-grid-photo-collage-maker-photo-editor/id543577420?mt=8"
    var package_name = ""
    if (platform.equals("1")) {
      platform = "android"
      package_name = confirmed_url.substring(confirmed_url.indexOf("id=") + 3, confirmed_url.length())
      package_name = package_name.split("&")(0).split(";")(0)
    } else if (platform.equals("2")) {
      platform = "ios"
      package_name = confirmed_url.substring(0, confirmed_url.lastIndexOf("?"))
      package_name = package_name.substring(package_name.lastIndexOf("/") + 1, package_name.length())
    }

    println(confirmed_url)
    println(platform)
    println(package_name)

  }

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    val spark = SparkSession.builder().appName("JdbcOperation").config(conf)
      .getOrCreate()

    val loadTime = spark.conf.get("spark.app.loadTime")
    var year = loadTime.substring(0, 4)
    var month = loadTime.substring(4, 6)
    val day = loadTime.substring(6, 8)
    val output_path = spark.conf.get("spark.app.output_path")
    try {
      /**
       * *
       *
       * 1:  第一步获得字典， 广播字典
       *
       *
       */
      val dim_category_rdd = spark.sparkContext
        .textFile("s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dim_category/dim_category.txt")
      val dict_rdd = dim_category_rdd.map { line =>
        {
          val p = line.split("\t")
          val id = p(0)
          val tag_code = p(5)
          var mv_first_tag_1 = ""
          try {
            mv_first_tag_1 = p(6)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_second_tag_1 = ""
          try {
            mv_second_tag_1 = p(7)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_first_tag_2 = ""
          try {
            mv_first_tag_2 = p(8)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }
          var mv_second_tag_2 = ""
          try {
            mv_second_tag_2 = p(9)
          } catch {
            case e: Exception =>
              e.printStackTrace()
          } finally {
          }

          val tag_array = new JSONArray()
          if (mv_second_tag_1.nonEmpty || mv_first_tag_1.nonEmpty) {
            var tag = ""
            if (mv_second_tag_1.nonEmpty) {
              tag = mv_second_tag_1
            } else if (mv_first_tag_1.nonEmpty) {
              tag = mv_first_tag_1
            }
            val tag_json = new JSONObject()
            tag_json.put("code", tag_code)
            tag_json.put("name", tag)
            tag_array.add(tag_json)
          }

          if (mv_second_tag_2.nonEmpty || mv_first_tag_2.nonEmpty) {
            var tag = ""
            if (mv_second_tag_2.nonEmpty) {
              tag = mv_second_tag_2
            } else if (mv_first_tag_2.nonEmpty) {
              tag = mv_first_tag_1
            }
            val tag_json = new JSONObject()
            tag_json.put("code", tag_code)
            tag_json.put("name", tag)
            tag_array.add(tag_json)
          }
          Row(
            id,
            tag_array.toString())
        }
      }
      println("*******start******")
      dict_rdd.take(10).foreach { x => println(x) }
      println("*******end******")
      var dict_json = new JSONObject()
      dict_rdd.collect().foreach { x =>
        {
          val id = x.getString(0)
          val tag = x(1)
          // val tag = x.getJ
          /**
           * val tag_json = new JSONObject()
           * tag_json.put("code", tag_code)
           * tag_json.put("name", tag)
           * **
           */
          dict_json.put(id, tag)
        }
      }
      val dict_json_brocast = spark.sparkContext.broadcast(dict_json)

      /**
       *
       * 2: 第二步， 处理publisher数据
       *
       */

      val sqlContext = spark.sqlContext
      val properties = new Properties()
      properties.put("user", "adnro")
      properties.put("password", "YcM123glh")
      val url = "jdbc:mysql://adn-data-foronlinetest.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/mob_adn"
      val table_DF = sqlContext.read.jdbc(url, "publisher_channel", properties).select("channel_name", "platform", "confirmed_url", "sub_category_v2").filter("platform='1' and sub_category_v2!=''")
        .toDF("app_name", "platform", "confirmed_url", "category_list")
      val result_rdd = table_DF.rdd.map { p =>
        {
          val dict_json = dict_json_brocast.value
          val app_name = p.getString(0)
          var platform = p.getLong(1).toString()
          val confirmed_url = p.getString(2)
          val category_array = p.getString(3).split(",")
          var package_name = ""
          if (platform.equals("1")) {
            try {
              platform = "android"
              package_name = confirmed_url.substring(confirmed_url.indexOf("id=") + 3, confirmed_url.length())
              package_name = package_name.split("&")(0).split(";")(0)
            } catch {
              case e: Exception =>
                package_name = confirmed_url
                e.printStackTrace()
            } finally {
            }
          } else if (platform.equals("2")) {
            try {
              platform = "ios"
              package_name = confirmed_url.substring(0, confirmed_url.lastIndexOf("?"))
              package_name = package_name.substring(package_name.lastIndexOf("/") + 1, package_name.length())
            } catch {
              case e: Exception =>
                package_name = confirmed_url
                e.printStackTrace()
            } finally {
            }
          }
          val category_list = new JSONArray()
          for (a <- 0 to category_array.length - 1) {
            val catrgory_id = category_array(a)
            if (dict_json.get(catrgory_id) != null) {
              val json_array_data = dict_json.getJSONArray(catrgory_id)
              for (i <- 0 until json_array_data.size()) {
                category_list.add(json_array_data.getJSONObject(i))
              }
            }
          }

          Row(
            package_name,
            app_name,
            category_list.toString(),
            "",
            "",
            confirmed_url,
            "")
        }
      }

      val schema = StructType(Array(
        StructField("package_name", StringType, true),
        StructField("app_name", StringType, true),
        StructField("category_list", StringType, true),
        StructField("user_rating_count", StringType, true),
        StructField("download", StringType, true),
        StructField("track_view_url", StringType, true),
        StructField("update_time", StringType, true)))

      var df = spark.createDataFrame(result_rdd, schema)
      df.coalesce(20).write.format("orc").mode(SaveMode.Overwrite).save(output_path)
    } catch {
      case e: Exception =>
        e.printStackTrace()
    } finally {
    }
    spark.sparkContext.stop()
  }

}