package mobvista.dmp.datasource.app_info_tag
import org.apache.spark.sql.SparkSession
object App_info_ios_3s extends Serializable {
  def main(args: Array[String]) {
    val spark = SparkSession.builder()
      .getOrCreate()
    val loadTime = spark.conf.get("spark.app.loadTime")
    var year = loadTime.substring(0, 4)
    var month = loadTime.substring(4, 6)
    val day = loadTime.substring(6, 8)
    val output_path = spark.conf.get("spark.app.output_path")
    try {
      import org.apache.spark.sql._
      import org.apache.spark.sql.types._
      //val ios_file = "s3://mob-emr-test/leo.liang/dmp_dummper/3s_ios.json.log"
      val ios_file = "s3://mob-emr-test/leo.liang/dmp_dummper/all_3s_ios.json.log"
      val result_rdd = spark.sparkContext.textFile(ios_file).map { x =>
        {
          import com.alibaba.fastjson.JSON
          val json = JSON.parseObject(x)
          var package_name = json.getString("package_name")
          val lang_data = json.getJSONObject("lang_data")
          var app_name_list = ""
          var lang_list = ""
          var language_array: scala.collection.mutable.Set[String] = scala.collection.mutable.Set()
          val itor = lang_data.entrySet().iterator()
          while (itor.hasNext()) {
            val obj=itor.next()
            val app_name = JSON.parseObject(obj.getValue.toString()).getString("app_name")
            if (!app_name_list.contains(app_name)) {
              app_name_list = app_name_list + "" + app_name + ","
            }
           language_array.add("\""+obj.getKey+"\"")
          }
          if (!app_name_list.isEmpty()) {
            app_name_list = app_name_list.substring(0, app_name_list.length() - 1)
          }
          
          lang_list="[" + language_array.mkString(",") + "]"
          val category = json.get("category")
          val cat_id = json.get("cat_id")
          val cat_ids_json = json.get("cat_ids_json")
          val cat_keys_json = json.get("cat_keys_json")
          val platfrom = "ios"
          val size_string = json.get("size_string")
          val size = json.get("size")
          val language_codes_ISO2A_json = json.get("language_codes_ISO2A_json")
          val i18n_lang_json = json.get("i18n_lang_json")
          val bundle_id = json.get("bundle_id")
          var release_time = json.getLong("release_time")
          val date = new java.util.Date()
          date.setTime(release_time * 1000l)
          val format1 = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'")
          val release_date = format1.format(date)
          val number_ratings = json.get("number_ratings")
          val number_ratings_current_version = json.get("number_ratings_current_version")
          val market_url = json.get("market_url")
          val market_update = json.getLong("market_update")
          val date2 = new java.util.Date()
          date2.setTime(market_update * 1000l)
          val format2 = new java.text.SimpleDateFormat("yyyyMMdd")
          val update_time = format2.format(date2)
          if (package_name.startsWith("id")) {
            package_name = package_name.substring(2)
          }
          val artist_id = json.get("artist_id")
          val artist_name = json.get("seller_name")
          val seller_name = artist_name
          val is_game_center_enabled = json.get("is_game_center_enabled")
          val content_advisory_rating = json.get("content_rating")
          val advisories = ""
          Row(
            package_name,
            app_name_list,
            size_string,
            lang_list,
            bundle_id,
            release_date,
            category,
            cat_id,
            cat_keys_json,
            cat_ids_json,
            number_ratings,
            market_url,
            update_time,
            artist_id,
            artist_name,
            seller_name,
            is_game_center_enabled,
            content_advisory_rating,
            advisories
          )
        }
      }
      val schema = StructType(Array(
        StructField("package_name", StringType, true),
        StructField("app_name", StringType, true),
        StructField("file_size_bytes", StringType, true),
        StructField("support_language", StringType, true),
        StructField("bundle_id", StringType, true),
        StructField("release_date", StringType, true),
        StructField("primary_category", StringType, true),
        StructField("primary_category_id", StringType, true),
        StructField("category_list", StringType, true),
        StructField("category_id_list", StringType, true),
        StructField("user_rating_count", StringType, true),
        StructField("track_view_url", StringType, true),
        StructField("update_time", StringType, true),
        StructField("artist_id", StringType, true),
        StructField("artist_name", StringType, true),
        StructField("seller_name", StringType, true),
        StructField("is_game_center_enabled", StringType, true),
        StructField("content_advisory_rating", StringType, true),
        StructField("advisories", StringType, true)
      ))
      var df = spark.createDataFrame(result_rdd, schema)
      df.coalesce(100).write.format("orc").mode(SaveMode.Overwrite).save(output_path)

    } catch {
      case e: Exception =>
        e.printStackTrace()
    } finally {0
    }

    spark.sparkContext.stop()
  }

}