You need to sign in or sign up before continuing.
ByteDanceDaily.scala 3.09 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
package mobvista.dmp.datasource.bytedance

import java.net.URI
import java.util

import mobvista.dmp.common.CommonSparkJob
import mobvista.prd.datasource.util.GsonUtil
import org.apache.commons.cli.Options
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{SaveMode, SparkSession}

import scala.collection.JavaConversions._

/**
  * @package: mobvista.dmp.datasource.bytedance
  * @author: wangjf
  * @date: 2019/3/4
  * @time: 下午1:44
  * @email: jinfeng.wang@mobvista.com
  * @phone: 152-1062-7698
  */
class ByteDanceDaily extends CommonSparkJob {

  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("input", true, "[must] input")
    options.addOption("output", true, "[must] output")
    options.addOption("coalesce", true, "[must] coalesce")
    options
  }

  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      return -1
    } else {
      printOptions(commandLine)
    }

    val input = commandLine.getOptionValue("input")
    val output = commandLine.getOptionValue("output")
    val coalesce = commandLine.getOptionValue("coalesce")

    val spark = SparkSession.builder()
      .appName("ByteDanceDaily")
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "snappy")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()
    val sc = spark.sparkContext

    FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output), true)
    try {
      val rdd = sc.textFile(input).map(r => {
        val map = Map("1128" -> "1142110895")
        val json = GsonUtil.String2JsonObject(r)
        val apps = json.get("val").getAsJsonObject.get("toutiaoDmpInstallApps").toString
        val device_id = json.get("key").getAsJsonObject.get("idfaGaid").getAsString.toUpperCase
        val device_type = "idfa"
        val platform = "ios"
        val country = "CN"
        val set = new util.HashSet[String]()
        if (apps.startsWith("[") && apps.endsWith("]")) {
          val app = apps.substring(1, apps.length - 1).split(",")
          app.foreach(r => {
            if (map.keySet.contains(r)) {
              set.add(map(r))
            }
          })
        }
        var package_array = ""
        set.iterator().foreach(s => {
          package_array += s + "#"
        })
        ByteDance(device_id, device_type, platform, package_array.substring(0, package_array.length - 1), country)
      })
      import spark.implicits._
      rdd.toDF.coalesce(coalesce.toInt).write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .orc(output)
    } finally {
      sc.stop()
      spark.stop()
    }
    0
  }
}

object ByteDanceDaily {
  def main(args: Array[String]): Unit = {
    new ByteDanceDaily().run(args)
  }
}