GaSubtractInterest.scala 2.94 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
package mobvista.dmp.datasource.ga

import java.net.URI

import mobvista.dmp.common.CommonSparkJob
import org.apache.commons.cli.Options
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{SaveMode, SparkSession}
import org.apache.spark.sql.expressions.UserDefinedFunction
import org.apache.spark.sql.functions._

/**
  * @package: mobvista.dmp.datasource.ga
  * @author: wangjf
  * @date: 2018/12/19
  * @time: 下午2:45
  * @email: jinfeng.wang@mobvista.com
  * @phone: 152-1062-7698
  */
class GaSubtractInterest extends CommonSparkJob with Serializable {
  override protected def run(args: Array[String]): Int = {
    val commandLine = commParser.parse(options, args)
    if (!checkMustOption(commandLine)) {
      printUsage(options)
      printOptions(commandLine)
      return 1
    } else {
      printOptions(commandLine)
    }

    val date = commandLine.getOptionValue("date")
    val ga_date = commandLine.getOptionValue("ga_date")
    val output = commandLine.getOptionValue("output")

    val spark = SparkSession.builder()
      .appName("GaSubtractInterest")
      .config("spark.rdd.compress", "true")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.sql.warehouse.dir", "s3://mob-emr-test/spark-warehouse")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .enableHiveSupport()
      .getOrCreate()
    val sc = spark.sparkContext

    FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output), true)

    try {
      val ga_device_total_sql = Constant.ods_ga_active_total_sql.replace("@ga_date", ga_date)
      val ga_device_total_df = spark.sql(ga_device_total_sql)
      val dm_interest_tag_sql = Constant.dm_interest_tag_sql.replace("@date", date)
      val dm_interest_tag_df = spark.sql(dm_interest_tag_sql)
      ga_device_total_df.except(dm_interest_tag_df)
        .withColumn("device_type", selectType(col("platform")))
        .withColumn("tags", selectTag(col("platform")))
        .select(col = "device_id", cols = "device_type", "platform", "tags")
        .write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .orc(output)
    } finally {
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }

  def selectType: UserDefinedFunction = udf((platform: String) =>
    if ("ios".equals(platform)) "idfa"
    else "gaid"
  )

  def selectTag: UserDefinedFunction = udf((platform: String) =>
    if ("ios".equals(platform)) "[{\"tag\":[{\"id\":\"34\",\"1\":\"Games\"}]}]"
    else "[{\"tag\":[{\"id\":\"69\",\"1\":\"Games\"}]}]"
  )

  override protected def buildOptions(): Options = {
    val options = new Options
    options.addOption("date", true, "[must] date")
    options.addOption("ga_date", true, "[must] ga_date")
    options.addOption("output", true, "[must] output")
    options
  }
}

object GaSubtractInterest {
  def main(args: Array[String]): Unit = {
    new GaSubtractInterest().run(args)
  }
}