RTDmpMainDealV2.scala 5.38 KB
package mobvista.dmp.datasource.rtdmp

import java.net.URI

import com.datastax.spark.connector._
import mobvista.dmp.common.CommonSparkJob
import mobvista.dmp.datasource.rtdmp.Constant.AudienceInfo
import mobvista.dmp.util.{DateUtil, PropertyUtil}
import org.apache.commons.cli.{BasicParser, Options}
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.{SaveMode, SparkSession}
import org.apache.spark.storage.StorageLevel

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer

/**
 * @package: mobvista.dmp.datasource.rtdmp
 * @author: wangjf
 * @date: 2020/7/13
 * @time: 11:25 上午
 * @email: jinfeng.wang@mobvista.com
 * @phone: 152-1062-7698
 */
class RTDmpMainDealV2 extends CommonSparkJob with Serializable {

  def commandOptions(): Options = {
    val options = new Options()
    options.addOption("time", true, "time")
    options.addOption("input", true, "input")
    options.addOption("output", true, "output")
    options.addOption("coalesce", true, "coalesce")
    options.addOption("partition", true, "partition")
    options
  }

  override protected def run(args: Array[String]): Int = {

    val parser = new BasicParser()
    val options = commandOptions()
    val commandLine = parser.parse(options, args)
    val time = commandLine.getOptionValue("time").replace(".", " ")
    val input = commandLine.getOptionValue("input")
    val output = commandLine.getOptionValue("output")
    val coalesce = commandLine.getOptionValue("coalesce")
    val partition = commandLine.getOptionValue("partition")

    val spark: SparkSession = SparkSession.builder()
      .appName(s"RTDmpMainDeal.$time")
      .config("spark.rdd.compress", "true")
      .config("spark.io.compression.codec", "snappy")
      .config("spark.sql.orc.filterPushdown", "true")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .config("spark.cassandra.connection.host", PropertyUtil.getProperty("ip.properties", "aws_host"))
      .config("spark.cassandra.connection.port", "9042")
      .config("spark.cassandra.query.retry.count", "10")
      .config("spark.cassandra.connection.compression", "LZ4")
      .config("spark.cassandra.connection.remoteConnectionsPerExecutor", "2560")
      .config("spark.cassandra.connection.localConnectionsPerExecutor", "512")
      .config("spark.cassandra.input.consistency.level", "LOCAL_ONE")
      .config("spark.cassandra.output.consistency.level", "LOCAL_ONE")
      .config("spark.cassandra.input.fetch.sizeInRows", "1024")
      .config("spark.cassandra.input.split.sizeInMB", "5")
      .config("spark.cassandra.concurrent.reads", "1024")
      .config("spark.cassandra.output.concurrent.writes", "32")
      .config("spark.cassandra.output.batch.grouping.buffer.size", "1024")
      .config("spark.cassandra.output.batch.size.bytes", "1024")
      .config("spark.cassandra.connection.keepAliveMS", "60000")
      .getOrCreate()
    val sc = spark.sparkContext

    try {

      val expire_time = DateUtil.getDayByString(time, "yyyy-MM-dd HH", -7)

      val keyspace = "rtdmp"
      val tableName = "audience_info"
      val columns = SomeColumns("devid", "audience_data", "update_time")

      //  默认计算上个小时的数据
      val update_time_start = DateUtil.format(time + ":00:00", "yyyy-MM-dd HH:mm:ss")
      val update_time_end = DateUtil.format(time + ":59:59", "yyyy-MM-dd HH:mm:ss")
      val audience_date_utime_start = 1577811600L
      val audience_date_utime_end = 4100731200L

      val set = ServerUtil.request(update_time_start, update_time_end, audience_date_utime_start, audience_date_utime_end, 0, 0, 2)
        .retain((_, v) => v._2 == 1)
        .keySet

      println("audienceIds -->> " + set.mkString(","))

      import spark.implicits._

      val selectDF = spark.read.orc(input)
        .rdd.map(r => {
        (r.getAs[String]("devid"), r.getAs[String]("audience_ids"))
      })

      selectDF.persist(StorageLevel.MEMORY_AND_DISK_SER)

      val joinDF = selectDF.map(r => {
        r._1
      }).toDF("devid").rdd
        .repartition(partition.toInt)
        .joinWithCassandraTable(keyspace, tableName, SomeColumns("devid", "audience_data")).on(SomeColumns("devid"))
        .map(r => {
          (r._2.getString("devid"), r._2.getString("audience_data"))
        })

      //  FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output), true)

      //  joinDF.repartition(coalesce.toInt).saveAsTextFile(output, classOf[GzipCodec])

      val df = selectDF.leftOuterJoin(joinDF).mapPartitions(rs => {
        val res = new ArrayBuffer[AudienceInfo]()
        rs.foreach(r => {

          res.add(AudienceInfo(r._1, r._2._1, r._2._2.getOrElse("{}")))
        })
        res.iterator
      }).mapPartitions(v => new CustomIteratorAudienceInfoV2(v, time, expire_time, set))

      df.persist(StorageLevel.MEMORY_AND_DISK_SER)

      FileSystem.get(new URI(s"s3://mob-emr-test"), sc.hadoopConfiguration).delete(new Path(output), true)

      df.repartition(coalesce.toInt)
        .toDF
        .write
        .mode(SaveMode.Overwrite)
        .option("orc.compress", "zlib")
        .orc(output)

      df.saveToCassandra(keyspace, tableName, columns)

    } finally {
      if (sc != null) {
        sc.stop()
      }
      if (spark != null) {
        spark.stop()
      }
    }
    0
  }
}

object RTDmpMainDealV2 {
  def main(args: Array[String]): Unit = {
    new RTDmpMainDealV2().run(args)
  }
}