Skip to content

M
mobvista-dmp
Commit 1dac1a5f by WangJinfeng
Options

init id_mapping

parent 1cc7ff0d
Showing with 785 additions and 87 deletions
azkaban/dmp_env.sh
......@@ -437,6 +437,8 @@ DSP_DEVICE_MAPPING="s3://mob-emr-test/dataplatform/DataWareHouse/data/dsp/device
ID_MAPPING="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwd/dwd_device_ids_inc_daily"
ADS_DEVICE_ID_MAPPING="s3://mob-emr-test/dataplatform/DataWareHouse/data/ads/ads_device_id_mapping"
JAR=./DMP.jar
# 检查_SUCCESS文件,若不存在则循环检测
......
azkaban/id_mapping/adn_request_id_mapping.sh
......@@ -19,11 +19,10 @@ spark-submit --class mobvista.dmp.datasource.id_mapping.AdnRequest \
--name "EtlDeviceIdDaily.$BUSINESS.$LOG_TIME" \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.network.timeout=720s \
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.default.parallelism=3000 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 8g --executor-cores 5 --num-executors 200 \
../${JAR} -date ${LOG_TIME} -business ${BUSINESS} -output ${OUTPUT_PATH} -coalesce 1000
--conf spark.sql.shuffle.partitions=5000 \
--conf spark.default.parallelism=5000 \
--master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 8g --executor-cores 5 --num-executors 200 \
../${JAR} -date ${LOG_TIME} -business ${BUSINESS} -output ${OUTPUT_PATH} -coalesce 2000
if [ $? -ne 0 ]; then
exit 255
......
azkaban/id_mapping/dsp_id_mapping.sh
......@@ -19,11 +19,10 @@ spark-submit --class mobvista.dmp.datasource.id_mapping.DspReq \
--name "EtlDeviceIdDaily.$BUSINESS.$LOG_TIME" \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.network.timeout=720s \
--conf spark.sql.shuffle.partitions=5000 \
--conf spark.default.parallelism=5000 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 12g --executor-cores 5 --num-executors 200 \
../${JAR} -date ${LOG_TIME} -business ${BUSINESS} -output ${OUTPUT_PATH} -coalesce 1000
--conf spark.sql.shuffle.partitions=10000 \
--conf spark.default.parallelism=10000 \
--master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 8g --executor-cores 5 --num-executors 200 \
../${JAR} -date ${LOG_TIME} -business ${BUSINESS} -output ${OUTPUT_PATH} -coalesce 2000
if [ $? -ne 0 ]; then
exit 255
......
azkaban/id_mapping/id_mapping.job
type=command
command=echo "ID Mapping End!"
\ No newline at end of file
command=sh -x id_mapping.sh
\ No newline at end of file
azkaban/id_mapping/id_mapping.sh 0 → 100644
#! /bin/bash
source ../dmp_env.sh
LOG_TIME=$(date +%Y-%m-%d -d "-1 day $ScheduleTime")
date_path=$(date +'%Y/%m/%d' -d "-1 day $ScheduleTime")
ADN_REQUEST_INPUT_PATH=${ID_MAPPING}/${date_path}/adn_request
DSP_INPUT_PATH=${ID_MAPPING}/${date_path}/dsp_req
check_await "${ADN_REQUEST_INPUT_PATH}/_SUCCESS"
check_await "${DSP_INPUT_PATH}/_SUCCESS"
OUTPUT_PATH=${ADS_DEVICE_ID_MAPPING}/${date_path}
country="US"
platform="ios"
spark-submit --class mobvista.dmp.datasource.id_mapping.IDMappingGraphx \
--name "IDMappingGraphx.${LOG_TIME}.${country}.${platform}" \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.network.timeout=720s \
--conf spark.sql.shuffle.partitions=5000 \
--conf spark.default.parallelism=5000 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 8g --executor-cores 5 --num-executors 100 \
../${JAR} -date ${LOG_TIME} -country ${country} -platform ${platform} -output ${OUTPUT_PATH} -coalesce 500
if [ $? -ne 0 ]; then
exit 255
fi
\ No newline at end of file
pom.xml
......@@ -92,6 +92,11 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-graphx_2.12</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>${joda.version}</version>
......@@ -126,6 +131,11 @@
<artifactId>orc-mapreduce</artifactId>
<version>${orc.version}</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
<version>2.8</version>
</dependency>
</dependencies>
</dependencyManagement>
......@@ -215,6 +225,10 @@
<artifactId>spark-yarn_2.12</artifactId>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-graphx_2.12</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
......@@ -293,6 +307,15 @@
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.437</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-native_2.12</artifactId>
<version>4.0.0</version>
</dependency>
</dependencies>
<build>
......
src/main/resources/config.properties
......@@ -101,9 +101,10 @@ ali_activation.package_name=com.taobao.foractivation.172393,com.taobao.foractiva
com.taobao.foractivation.260951,com.taobao.foractivation.260951_oaid,202005260951,com.taobao.foractivation.261865,com.taobao.foractivation.261865_oaid,\
202005261865
dsp_req.package_name=com.taobao.taobao_oppo,com.eg.android.AlipayGphone_oppo,com.ucmobile_oppo,com.qiyi.video_oppo,com.taobao.taobao_notinstall_oppo,\
# 2021-12-08 14:03:45 ?? com.taobao.idlefish_oppo,com.qiyi.video_oppo
dsp_req.package_name=com.taobao.taobao_oppo,com.eg.android.AlipayGphone_oppo,com.ucmobile_oppo,com.taobao.taobao_notinstall_oppo,\
com.eg.android.AlipayGphone_bes,com.youku.phone_notinstall_oppo,com.sankuai.meituan_oppo,com.meituan.itakeaway_oppo,com.taobao.idlefish_bes,\
com.taobao.idlefish_oppo,com.UCMobile_bes,com.taobao.taobao_bes,com.tencent.news_fromtencent,com.taobao.taobao_iqiyi,com.taobao.taobao,com.UCMobile_iqiyi,\
com.UCMobile_bes,com.taobao.taobao_bes,com.tencent.news_fromtencent,com.taobao.taobao_iqiyi,com.taobao.taobao,com.UCMobile_iqiyi,\
com.UCMobile,com.eg.android.AlipayGphone_iqiyi,com.eg.android.AlipayGphone,com.taobao.idlefish_iqiyi,com.taobao.idlefish,com.sankuai.meituan_iqiyi,com.sankuai.meituan,\
com.tencent.news_iqiyi,com.tencent.news,134037632320210617,44094811020210617,147250281920210617,com.kuaishou.nebula_fromkuaishou,com.smile.gifmaker_fromkuaishou,\
com.sankuai.meituan_bes
......
src/main/scala/mobvista/dmp/common/MobvistaConstant.scala
......@@ -321,7 +321,7 @@ object MobvistaConstant {
// 全0
val allZero = "00000000-0000-0000-0000-000000000000"
// IMEI
val imeiPtn = "^([0-9]{15,17})$"
val imeiPtn = "^([0-9]{14,17})$"
// 14~16位连续多位相同字符，非法IMEI过滤
val imeiPtnAll = """^([0-9])\1{14,16}"""
// androidId
......@@ -334,6 +334,8 @@ object MobvistaConstant {
val umd5Ptn = """^([0-9A-Za-z])\1{29,31}"""
// OAID
val oaidPtb = """^[0-9A-Za-z-]{16,64}$"""
// IP
val ipPtn = """^(25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])){3}$"""
def checkDeviceId(device_id: String): Boolean = {
StringUtils.isNotBlank(device_id) &&
......
src/main/scala/mobvista/dmp/datasource/dsp/DspOrgLogEtlHoursDemo.scala
package mobvista.dmp.datasource.dsp
import java.net.URI
import mobvista.dmp.common.{CommonMapReduce, CommonSparkJob}
import mobvista.dmp.util.{MD5Util, MRUtils}
import mobvista.prd.datasource.util.GsonUtil
......@@ -13,6 +11,7 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Row, SaveMode, SparkSession}
import org.apache.spark.storage.StorageLevel
import java.net.URI
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
......@@ -59,7 +58,7 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
select idfa,googleadid gaid,
| os platform,countrycode country,
| deviceip ip,gender,yob birthday,
| make,model,osv osversion,
| make,model,osv osversion,ext6 ua,
| appid packagename,ext3,exchanges,ext5 exitid,`time`,
| body json_msg,rg region
| from adn_dsp.log_adn_dsp_request_orc_hour where concat(yr,mt,dt,hh)='${yyyymmddhh}'
......@@ -69,15 +68,15 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
.map(parseMapData).toDF()
.persist(StorageLevel.MEMORY_AND_DISK_SER)
df.select("idfa", "gaid", "platform", "country", "ip", "gender", "birthday", "maker", "model", "osVersion", "packageName", "exitId", "time", "geoInfo", "longitude", "latitude", "segment", "dealerid", "exchanges", "region")
.toDF("idfa", "gaid", "platform", "country", "ip", "gender", "birthday", "maker", "model", "osversion", "packagename", "exitid", "time", "geoinfo", "longitude", "latitude", "segment", "dealerid", "exchanges", "region")
df.select("idfa", "gaid", "platform", "country", "ip", "gender", "birthday", "maker", "model", "osVersion", "ua", "packageName", "exitId", "time", "geoInfo", "longitude", "latitude", "segment", "dealerid", "exchanges", "region")
.toDF("idfa", "gaid", "platform", "country", "ip", "gender", "birthday", "maker", "model", "osversion", "ua", "packagename", "exitid", "time", "geoinfo", "longitude", "latitude", "segment", "dealerid", "exchanges", "region")
.createOrReplaceTempView("dsp_org_etl_hours")
val etl_sql =
"""
|select idfa,gaid,platform,country,ip,gender,birthday,maker,model,osversion, concat_ws('#',collect_set(packagename)) packagename,exitid,max(`time`) datetime,segment,dealerid,exchanges,concat_ws('#',collect_set(region)) region
|select idfa,gaid,platform,country,ip,gender,birthday,maker,model,osversion,ua,concat_ws('#',collect_set(packagename)) packagename,exitid,max(`time`) datetime,segment,dealerid,exchanges,concat_ws('#',collect_set(region)) region
|from dsp_org_etl_hours
|group by idfa,gaid,platform,country,ip,gender,birthday,maker,model,osversion,exitid,segment,dealerid,exchanges
|group by idfa,gaid,platform,country,ip,gender,birthday,maker,model,osversion,ua,exitid,segment,dealerid,exchanges
""".stripMargin
spark.sql(etl_sql).repartition(coalesce.toInt)
.write
......@@ -135,6 +134,7 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
val maker = row.getAs[String]("make")
val model = row.getAs[String]("model")
val osVersion = row.getAs[String]("osversion")
val ua = row.getAs[String]("ua")
val country = row.getAs[String]("country")
val birthday = row.getAs[String]("birthday")
val gender = row.getAs[String]("gender")
......@@ -203,9 +203,8 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
}
DspReqVODemo(idfa, gaid, platform, country, ip, gender, birthday, maker, model, osVersion,
DspReqVODemo(idfa, gaid, platform, country, ip, gender, birthday, maker, model, osVersion, ua,
packageName, exitId, time, geoInfo, longitude, latitude, segment, dealerid, exchanges, region)
}
override protected def buildOptions(): Options = {
......
src/main/scala/mobvista/dmp/datasource/dsp/DspReqVODemo.scala
package mobvista.dmp.datasource.dsp
case class DspReqVODemo(idfa:String,gaid: String,
platform: String,
country: String,
ip: String,
gender: String,
birthday: String,
maker: String,
model: String,
osVersion: String,
packageName: String,
exitId: String,
time: String,
geoInfo: String,
longitude: String,
latitude: String,
segment: String,
dealerid:String,
exchanges:String,
case class DspReqVODemo(idfa: String,
gaid: String,
platform: String,
country: String,
ip: String,
gender: String,
birthday: String,
maker: String,
model: String,
osVersion: String,
ua: String,
packageName: String,
exitId: String,
time: String,
geoInfo: String,
longitude: String,
latitude: String,
segment: String,
dealerid: String,
exchanges: String,
region: String)
src/main/scala/mobvista/dmp/datasource/id_mapping/AdnRequest.scala
package mobvista.dmp.datasource.id_mapping
import mobvista.dmp.datasource.id_mapping.Constant.{getDevId, parseUA}
import mobvista.dmp.datasource.id_mapping.Constant.{getDevId, parseUA, process}
import org.apache.commons.lang3.StringUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SparkSession}
......@@ -20,22 +20,18 @@ class AdnRequest extends EtlDeviceIdDaily {
// DWD
// val sql = Constant.adn_request_sql.replace("@date", date)
// ODS
val sql = Constant.adn_request_sql_v2.replace("@date", date)
val rdd = spark.sql(sql).coalesce(2000).rdd.map(row => {
val sql = Constant.adn_request_sql_v3.replace("@date", date)
val rdd = spark.sql(sql).coalesce(5000).rdd.map(row => {
val gaid = row.getAs[String]("gaid")
val idfa = row.getAs[String]("idfa")
val imei = row.getAs[String]("imei")
val androidId = row.getAs[String]("androidid")
val extSysid = row.getAs[String]("extsysid")
var platform = row.getAs[String]("platform")
platform = if (StringUtils.isNotBlank(platform)) {
platform.toLowerCase()
} else {
""
}
val platform = row.getAs[String]("platform")
val oaid = row.getAs[String]("oaid")
val idfv = row.getAs[String]("idfv")
var country = row.getAs[String]("country")
val pkg_name = row.getAs[String]("pkg_name")
val country = row.getAs[String]("country")
val ip = row.getAs[String]("ip")
val ua = row.getAs[String]("ua")
val brand = row.getAs[String]("brand")
......@@ -43,11 +39,7 @@ class AdnRequest extends EtlDeviceIdDaily {
val os_version = row.getAs[String]("os_version")
val osv_upt = row.getAs[String]("osv_upt")
val upt = row.getAs[String]("upt")
country = if (StringUtils.isNotBlank(country)) {
country.toUpperCase()
} else {
""
}
val network_type = row.getAs[String]("network_type")
var sysId = ""
var bkupId = ""
if (StringUtils.isNotBlank(extSysid)) {
......@@ -57,19 +49,14 @@ class AdnRequest extends EtlDeviceIdDaily {
} else {
""
}
bkupId = if (arr.length == 2 && StringUtils.isNotBlank(arr(1))) {
arr(1)
} else {
""
}
}
val rw = if ("ios".equalsIgnoreCase(platform)) {
Row(idfa, idfv, sysId, bkupId, "", "", country, ip, ua, brand, model, os_version, osv_upt, upt)
} else {
Row(imei, androidId, oaid, gaid, sysId, bkupId, "", "", country, ip, ua, brand, model, os_version, osv_upt, upt)
}
(platform, rw)
process(idfa, idfv, pkg_name, imei, androidId, oaid, gaid, sysId, bkupId, country, ip, ua, brand, model, os_version,
osv_upt, upt, network_type, platform)
})
rdd
}
......
src/main/scala/mobvista/dmp/datasource/id_mapping/Constant.scala
package mobvista.dmp.datasource.id_mapping
import mobvista.dmp.common.MobvistaConstant
import org.apache.commons.lang.StringUtils
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import mobvista.dmp.common.MobvistaConstant._
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import java.net.URLDecoder
......@@ -15,12 +16,47 @@ import java.net.URLDecoder
*/
object Constant {
case class IosTab(idfa: String, idfv: String, sysid: String, bkupid: String, xwho: String, user_id: String)
/**
*
* @param idfa IDFA
* @param sysId SYSID
* @param idfv_bundle IDFV+bundleID
* @param bmosv_osv_upt brand+model+os+os_version+osv_upt
* @param bmosv_upt brand+model+os+os_version+upt
* @param bmosv_ipua_bundle brand+model+os+os_version+ip+ua+bundle
* @param xwho xwho 热云账号ID
* @param user_id user_id 热云账号ID
* @param bkupid BKUPID
* @param cnt 一天内同一事件出现的频次
*/
case class IosVert(idfa: String, sysId: String, idfv_bundle: String, bmosv_osv_upt: String, bmosv_upt: String,
bmosv_ipua_bundle: String, xwho: String, user_id: String, bkupid: String, cnt: Int) extends Serializable
val iosVertSchema: StructType = StructType(Array(
StructField("idfa", StringType),
StructField("sysid", StringType),
StructField("idfv_bundle", StringType),
StructField("bmosv_osv_upt", StringType),
StructField("bmosv_upt", StringType),
StructField("bmosv_ipua_bundle", StringType),
StructField("xwho", StringType),
StructField("user_id", StringType),
StructField("bkupid", StringType),
StructField("cnt", LongType)
))
val iosIDSet = Set("idfa", "sysid", "xwho", "user_id", "idfv_bundle", "bmosv_osv_upt", "bmosv_upt", "bmosv_ipua_bundle", "bkupid")
val iosMainIDSet = Set("idfa", "sysid", "xwho", "user_id")
case class IosTab(idfa: String, idfv: String, pkg_name: String, sysid: String, bkupid: String, xwho: String, user_id: String,
country: String, ip: String, ua: String, brand: String, model: String, os_version: String, osv_upt: String, upt: String)
extends java.io.Serializable
val iosSchema: StructType = StructType(Array(
StructField("idfa", StringType),
StructField("idfv", StringType),
StructField("pkg_name", StringType),
StructField("sysid", StringType),
StructField("bkupid", StringType),
StructField("xwho", StringType),
......@@ -32,15 +68,71 @@ object Constant {
StructField("model", StringType), // 型号
StructField("os_version", StringType), // 操作系统版本
StructField("osv_upt", StringType), // 操作系统更新时间
StructField("upt", StringType) // 开机时间
StructField("upt", StringType), // 开机时间
StructField("event_type", StringType), // 事件类型
StructField("network_type", StringType) // 网络类型，用于判断API/SDK
))
case class AdrTab(imei: String, android_id: String, oaid: String, gaid: String, sysid: String, bkupid: String, xwho: String, user_id: String)
/**
*
* @param imei IMEI
* @param gaid GAID
* @param oaid OAID
* @param sysId SYSID
* @param android_pkg androidId+bundleID
* @param bmosv_upt brand+model+os+os_version+upt
* @param bmosv_ipua_pkg brand+model+os+os_version+ip+ua+pkg
* @param xwho xwho 热云账号ID
* @param user_id user_id 热云账号ID
* @param bkupid BKUPID
* @param cnt 一天内同一事件出现的频次
*/
case class AdrVert(imei: String, gaid: String, oaid: String, sysId: String, android_pkg: String, bmosv_upt: String,
bmosv_ipua_pkg: String, xwho: String, user_id: String, bkupid: String, cnt: Int) extends Serializable
val adrCNVertSchema: StructType = StructType(Array(
StructField("imei", StringType),
StructField("oaid", StringType),
StructField("gaid", StringType),
StructField("sysid", StringType),
StructField("android_pkg", StringType),
StructField("bmosv_upt", StringType),
StructField("bmosv_ipua_pkg", StringType),
StructField("xwho", StringType),
StructField("user_id", StringType),
StructField("bkupid", StringType),
StructField("cnt", LongType)
))
val adrVertSchema: StructType = StructType(Array(
StructField("gaid", StringType),
StructField("imei", StringType),
StructField("oaid", StringType),
StructField("sysid", StringType),
StructField("android_pkg", StringType),
StructField("bmosv_upt", StringType),
StructField("bmosv_ipua_pkg", StringType),
StructField("xwho", StringType),
StructField("user_id", StringType),
StructField("bkupid", StringType),
StructField("cnt", IntegerType)
))
val androidCNIDSet = Set("imei", "oaid", "gaid", "sysid", "xwho", "user_id", "android_pkg", "bmosv_upt", "bmosv_ipua_pkg", "bkupid")
val androidIDSet = Set("gaid", "imei", "oaid", "sysid", "xwho", "user_id", "android_pkg", "bmosv_upt", "bmosv_ipua_pkg", "bkupid")
val androidMainIDSet = Set("imei", "gaid", "oaid", "sysid", "xwho", "user_id")
case class AdrTab(imei: String, android_id: String, pkg_name: String, oaid: String, gaid: String, sysid: String, bkupid: String,
xwho: String, user_id: String, country: String, ip: String, ua: String, brand: String, model: String,
os_version: String, osv_upt: String, upt: String)
extends java.io.Serializable
val adrSchema: StructType = StructType(Array(
StructField("imei", StringType),
StructField("android_id", StringType),
StructField("pkg_name", StringType),
StructField("oaid", StringType),
StructField("gaid", StringType),
StructField("sysid", StringType), // MTG 自建ID
......@@ -54,7 +146,33 @@ object Constant {
StructField("model", StringType), // 型号
StructField("os_version", StringType), // 操作系统版本
StructField("osv_upt", StringType), // 操作系统更新时间
StructField("upt", StringType) // 开机时间
StructField("upt", StringType), // 开机时间
StructField("event_type", StringType), // 事件类型
StructField("network_type", StringType) // 网络类型，用于判断API/SDK
))
val otherSchema: StructType = StructType(Array(
StructField("idfa", StringType),
StructField("idfv", StringType),
StructField("pkg_name", StringType),
StructField("imei", StringType),
StructField("android_id", StringType),
StructField("oaid", StringType),
StructField("gaid", StringType),
StructField("sysid", StringType), // MTG 自建ID
StructField("bkupid", StringType), // MTG 自建ID
StructField("xwho", StringType), // tkio 账号
StructField("user_id", StringType), // tkio 账号
StructField("country", StringType), // 国家信息
StructField("ip", StringType), // IP
StructField("ua", StringType), // UA
StructField("brand", StringType), // 品牌
StructField("model", StringType), // 型号
StructField("os_version", StringType), // 操作系统版本
StructField("osv_upt", StringType), // 操作系统更新时间
StructField("upt", StringType), // 开机时间
StructField("event_type", StringType), // 事件类型
StructField("network_type", StringType) // 网络类型，用于判断API/SDK
))
val dsp_req_sql =
......@@ -67,10 +185,10 @@ object Constant {
val dsp_req_sql_v2 =
"""
|SELECT idfa, googleadid gaid, ext5 exitid, os platform, countrycode country, deviceip ip,
| parseUA(ext6) ua, make brand, model, osv os_version, '0' osv_upt, '0' upt
| parseUA(ext6) ua, make brand, model, osv os_version, '0' osv_upt, '0' upt, cpackagename pkg_name,
| cncttype network_type
| FROM adn_dsp.log_adn_dsp_request_orc_hour
| WHERE CONCAT(yr,mt,dt) = '@date' @hour
| GROUP BY idfa, googleadid, ext5, os, countrycode, deviceip, parseUA(ext6), make, model, osv
|""".stripMargin
val adn_request_sql =
......@@ -83,7 +201,7 @@ object Constant {
// ext_data
val adn_request_sql_v2 =
"""
|SELECT idfa, gaid, ext_oaid oaid, getDevId(cdn_ab) idfv, dev_id androidid, ext_sysid extsysid, imei, platform,
|SELECT idfa, gaid, ext_oaid oaid, getDevId(cdn_ab) idfv, ext_packagename pkg_name, dev_id androidid, ext_sysid extsysid, imei, platform,
| country_code country, ip, parseUA(ext_systemuseragent) ua, ext_brand brand, ext_model model, os_version,
| COALESCE(get_json_object(ext_data,'$.osvut'),0) osv_upt, COALESCE(get_json_object(ext_data,'$.upt'),0) upt
| FROM dwh.ods_adn_trackingnew_request
......@@ -92,6 +210,35 @@ object Constant {
| ext_brand, ext_model, os_version, COALESCE(get_json_object(ext_data,'$.osvut'),0), COALESCE(get_json_object(ext_data,'$.upt'),0)
|""".stripMargin
// not distinct
val adn_request_sql_v3 =
"""
|SELECT idfa, gaid, ext_oaid oaid, getDevId(cdn_ab) idfv, ext_packagename pkg_name, dev_id androidid, ext_sysid extsysid, imei, platform,
| country_code country, ip, parseUA(ext_systemuseragent) ua, ext_brand brand, ext_model model, os_version,
| COALESCE(get_json_object(ext_data,'$.osvut'),0) osv_upt, COALESCE(get_json_object(ext_data,'$.upt'),0) upt, network_type
| FROM dwh.ods_adn_trackingnew_request
| WHERE CONCAT(yyyy,mm,dd) = '@date'
|""".stripMargin
val ios_id_mapping_sql: String =
"""
|SELECT idfa, idfv, pkg_name, sysid, bkupid, xwho, user_id, country, ip, ua, brand, model, os_version, osv_upt, upt, count(1) cnt
| FROM dwd.dwd_device_ios_ids_inc_daily WHERE dt = '@date'
| GROUP BY idfa, idfv, pkg_name, sysid, bkupid, xwho, user_id, country, ip, ua, brand, model, os_version, osv_upt, upt
|""".stripMargin
val android_id_mapping_sql: String =
"""
|SELECT imei, android_id, pkg_name, oaid, gaid, sysid, bkupid, xwho, user_id, country, ip, ua, brand, model, os_version, osv_upt, upt, count(1) cnt
| FROM dwd.dwd_device_android_ids_inc_daily WHERE dt = '@date' @filter_country
| GROUP BY imei, android_id, oaid, gaid, sysid, bkupid, xwho, user_id, country, ip, ua, brand, model, os_version, osv_upt, upt
|""".stripMargin
val old_id_mapping_sql: String =
"""
|
|""".stripMargin
val sss_sql =
"""
|""".stripMargin
......@@ -108,12 +255,135 @@ object Constant {
var dev_id = ""
if (StringUtils.isNotBlank(devId) && !",".equals(devId)) {
val ids = devId.split(",", -1)
if (StringUtils.isNotBlank(ids(0)) && ids(0).matches(MobvistaConstant.didPtn) && !ids(0).matches(MobvistaConstant.allZero)) {
if (StringUtils.isNotBlank(ids(0)) && ids(0).matches(didPtn) && !ids(0).matches(allZero)) {
dev_id = ids(0)
} else if (ids.length == 2 && !ids(1).matches(MobvistaConstant.didPtn) && !ids(1).matches(MobvistaConstant.allZero)) {
} else if (ids.length == 2 && !ids(1).matches(didPtn) && !ids(1).matches(allZero)) {
dev_id = ids(1)
}
}
dev_id
}
}
val iosIDScoreMap: Map[String, Double] = Map("idfa" -> 10, "sysid" -> 1, "xwho" -> 0.9, "user_id" -> 0.9, "bmosv_osv_upt" -> 0.9,
"idfv_bundle" -> 0.8, "bmosv_upt" -> 0.7, "bmosv_ipua_bundle" -> 0.6, "bkupid" -> 0.3)
val androidCNIDScoreMap: Map[String, Double] = Map("imei" -> 10, "oaid" -> 10, "gaid" -> 10, "sysid" -> 1, "xwho" -> 0.9, "user_id" -> 0.9,
"android_pkg" -> 0.8, "bmosv_upt" -> 0.7, "bmosv_ipua_pkg" -> 0.6, "bkupid" -> 0.3)
val androidIDScoreMap: Map[String, Double] = Map("gaid" -> 10, "imei" -> 10, "oaid" -> 10, "sysid" -> 1, "xwho" -> 0.9, "user_id" -> 0.9,
"android_pkg" -> 0.8, "bmosv_upt" -> 0.7, "bmosv_ipua_pkg" -> 0.6, "bkupid" -> 0.3)
def process(idfa: String, idfv: String, pkg_name: String, imei: String, androidId: String, oaid: String, gaid: String, sysId: String,
bkupId: String, country: String, ip: String, ua: String, brand: String, model: String, os_version: String, osv_upt: String,
upt: String, network_type: String, platform: String): (String, Row) = {
val f_idfa = if (StringUtils.isNotBlank(idfa) && idfa.matches(didPtn) && !idfa.matches(allZero)) {
idfa
} else {
""
}
val f_idfv = if (StringUtils.isNotBlank(idfv) && idfv.matches(didPtn) && !idfa.matches(allZero)) {
idfv
} else {
""
}
val f_imei = if (StringUtils.isNotBlank(imei) && imei.matches(imeiPtn) && !imei.matches(imeiPtnAll)) {
imei
} else {
""
}
val f_androidId = if (StringUtils.isNotBlank(androidId) && androidId.matches(andriodIdPtn) && !androidId.matches(andriodIdAll)) {
androidId
} else {
""
}
val f_oaid = if (StringUtils.isNotBlank(oaid) && oaid.length >= 16 && oaid.length <= 64) {
oaid
} else {
""
}
val f_gaid = if (StringUtils.isNotBlank(gaid) && gaid.matches(didPtn) && !gaid.matches(allZero)) {
gaid
} else {
""
}
val f_sysId = if (StringUtils.isNotBlank(sysId) && sysId.matches(didPtn) && !sysId.matches(allZero)) {
sysId
} else {
""
}
val f_bkupId = if (StringUtils.isNotBlank(bkupId) && bkupId.matches(didPtn) && !bkupId.matches(allZero)) {
bkupId
} else {
""
}
val f_country = if (StringUtils.isNotBlank(country)) {
country.toUpperCase
} else {
""
}
val f_ip = if (StringUtils.isNotBlank(ip) && ip.matches(ipPtn)) {
ip
} else {
""
}
var f_platform = if (StringUtils.isNotBlank(platform)) {
platform.toLowerCase()
} else {
""
}
val f_ua = if (StringUtils.isNotBlank(ua)) {
ua.trim
} else {
""
}
val f_brand = if (StringUtils.isNotBlank(brand)) {
brand.trim
} else {
""
}
val f_model = if (StringUtils.isNotBlank(model)) {
model.trim
} else {
""
}
val f_osv = if (StringUtils.isNotBlank(os_version)) {
os_version.trim
} else {
""
}
val f_osv_upt = if (StringUtils.isNotBlank(osv_upt) && osv_upt.length > 5) {
osv_upt.trim
} else {
""
}
val f_upt = if (StringUtils.isNotBlank(upt) && upt.length > 5) {
upt.trim
} else {
""
}
val f_network_type = if (StringUtils.isNotBlank(network_type) && !network_type.equalsIgnoreCase("unknown")) {
network_type.trim
} else {
""
}
f_platform = if (f_platform.contains("ios") || f_platform.contains("iphone") || f_brand.toLowerCase.contains("apple")
|| f_model.toLowerCase.contains("iphone") || f_model.toLowerCase.contains("ipad") || f_osv.toLowerCase.contains("ios")
|| f_ua.toLowerCase.contains("iphone") || f_ua.toLowerCase.contains("ipad")) {
"ios"
} else if (f_platform.contains("android") || f_osv.toLowerCase.contains("android") || f_ua.toLowerCase.contains("android")
|| f_imei.length >= 14 || (f_oaid.length >= 16 && f_oaid.length <= 64) || f_androidId.length >= 15 || f_gaid.length == 36) {
"android"
} else {
"other"
}
val rw = if ("ios".equals(f_platform)) {
Row(f_idfa, f_idfv, pkg_name, f_sysId, f_bkupId, "", "", f_country, f_ip, f_ua, f_brand, f_model, f_osv, f_osv_upt, f_upt, "", f_network_type)
} else if ("android".equals(f_platform)) {
Row(f_imei, f_androidId, pkg_name, f_oaid, f_gaid, f_sysId, f_bkupId, "", "", f_country, f_ip, f_ua, f_brand, f_model, f_osv, f_osv_upt, f_upt, "", f_network_type)
} else {
Row(f_idfa, f_idfv, pkg_name, f_imei, f_androidId, f_oaid, f_gaid, f_sysId, f_bkupId, "", "", f_country, f_ip, f_ua, f_brand, f_model, f_osv, f_osv_upt, f_upt, "", f_network_type)
}
(f_platform, rw)
}
}
\ No newline at end of file
src/main/scala/mobvista/dmp/datasource/id_mapping/DspReq.scala
package mobvista.dmp.datasource.id_mapping
import mobvista.dmp.common.MobvistaConstant
import mobvista.dmp.datasource.id_mapping.Constant.parseUA
import mobvista.dmp.datasource.id_mapping.Constant.{parseUA, process}
import org.apache.commons.lang3.StringUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SparkSession}
......@@ -34,12 +34,14 @@ class DspReq extends EtlDeviceIdDaily {
}
val sql = Constant.dsp_req_sql_v2.replace("@date", date)
.replace("@hour", hour)
val rdd = spark.sql(sql).coalesce(2000).rdd.map(row => {
val rdd = spark.sql(sql).coalesce(20000).rdd.map(row => {
var idfa = row.getAs[String]("idfa")
var gaid = row.getAs[String]("gaid")
val platform = row.getAs[String]("platform")
val exitId = row.getAs[String]("exitid")
var country = row.getAs[String]("country")
val pkg_name = row.getAs[String]("pkg_name")
val network_type = row.getAs[String]("network_type")
country = if (StringUtils.isNotBlank(country)) {
country.toUpperCase()
} else {
......@@ -83,12 +85,8 @@ class DspReq extends EtlDeviceIdDaily {
}
}
val rw = if ("ios".equalsIgnoreCase(platform)) {
Row(idfa, idfv, "", "", "", "", country, ip, ua, brand, model, os_version, osv_upt, upt)
} else {
Row(imei, androidId, oaid, gaid, "", "", "", "", country, ip, ua, brand, model, os_version, osv_upt, upt)
}
(platform, rw)
process(idfa, idfv, pkg_name, imei, androidId, oaid, gaid, sysId = "", bkupId = "", country, ip, ua, brand,
model, os_version, osv_upt, upt, network_type, platform)
})
rdd
}
......
src/main/scala/mobvista/dmp/datasource/id_mapping/EtlDeviceIdDaily.scala
......@@ -42,7 +42,7 @@ abstract class EtlDeviceIdDaily extends CommonSparkJob with Serializable {
if ("dsp_req".equalsIgnoreCase(business)) {
for (i <- 0 until 4) {
val df = processData(date, i, spark)
.repartition(20000)
.repartition(5000)
.persist(StorageLevel.MEMORY_AND_DISK_SER)
val iosTab = df.filter(plf => {
......@@ -72,11 +72,25 @@ abstract class EtlDeviceIdDaily extends CommonSparkJob with Serializable {
.option("orc.compress", "zlib")
.orc(output + s"/android/${i}")
val otherTab = df.filter(plf => {
"other".equals(plf._1)
}).map(row => {
row._2
})
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output + s"/other/${i}"), true)
spark.createDataFrame(otherTab, otherSchema)
.coalesce(coalesce)
.write.mode(SaveMode.Overwrite)
.option("orc.compress", "zlib")
.orc(output + s"/other/${i}")
df.unpersist(true)
}
} else {
val df = processData(date, 0, spark)
.repartition(20000)
.repartition(5000)
.persist(StorageLevel.MEMORY_AND_DISK_SER)
val iosTab = df.filter(plf => {
......@@ -105,6 +119,20 @@ abstract class EtlDeviceIdDaily extends CommonSparkJob with Serializable {
.write.mode(SaveMode.Overwrite)
.option("orc.compress", "zlib")
.orc(output + s"/android")
val otherTab = df.filter(plf => {
"other".equals(plf._1)
}).map(row => {
row._2
})
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(output + s"/other"), true)
spark.createDataFrame(otherTab, otherSchema)
.coalesce(coalesce)
.write.mode(SaveMode.Overwrite)
.option("orc.compress", "zlib")
.orc(output + s"/other")
}
} finally {
if (spark != null) {
......
src/main/scala/mobvista/dmp/datasource/id_mapping/IDMappingGraphx.scala 0 → 100644
package mobvista.dmp.datasource.id_mapping
import com.alibaba.fastjson.JSONObject
import mobvista.dmp.common.{CommonSparkJob, MobvistaConstant}
import mobvista.dmp.datasource.id_mapping.Constant._
import org.apache.commons.cli.{BasicParser, Options}
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{Row, SparkSession}
import java.net.URI
import scala.collection.mutable.ArrayBuffer
/**
* @package: mobvista.dmp.datasource.id_mapping
* @author: wangjf
* @date: 2021/12/7
* @time: 2:39 下午
* @email: jinfeng.wang@mobvista.com
*/
class IDMappingGraphx extends CommonSparkJob with Serializable {
def commandOptions(): Options = {
val options = new Options()
options.addOption("country", true, "country")
options.addOption("platform", true, "platform")
options.addOption("date", true, "date")
options.addOption("output", true, "output")
options.addOption("coalesce", true, "coalesce")
options
}
override protected def run(args: Array[String]): Int = {
val parser = new BasicParser()
val options = commandOptions()
val commandLine = parser.parse(options, args)
val country = commandLine.getOptionValue("country")
val platform = commandLine.getOptionValue("platform")
val date = commandLine.getOptionValue("date")
val output = commandLine.getOptionValue("output")
val coalesce = Integer.parseInt(commandLine.getOptionValue("coalesce"))
val spark = MobvistaConstant.createSparkSession(s"IDMappingGraphx.$date.$country.$platform")
try {
oldAndTodayIdMapping(country, platform, date, spark, output, output, coalesce)
} finally {
if (spark != null) {
spark.stop()
}
}
0
}
def oldAndTodayIdMapping(country: String, platform: String, date: String, spark: SparkSession, outPutPath: String,
edgeoutPutPath: String, coalesce: Int) = {
implicit val formats = org.json4s.DefaultFormats
var dailySQL = ""
var schame: StructType = null
var idSet: Set[String] = null
var idMainSet: Set[String] = null
// 1.今日数据加载
platform match {
case "ios" =>
dailySQL = Constant.ios_id_mapping_sql.replace("@date", date)
schame = iosVertSchema
idSet = iosIDSet
idMainSet = iosMainIDSet
case "android" => {
schame = adrVertSchema
idMainSet = androidMainIDSet
country match {
case "CN" =>
idSet = androidCNIDSet
dailySQL = Constant.android_id_mapping_sql.replace("@date", date).replace("@filter_country", s"AND country = '${country}'")
case _ =>
idSet = androidIDSet
dailySQL = Constant.android_id_mapping_sql.replace("@date", date).replace("@filter_country", s"AND country != '${country}'")
}
}
case _ =>
""
}
val todayDF = spark.createDataFrame(spark.sql(dailySQL).rdd.map(row => {
processData(row, platform)
}), schema = schame)
val vertex = todayDF.rdd.map(row => {
val res = processVertex(date, row, idSet.toArray, idMainSet)
/*
val res = platform match {
case "ios" =>
processVertex(date, rows, Constant.iosIDSet.toArray, Constant.iosMainIDSet)
case _ => {
country.toUpperCase match {
case "CN" =>
processVertex(date, rows, Constant.androidCNIDSet.toArray, Constant.androidMainIDSet)
case _ =>
processVertex(date, rows, Constant.androidIDSet.toArray, Constant.androidMainIDSet)
}
}
}
*/
res
}).flatMap(l => l)
val maxGraph = vertex.combineByKey(
(v: (String, JSONObject)) => Iterable(v),
(c: Iterable[(String, JSONObject)], v: (String, JSONObject)) => c ++ Seq(v),
(c1: Iterable[(String, JSONObject)], c2: Iterable[(String, JSONObject)]) => c1 ++ c2
)
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(outPutPath), true)
maxGraph
.repartition(coalesce)
.saveAsTextFile(outPutPath, classOf[GzipCodec])
/*
val multiOneIDRDD = maxGraph.filter(kv => {
kv._2.size > 1
}).map(rs => {
platform match {
case "ios" =>
updateOneID(rs, Constant.iosMainIDSet)
case _ =>
updateOneID(rs, Constant.androidMainIDSet)
}
}).flatMap(l => l)
val singleOneIDRDD = maxGraph.filter(kv => {
kv._2.size == 1
}).map(kv => {
val oneID = new JSONObject()
val srcID = kv._1
kv._2.foreach(it => {
oneID.put(it._1, it._2)
})
(srcID, oneID)
})
FileSystem.get(new URI(s"s3://mob-emr-test"), spark.sparkContext.hadoopConfiguration).delete(new Path(outPutPath), true)
multiOneIDRDD.union(singleOneIDRDD)
.repartition(coalesce)
.saveAsTextFile(outPutPath, classOf[GzipCodec])
*/
}
def processData(row: Row, platform: String): Row = {
platform match {
case "ios" =>
var idfa = row.getAs[String]("idfa")
idfa = if (StringUtils.isNotBlank(idfa) && idfa.matches(didPtn) && !idfa.matches(allZero)) {
idfa
} else {
""
}
var idfv = row.getAs[String]("idfv")
idfv = if (StringUtils.isNotBlank(idfv) && idfv.matches(didPtn) && !idfv.matches(allZero)) {
idfv
} else {
""
}
val pkg_name = row.getAs[String]("pkg_name")
val sysid = row.getAs[String]("sysid")
val bkupid = row.getAs[String]("bkupid")
val xwho = row.getAs[String]("xwho")
val user_id = row.getAs[String]("user_id")
val country = row.getAs[String]("country")
val ip = row.getAs[String]("ip")
val ua = row.getAs[String]("ua")
val brand = row.getAs[String]("brand")
val model = row.getAs[String]("model")
val os_version = row.getAs[String]("os_version")
val osv_upt = row.getAs[String]("osv_upt")
val upt = row.getAs[String]("upt")
val cnt = row.getAs[Long]("cnt")
val idfv_bundle = if (StringUtils.isNotBlank(idfv) && StringUtils.isNotBlank(pkg_name)) {
idfv + pkg_name
} else {
""
}
val bmosv_osv_upt = if (StringUtils.isNotBlank(osv_upt)) {
brand + model + os_version + osv_upt
} else {
""
}
val bmosv_upt = if (StringUtils.isNotBlank(upt)) {
brand + model + os_version + upt
} else {
""
}
val bmosv_ipua_bundle = if (StringUtils.isNotBlank(ip) && StringUtils.isNotBlank(pkg_name)) {
brand + model + os_version + ip + ua + pkg_name
} else {
""
}
// IosVert(idfa, sysid, idfv_bundle, bmosv_osv_upt, bmosv_upt, bmosv_ipua_bundle, xwho, user_id, bkupid, cnt)
Row(idfa, sysid, idfv_bundle, bmosv_osv_upt, bmosv_upt, bmosv_ipua_bundle, xwho, user_id, bkupid, cnt)
case "android" =>
val imei = row.getAs[String]("imei")
val android_id = row.getAs[String]("android_id")
val pkg_name = row.getAs[String]("pkg_name")
val oaid = row.getAs[String]("oaid")
val gaid = row.getAs[String]("gaid")
val sysid = row.getAs[String]("sysid")
val bkupid = row.getAs[String]("bkupid")
val xwho = row.getAs[String]("xwho")
val user_id = row.getAs[String]("user_id")
val country = row.getAs[String]("country")
val ip = row.getAs[String]("ip")
val ua = row.getAs[String]("ua")
val brand = row.getAs[String]("brand")
val model = row.getAs[String]("model")
val os_version = row.getAs[String]("os_version")
val upt = row.getAs[String]("upt")
val cnt = row.getAs[Long]("cnt")
val android_pkg = if (StringUtils.isNotBlank(android_id) && StringUtils.isNotBlank(pkg_name)) {
android_id + pkg_name
} else {
""
}
val bmosv_upt = if (StringUtils.isNotBlank(upt)) {
brand + model + os_version + upt
} else {
""
}
val bmosv_ipua_pkg = if (StringUtils.isNotBlank(ip) && StringUtils.isNotBlank(pkg_name)) {
brand + model + os_version + ip + ua + pkg_name
} else {
""
}
// AdrVert(imei, gaid, oaid, sysid, android_pkg, bmosv_upt, bmosv_ipua_pkg, xwho, user_id, bkupid, cnt)
if ("CN".equalsIgnoreCase(country)) {
Row(imei, oaid, gaid, sysid, android_pkg, bmosv_upt, bmosv_ipua_pkg, xwho, user_id, bkupid, cnt)
} else {
Row(gaid, imei, oaid, sysid, android_pkg, bmosv_upt, bmosv_ipua_pkg, xwho, user_id, bkupid, cnt)
}
case _ =>
Row("")
}
}
def processVertex(date: String, row: Row, ids: Array[String], mainIDSet: Set[String]): ArrayBuffer[(String, (String, JSONObject))] = {
val array = new ArrayBuffer[(String, (String, JSONObject))]()
implicit val formats = org.json4s.DefaultFormats
// val json = JSON.parseObject(Serialization.write(row))
// 事件频次
val cnt = row.getAs[Long]("cnt")
// date 活跃日期，用于计算权重
var flag = true
for (i <- 0 to ids.length - 2) {
if (StringUtils.isNotBlank(row.getAs[String](String.valueOf(ids(i)))) && flag) {
val jsonObject = new JSONObject()
val oneIDType = ids(i)
jsonObject.put("id_type", oneIDType)
jsonObject.put("active_type", date)
jsonObject.put("cnt", cnt)
val oneID = row.getAs[String](String.valueOf(ids(i)))
array += ((oneID, (oneID, jsonObject)))
for (j <- i + 1 until ids.length) {
val srcOrg = row.getAs[String](String.valueOf(ids(j)))
if (mainIDSet.contains(oneIDType)) {
array += ((srcOrg, (oneID, jsonObject)))
} else {
array += ((oneID, (srcOrg, jsonObject)))
}
}
flag = false
}
}
array
}
def updateOneID(kv: (String, Iterable[(String, JSONObject)]), mainIDSet: Set[String]): ArrayBuffer[(String, JSONObject)] = {
val array = new ArrayBuffer[(String, JSONObject)]()
val iters = kv._2
// val oneID = new ArrayBuffer[(String, String)]()
val oneID = new JSONObject()
iters.foreach(kv => {
oneID.put(kv._1, kv._2)
})
iters.filter(tp => {
mainIDSet.contains(tp._2.getString("id_type"))
}).foreach(itr => {
val k = itr._1
array += ((k, oneID))
})
array
}
}
object IDMappingGraphx {
def main(args: Array[String]): Unit = {
new IDMappingGraphx().run(args)
}
}
\ No newline at end of file
src/main/scala/mobvista/dmp/utils/common/MD5Util.scala 0 → 100644
package mobvista.dmp.utils.common
import mobvista.dmp.common.MobvistaConstant
import java.io.UnsupportedEncodingException
import java.math.BigInteger
import java.security.{MessageDigest, NoSuchAlgorithmException}
/**
* @package: mobvista.dmp.utils.common
* @author: wangjf
* @date: 2021/12/7
* @time: 4:13 下午
* @email: jinfeng.wang@mobvista.com
*/
object MD5Util {
def getMD5Long(str: String): Long = {
val id_map: Map[String, String] = Map("a" -> "1", "b" -> "2", "c" -> "3", "d" -> "4", "e" -> "5", "f" -> "6", "g" -> "7")
var strm = ""
try {
// 第一步，获取MessageDigest对象，参数为MD5表示这是一个MD5算法
val md5 = MessageDigest.getInstance("MD5")
// 第二步跳过，输入源数据，参数类型为byte[]
// 第三步，计算MD5值
val array = md5.digest(str.getBytes("UTF-8"))
// 第四步，结果转换并返回
val bigInt = new BigInteger(1, array).toString(16).substring(8, 24)
for (c <- bigInt.toCharArray) {
if (id_map.contains(String.valueOf(c))) strm += id_map(String.valueOf(c))
else strm += String.valueOf(c)
}
} catch {
case e@(_: NoSuchAlgorithmException | _: UnsupportedEncodingException) =>
e.printStackTrace()
}
strm.toLong
}
def main(args: Array[String]): Unit = {
val data =
"""
|{ "@timestamp": "07/Dec/2021:23:13:36 +0800", "@fields": { "remote_addr": "100.122.18.27", "remote_user": "-", "body_bytes_sent": "0", "status": "200", "http_host": "analytics-http-ab.mintegral.net", "request": "POST / HTTP/1.1", "request_method": "POST", "request_body": "platform=1&package_name=com.Upperpik.HairChallenge&os_version=11&brand=samsung&model=SM-T510&gaid=6b4e83cb-b21b-41e4-a187-ed1c2c56197e&network_type=9&network_str=&language=es-ES&timezone=GMT%252B01%253A00&ua=Mozilla%252F5.0%2B%2528Linux%253B%2BAndroid%2B11%253B%2BSM-T510
|""".stripMargin
println(MobvistaConstant.String2JSONObject(data))
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment