Skip to content

M
mobvista-dmp
Commit ca633a12 by fan.jiang
Options

dsp_req分区adx dealid入库需求

parent 4b2c5073
Showing with 62 additions and 66 deletions
src/main/scala/mobvista/dmp/datasource/dsp/DspOrgEtlDailys.scala
......@@ -340,62 +340,47 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
val dealerid = row.getAs[String]("dealerid")
val dealeridArray: Array[String] = dealerid.split(",")
if ("oppocn".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
if (dealeridArray.contains("2532") || dealeridArray.contains("2533")) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao", androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao_oppo", androidId, time, segment, region)
*/
value.packageName = "com.taobao.taobao"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.taobao.taobao_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
} else {
// 2020.11.26 需求 若adx（exchanges字段）为oppo且dealid不等于2532或2533，则伪包名为com.taobao.taobao_notinstall_oppo
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao_notinstall_oppo", androidId, time, segment, region)
*/
value.packageName = "com.taobao.taobao_notinstall_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
// 添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
if (dealeridArray.contains("2728")) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.UCMobile", androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.ucmobile_oppo", androidId, time, segment, region)
*/
value.packageName = "com.UCMobile"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.ucmobile_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
if (dealeridArray.contains("4059")) {
value.packageName = "com.ss.android.ugc.aweme"
val mapData_oppoziyou= Map("4059" -> "com.ss.android.ugc.aweme",
"4140" -> "com.xunmeng.pinduoduo",
"4183" -> "me.ele")
for (item <- mapData_oppoziyou) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.ss.android.ugc.aweme_oppoziyou"
value.packageName = item._2 + "_oppoziyou"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
if (dealeridArray.contains("4053")) {
value.packageName = "com.ss.android.ugc.aweme"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.ss.android.ugc.aweme_oppolianmeng"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
if (dealeridArray.contains("4140")) {
value.packageName = "com.xunmeng.pinduoduo"
val mapData_oppolianmeng= Map("4053" -> "com.ss.android.ugc.aweme",
"4143" -> "com.xunmeng.pinduoduo",
"4182" -> "me.ele")
for (item <- mapData_oppolianmeng) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.xunmeng.pinduoduo_oppoziyou"
value.packageName = item._2 + "_oppolianmeng"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
if (dealeridArray.contains("4143")) {
value.packageName = "com.xunmeng.pinduoduo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "com.xunmeng.pinduoduo_oppolianmeng"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
val mapData_oppocn: Map[Int, String] = Map(4060 -> "com.ss.android.ugc.aweme_oppoziyou_notinstall",
......@@ -405,7 +390,11 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
4141 -> "com.xunmeng.pinduoduo_oppoziyou_notinstall",
4142 -> "com.xunmeng.pinduoduo_oppoziyou_hist_notinstall",
4144 -> "com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall",
4145 -> "com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall")
4145 -> "com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall",
4184 -> "me.ele_oppoziyou_hist_notinstall",
4185 -> "me.ele_oppoziyou_notinstall",
4180 -> "me.ele_oppolianmeng_notinstall",
4181 -> "me.ele_oppolianmeng_hist_notinstall")
for (item <- mapData_oppocn) {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
......@@ -432,15 +421,9 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
3160 -> "com.tencent.news")
for (item <- mapData) {
if (dealeridArray.contains(item._1.toString)) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData(num), androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData(num) + "_oppo", androidId, time, segment, region)
*/
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2+"_oppo"
value.packageName = item._2 + "_oppo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
......@@ -449,26 +432,20 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
2773 -> "com.jingdong.app.mallr_notinstall_oppo")
for (item <- mapData) {
if (dealeridArray.contains(item._1.toString)) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData(num), androidId, time, segment, region)
*/
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
if ("inmobi".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
if (dealeridArray.contains("1594807676568")) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao_inmobi", androidId, time, segment, region)
*/
value.packageName = "com.taobao.taobao_inmobi"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
//2020.11.26 增加bes入库需求
// wiki https://confluence.mobvista.com/pages/viewpage.action?pageId=47976499
//2021.04.22 添加adx（exchanges字段）为bes且dealid=100188，则伪包名为com.tencent.news_bes和com.tencent.news
......@@ -484,48 +461,50 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
"100188" -> "com.tencent.news",
"100310" -> "com.taobao.litetao",
"100203" -> "com.ss.android.ugc.aweme",
"00251" -> "com.xunmeng.pinduoduo")
"00251" -> "com.xunmeng.pinduoduo",
"100304" -> "me.ele")
val mapData_ios_bes = Map("100197" -> "id387682726",
"100345" -> "id333206289",
"100344" -> "id1340376323",
"100353" -> "id1044283059")
"100353" -> "id1044283059",
"100368" -> "id507161324")
if ("bes".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
for (item <- mapData_bes) {
if (dealeridArray.contains(item._1)) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData_bes(num), androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData_bes(num) + "_bes", androidId, time, segment, region)
*/
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2+"_bes"
value.packageName = item._2 + "_bes"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
if("ios".equalsIgnoreCase(platform)) {
for (item <- mapData_ios_bes) {
if (dealeridArray.contains(item._1)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = item._2+"2021090301"
if("100368".equalsIgnoreCase(item._1)){
value.packageName = item._2 + "2021091701"
}else{
value.packageName = item._2 + "2021090301"
}
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
}
//2021.04.28 天级别从dsp请求日志（adn_dsp.log_adn_dsp_request_orc_hour）中拉取exchanges='tencent' 并且appid = 'com.tencent.news',入库伪包名为com.tencent.news_fromtencent
if ("tencent".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
if (packageName.split("#", -1).contains("com.tencent.news")) {
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.tencent.news_fromtencent", androidId, time, segment, region)
*/
value.packageName = "com.tencent.news_fromtencent"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
//2021.06.17 天级别从dsp请求日志（adn_dsp.log_adn_dsp_request_orc_hour）中拉取exchanges='kuaishou' 并且appid 符合下列名称的入库
val mapData_kuaishou = Map("com.smile.gifmaker" -> "com.smile.gifmaker_fromkuaishou",
"com.kuaishou.nebula" -> "com.kuaishou.nebula_fromkuaishou",
......@@ -547,10 +526,12 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
3996 -> "com.sankuai.meituan",
3334 -> "com.ss.android.ugc.aweme",
112799 -> "com.ss.android.ugc.aweme",
7156 -> "com.tencent.news")
7156 -> "com.tencent.news",
1649 -> "me.ele")
val mapData_ios_iqiyi = Map(
3334 -> "id1142110895",
112799 -> "id1142110895")
112799 -> "id1142110895",
1649 -> "id507161324")
if("iqiyi".equals(exchanges)){
if("android".equalsIgnoreCase(platform)) {
for (item <- mapData_iqiyi) {
......@@ -567,7 +548,11 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
if (dealeridArray.contains(item._1.toString)) {
value.packageName = item._2
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
if("1649".equalsIgnoreCase(item._1.toString)){
value.packageName = item._2 + "2021091702"
}else{
value.packageName = item._2 + "2021090302"
}
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
......@@ -588,6 +573,17 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
}
}
if ("vivo".equals(exchanges)) {
if("android".equalsIgnoreCase(platform)) {
if (dealeridArray.contains("ba6e4f5323603561086a985b6fc5ffb4")) {
value.packageName = "me.ele"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
value.packageName = "me.ele_vivo"
arrayBuffer = addDatasV2(arrayBuffer, deviceId, gaidmd5, imei, imeimd5, oaid, oaidmd5, androidId, idfv, deviceType, platform, value)
}
}
}
//adx=iqiyi 且os=android的imeiMD5和oaidMD5去重设备， 分别和com.taobao.foractivation.227229和com.taobao.foractivation.227229_oaid做差集 2020.12.15
//2021.04.22下掉,该需求不需要了
// if("iqiyi".equals(exchanges)){
......
src/main/scala/mobvista/dmp/datasource/dsp/DspOrgLogEtlHoursDemo.scala
......@@ -141,7 +141,7 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
val exitId = row.getAs[String]("exitid")
val exchanges = row.getAs[String]("exchanges")
var dealerid = ""
if("mopub".equalsIgnoreCase(exchanges) || "oppocn".equalsIgnoreCase(exchanges) || "inmobi".equalsIgnoreCase(exchanges) || "bes".equalsIgnoreCase(exchanges) || "iqiyi".equalsIgnoreCase(exchanges) ){
if("mopub".equalsIgnoreCase(exchanges) || "oppocn".equalsIgnoreCase(exchanges) || "inmobi".equalsIgnoreCase(exchanges) || "bes".equalsIgnoreCase(exchanges) || "iqiyi".equalsIgnoreCase(exchanges) || "vivo".equalsIgnoreCase(exchanges) ){
val ext3 = row.getAs[String]("ext3")
if(StringUtils.isNotBlank(ext3) && ext3.startsWith("{")){
try{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment