Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
mobvista-dmp
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
王金锋
mobvista-dmp
Commits
ca633a12
Commit
ca633a12
authored
Sep 17, 2021
by
fan.jiang
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
dsp_req分区adx dealid入库需求
parent
4b2c5073
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
145 additions
and
149 deletions
+145
-149
DspOrgEtlDailys.scala
...n/scala/mobvista/dmp/datasource/dsp/DspOrgEtlDailys.scala
+144
-148
DspOrgLogEtlHoursDemo.scala
...a/mobvista/dmp/datasource/dsp/DspOrgLogEtlHoursDemo.scala
+1
-1
No files found.
src/main/scala/mobvista/dmp/datasource/dsp/DspOrgEtlDailys.scala
View file @
ca633a12
...
...
@@ -340,133 +340,110 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
val
dealerid
=
row
.
getAs
[
String
](
"dealerid"
)
val
dealeridArray
:
Array
[
String
]
=
dealerid
.
split
(
","
)
if
(
"oppocn"
.
equals
(
exchanges
))
{
if
(
dealeridArray
.
contains
(
"2532"
)
||
dealeridArray
.
contains
(
"2533"
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao", androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao_oppo", androidId, time, segment, region)
*/
value
.
packageName
=
"com.taobao.taobao"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.taobao.taobao_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
else
{
// 2020.11.26 需求 若adx(exchanges字段)为oppo且dealid不等于2532或2533,则伪包名为com.taobao.taobao_notinstall_oppo
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao_notinstall_oppo", androidId, time, segment, region)
*/
value
.
packageName
=
"com.taobao.taobao_notinstall_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
// 添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
if
(
dealeridArray
.
contains
(
"2728"
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.UCMobile", androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.ucmobile_oppo", androidId, time, segment, region)
*/
value
.
packageName
=
"com.UCMobile"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.ucmobile_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
if
(
dealeridArray
.
contains
(
"4059"
))
{
value
.
packageName
=
"com.ss.android.ugc.aweme"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.ss.android.ugc.aweme_oppoziyou"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
if
(
dealeridArray
.
contains
(
"4053"
))
{
value
.
packageName
=
"com.ss.android.ugc.aweme"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.ss.android.ugc.aweme_oppolianmeng"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
if
(
dealeridArray
.
contains
(
"4140"
))
{
value
.
packageName
=
"com.xunmeng.pinduoduo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.xunmeng.pinduoduo_oppoziyou"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
if
(
dealeridArray
.
contains
(
"4143"
))
{
value
.
packageName
=
"com.xunmeng.pinduoduo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.xunmeng.pinduoduo_oppolianmeng"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
val
mapData_oppocn
:
Map
[
Int
,
String
]
=
Map
(
4060
->
"com.ss.android.ugc.aweme_oppoziyou_notinstall"
,
4061
->
"com.ss.android.ugc.aweme_oppoziyou_hist_notinstall"
,
4054
->
"com.ss.android.ugc.aweme_oppolianmeng_hist1year_notinstall"
,
4055
->
"com.ss.android.ugc.aweme_oppolianmeng_histhalfyear_notinstall"
,
4141
->
"com.xunmeng.pinduoduo_oppoziyou_notinstall"
,
4142
->
"com.xunmeng.pinduoduo_oppoziyou_hist_notinstall"
,
4144
->
"com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall"
,
4145
->
"com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall"
)
for
(
item
<-
mapData_oppocn
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
value
.
packageName
=
item
.
_2
if
(
"android"
.
equalsIgnoreCase
(
platform
))
{
if
(
dealeridArray
.
contains
(
"2532"
)
||
dealeridArray
.
contains
(
"2533"
))
{
value
.
packageName
=
"com.taobao.taobao"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
// 添加 oppo 在请求中开始传支付宝,京东,爱奇艺这几个包名的安装信息,并入库。 (2020-10-28 17:17 补充说明: com.eg.android.AlipayGphone_oppo伪包名大写,但已经入库,修改困难。以后入库伪包名统一定为小写,类似上面的 com.ucmobile_oppo)
// 2020.11.11 添加 2783对应的咸鱼的 com.taobao.idlefish 和 com.taobao.idlefish_oppo 入库
// 2020.11.26 添加 2840对应的 com.youku.phone_notinstall 和 com.youku.phone_notinstall_oppo 入库
// 2020.12.24 添加 2889对应的 com.sankuai.meituan和 com.sankuai.meituan_oppo 入库
// 2020.12.24 添加 2890对应的 com.meituan.itakeaway和 com.meituan.itakeaway_oppo 入库
// 2021.01.06添加 快手、快手极速版、优酷
// 2021.04.22 添加adx(exchanges字段)为oppo且dealid=3160,则伪包名为com.tencent.news_oppo和com.tencent.news
var
mapData
:
Map
[
Int
,
String
]
=
Map
(
2716
->
"com.jingdong.app.mall"
,
2717
->
"com.eg.android.AlipayGphone"
,
2718
->
"com.qiyi.video"
,
2783
->
"com.taobao.idlefish"
,
2840
->
"com.youku.phone_notinstall"
,
2889
->
"com.sankuai.meituan"
,
2890
->
"com.meituan.itakeaway"
,
2904
->
"com.smile.gifmaker"
,
2905
->
"com.kuaishou.nebula"
,
2906
->
"com.youku.phone"
,
3160
->
"com.tencent.news"
)
for
(
item
<-
mapData
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData(num), androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData(num) + "_oppo", androidId, time, segment, region)
*/
value
.
packageName
=
item
.
_2
value
.
packageName
=
"com.taobao.taobao_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"_oppo"
}
else
{
// 2020.11.26 需求 若adx(exchanges字段)为oppo且dealid不等于2532或2533,则伪包名为com.taobao.taobao_notinstall_oppo
value
.
packageName
=
"com.taobao.taobao_notinstall_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
//2020.11.11添加快手、京东入库
mapData
=
Map
(
2774
->
"com.smile.gifmaker_notinstall_oppo"
,
2773
->
"com.jingdong.app.mallr_notinstall_oppo"
)
for
(
item
<-
mapData
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData(num), androidId, time, segment, region)
*/
value
.
packageName
=
item
.
_2
// 添加com.UCMobile 对应伪包名小写com.ucmobile_oppo
if
(
dealeridArray
.
contains
(
"2728"
))
{
value
.
packageName
=
"com.UCMobile"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"com.ucmobile_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
val
mapData_oppoziyou
=
Map
(
"4059"
->
"com.ss.android.ugc.aweme"
,
"4140"
->
"com.xunmeng.pinduoduo"
,
"4183"
->
"me.ele"
)
for
(
item
<-
mapData_oppoziyou
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"_oppoziyou"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
val
mapData_oppolianmeng
=
Map
(
"4053"
->
"com.ss.android.ugc.aweme"
,
"4143"
->
"com.xunmeng.pinduoduo"
,
"4182"
->
"me.ele"
)
for
(
item
<-
mapData_oppolianmeng
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"_oppolianmeng"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
val
mapData_oppocn
:
Map
[
Int
,
String
]
=
Map
(
4060
->
"com.ss.android.ugc.aweme_oppoziyou_notinstall"
,
4061
->
"com.ss.android.ugc.aweme_oppoziyou_hist_notinstall"
,
4054
->
"com.ss.android.ugc.aweme_oppolianmeng_hist1year_notinstall"
,
4055
->
"com.ss.android.ugc.aweme_oppolianmeng_histhalfyear_notinstall"
,
4141
->
"com.xunmeng.pinduoduo_oppoziyou_notinstall"
,
4142
->
"com.xunmeng.pinduoduo_oppoziyou_hist_notinstall"
,
4144
->
"com.xunmeng.pinduoduo_oppolianmeng_hist1year_notinstall"
,
4145
->
"com.xunmeng.pinduoduo_oppolianmeng_histhalfyear_notinstall"
,
4184
->
"me.ele_oppoziyou_hist_notinstall"
,
4185
->
"me.ele_oppoziyou_notinstall"
,
4180
->
"me.ele_oppolianmeng_notinstall"
,
4181
->
"me.ele_oppolianmeng_hist_notinstall"
)
for
(
item
<-
mapData_oppocn
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
// 添加 oppo 在请求中开始传支付宝,京东,爱奇艺这几个包名的安装信息,并入库。 (2020-10-28 17:17 补充说明: com.eg.android.AlipayGphone_oppo伪包名大写,但已经入库,修改困难。以后入库伪包名统一定为小写,类似上面的 com.ucmobile_oppo)
// 2020.11.11 添加 2783对应的咸鱼的 com.taobao.idlefish 和 com.taobao.idlefish_oppo 入库
// 2020.11.26 添加 2840对应的 com.youku.phone_notinstall 和 com.youku.phone_notinstall_oppo 入库
// 2020.12.24 添加 2889对应的 com.sankuai.meituan和 com.sankuai.meituan_oppo 入库
// 2020.12.24 添加 2890对应的 com.meituan.itakeaway和 com.meituan.itakeaway_oppo 入库
// 2021.01.06添加 快手、快手极速版、优酷
// 2021.04.22 添加adx(exchanges字段)为oppo且dealid=3160,则伪包名为com.tencent.news_oppo和com.tencent.news
var
mapData
:
Map
[
Int
,
String
]
=
Map
(
2716
->
"com.jingdong.app.mall"
,
2717
->
"com.eg.android.AlipayGphone"
,
2718
->
"com.qiyi.video"
,
2783
->
"com.taobao.idlefish"
,
2840
->
"com.youku.phone_notinstall"
,
2889
->
"com.sankuai.meituan"
,
2890
->
"com.meituan.itakeaway"
,
2904
->
"com.smile.gifmaker"
,
2905
->
"com.kuaishou.nebula"
,
2906
->
"com.youku.phone"
,
3160
->
"com.tencent.news"
)
for
(
item
<-
mapData
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"_oppo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
//2020.11.11添加快手、京东入库
mapData
=
Map
(
2774
->
"com.smile.gifmaker_notinstall_oppo"
,
2773
->
"com.jingdong.app.mallr_notinstall_oppo"
)
for
(
item
<-
mapData
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
}
}
if
(
"inmobi"
.
equals
(
exchanges
))
{
if
(
dealeridArray
.
contains
(
"1594807676568"
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.taobao.taobao_inmobi", androidId, time, segment, region)
*/
value
.
packageName
=
"com.taobao.taobao_inmobi"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
if
(
"android"
.
equalsIgnoreCase
(
platform
))
{
if
(
dealeridArray
.
contains
(
"1594807676568"
))
{
value
.
packageName
=
"com.taobao.taobao_inmobi"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
}
//2020.11.26 增加bes入库需求
...
...
@@ -484,46 +461,48 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
"100188"
->
"com.tencent.news"
,
"100310"
->
"com.taobao.litetao"
,
"100203"
->
"com.ss.android.ugc.aweme"
,
"00251"
->
"com.xunmeng.pinduoduo"
)
"00251"
->
"com.xunmeng.pinduoduo"
,
"100304"
->
"me.ele"
)
val
mapData_ios_bes
=
Map
(
"100197"
->
"id387682726"
,
"100345"
->
"id333206289"
,
"100344"
->
"id1340376323"
,
"100353"
->
"id1044283059"
)
"100353"
->
"id1044283059"
,
"100368"
->
"id507161324"
)
if
(
"bes"
.
equals
(
exchanges
))
{
for
(
item
<-
mapData_bes
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData_bes(num), androidId, time, segment, region)
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
mapData_bes(num) + "_bes", androidId, time, segment, region)
*/
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"_bes"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
if
(
"android"
.
equalsIgnoreCase
(
platform
))
{
for
(
item
<-
mapData_bes
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"_bes"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
}
for
(
item
<-
mapData_ios_bes
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"2021090301"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
if
(
"ios"
.
equalsIgnoreCase
(
platform
))
{
for
(
item
<-
mapData_ios_bes
)
{
if
(
dealeridArray
.
contains
(
item
.
_1
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
if
(
"100368"
.
equalsIgnoreCase
(
item
.
_1
)){
value
.
packageName
=
item
.
_2
+
"2021091701"
}
else
{
value
.
packageName
=
item
.
_2
+
"2021090301"
}
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
}
}
//2021.04.28 天级别从dsp请求日志(adn_dsp.log_adn_dsp_request_orc_hour)中拉取exchanges='tencent' 并且appid = 'com.tencent.news',入库伪包名为com.tencent.news_fromtencent
if
(
"tencent"
.
equals
(
exchanges
))
{
if
(
packageName
.
split
(
"#"
,
-
1
).
contains
(
"com.tencent.news"
))
{
/*
arrayBuffer = addDatas(arrayBuffer, imei, imeimd5, oaid, oaidmd5, deviceId, deviceType, platform, country, ip, gender, birthday, maker, model, osVersion,
"com.tencent.news_fromtencent", androidId, time, segment, region)
*/
value
.
packageName
=
"com.tencent.news_fromtencent"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
if
(
"android"
.
equalsIgnoreCase
(
platform
))
{
if
(
packageName
.
split
(
"#"
,
-
1
).
contains
(
"com.tencent.news"
))
{
value
.
packageName
=
"com.tencent.news_fromtencent"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
}
//2021.06.17 天级别从dsp请求日志(adn_dsp.log_adn_dsp_request_orc_hour)中拉取exchanges='kuaishou' 并且appid 符合下列名称的入库
...
...
@@ -547,10 +526,12 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
3996
->
"com.sankuai.meituan"
,
3334
->
"com.ss.android.ugc.aweme"
,
112799
->
"com.ss.android.ugc.aweme"
,
7156
->
"com.tencent.news"
)
7156
->
"com.tencent.news"
,
1649
->
"me.ele"
)
val
mapData_ios_iqiyi
=
Map
(
3334
->
"id1142110895"
,
112799
->
"id1142110895"
)
112799
->
"id1142110895"
,
1649
->
"id507161324"
)
if
(
"iqiyi"
.
equals
(
exchanges
)){
if
(
"android"
.
equalsIgnoreCase
(
platform
))
{
for
(
item
<-
mapData_iqiyi
)
{
...
...
@@ -567,7 +548,11 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
if
(
dealeridArray
.
contains
(
item
.
_1
.
toString
))
{
value
.
packageName
=
item
.
_2
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
item
.
_2
+
"2021090302"
if
(
"1649"
.
equalsIgnoreCase
(
item
.
_1
.
toString
)){
value
.
packageName
=
item
.
_2
+
"2021091702"
}
else
{
value
.
packageName
=
item
.
_2
+
"2021090302"
}
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
...
...
@@ -588,6 +573,17 @@ class DspOrgEtlDailys extends CommonSparkJob with Serializable {
}
}
if
(
"vivo"
.
equals
(
exchanges
))
{
if
(
"android"
.
equalsIgnoreCase
(
platform
))
{
if
(
dealeridArray
.
contains
(
"ba6e4f5323603561086a985b6fc5ffb4"
))
{
value
.
packageName
=
"me.ele"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
value
.
packageName
=
"me.ele_vivo"
arrayBuffer
=
addDatasV2
(
arrayBuffer
,
deviceId
,
gaidmd5
,
imei
,
imeimd5
,
oaid
,
oaidmd5
,
androidId
,
idfv
,
deviceType
,
platform
,
value
)
}
}
}
//adx=iqiyi 且os=android的imeiMD5和oaidMD5去重设备, 分别和com.taobao.foractivation.227229和com.taobao.foractivation.227229_oaid做差集 2020.12.15
//2021.04.22下掉,该需求不需要了
// if("iqiyi".equals(exchanges)){
...
...
src/main/scala/mobvista/dmp/datasource/dsp/DspOrgLogEtlHoursDemo.scala
View file @
ca633a12
...
...
@@ -141,7 +141,7 @@ class DspOrgLogEtlHoursDemo extends CommonSparkJob with Serializable {
val
exitId
=
row
.
getAs
[
String
](
"exitid"
)
val
exchanges
=
row
.
getAs
[
String
](
"exchanges"
)
var
dealerid
=
""
if
(
"mopub"
.
equalsIgnoreCase
(
exchanges
)
||
"oppocn"
.
equalsIgnoreCase
(
exchanges
)
||
"inmobi"
.
equalsIgnoreCase
(
exchanges
)
||
"bes"
.
equalsIgnoreCase
(
exchanges
)
||
"iqiyi"
.
equalsIgnoreCase
(
exchanges
)
){
if
(
"mopub"
.
equalsIgnoreCase
(
exchanges
)
||
"oppocn"
.
equalsIgnoreCase
(
exchanges
)
||
"inmobi"
.
equalsIgnoreCase
(
exchanges
)
||
"bes"
.
equalsIgnoreCase
(
exchanges
)
||
"iqiyi"
.
equalsIgnoreCase
(
exchanges
)
||
"vivo"
.
equalsIgnoreCase
(
exchanges
)
){
val
ext3
=
row
.
getAs
[
String
](
"ext3"
)
if
(
StringUtils
.
isNotBlank
(
ext3
)
&&
ext3
.
startsWith
(
"{"
)){
try
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment