Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
mobvista-dmp
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
王金锋
mobvista-dmp
Commits
986ad516
Commit
986ad516
authored
Jul 26, 2021
by
WangJinfeng
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fix rtdmp
parent
c89c29dc
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
5 additions
and
2 deletions
+5
-2
RTDmpMain.scala
src/main/scala/mobvista/dmp/datasource/rtdmp/RTDmpMain.scala
+5
-2
No files found.
src/main/scala/mobvista/dmp/datasource/rtdmp/RTDmpMain.scala
View file @
986ad516
package
mobvista.dmp.datasource.rtdmp
import
com.alibaba.fastjson.
{
JSON
,
JSONObject
}
import
com.alibaba.fastjson.
JSONObject
import
mobvista.dmp.common.
{
CommonSparkJob
,
MobvistaConstant
}
import
mobvista.dmp.datasource.rtdmp.Constant.AudienceMerge
import
org.apache.commons.cli.
{
BasicParser
,
Options
}
...
...
@@ -85,7 +85,9 @@ class RTDmpMain extends CommonSparkJob with Serializable {
import
spark.implicits._
val
df
=
hour_rdd
.
fullOuterJoin
(
merge_rdd
).
map
(
t
=>
{
val
df
=
hour_rdd
.
fullOuterJoin
(
merge_rdd
)
.
mapPartitions
(
ts
=>
{
ts
.
map
(
t
=>
{
val
devid
=
t
.
_1
val
opt1
=
t
.
_2
.
_1
val
opt2
=
t
.
_2
.
_2
...
...
@@ -104,6 +106,7 @@ class RTDmpMain extends CommonSparkJob with Serializable {
.
retain
((
_
,
v
)
=>
v
.
compareTo
(
expire_time
)
>
0
)
AudienceMerge
(
devid
,
new
ObjectMapper
().
writeValueAsString
(
retain_old_audience
.
asJava
),
opt2
.
get
.
_2
,
opt2
.
get
.
_3
)
}
})
}).
filter
(
o
=>
{
!
MobvistaConstant
.
String2JSONObject
(
o
.
audience_map
).
isEmpty
})
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment