Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
943eb2bb
Commit
943eb2bb
authored
7 years ago
by
apoorvnaik
Committed by
Madhan Neethiraj
7 years ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-2198: fix for Hive Hook OOM for large notification messages
Signed-off-by:
Madhan Neethiraj
<
madhan@apache.org
>
parent
2ed39cce
master
No related merge requests found
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
12 additions
and
2 deletions
+12
-2
HiveHook.java
...ge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+12
-2
No files found.
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
View file @
943eb2bb
...
@@ -59,6 +59,7 @@ import java.net.MalformedURLException;
...
@@ -59,6 +59,7 @@ import java.net.MalformedURLException;
import
java.net.URI
;
import
java.net.URI
;
import
java.security.PrivilegedExceptionAction
;
import
java.security.PrivilegedExceptionAction
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Collection
;
import
java.util.Comparator
;
import
java.util.Comparator
;
import
java.util.Date
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.HashMap
;
...
@@ -698,7 +699,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
...
@@ -698,7 +699,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
}
}
colLineageProcessInstances
.
add
(
0
,
processReferenceable
);
colLineageProcessInstances
.
add
(
0
,
processReferenceable
);
entities
.
addAll
(
colLineageProcessInstances
);
entities
.
addAll
(
colLineageProcessInstances
);
event
.
addMessage
(
new
HookNotification
.
EntityUpdateRequest
(
event
.
getUser
(),
new
ArrayList
<>(
entities
)));
addEntityUpdateNotificationMessagess
(
event
,
entities
);
}
else
{
}
else
{
LOG
.
info
(
"Skipped query {} since it has no getInputs() or resulting getOutputs()"
,
event
.
getQueryStr
());
LOG
.
info
(
"Skipped query {} since it has no getInputs() or resulting getOutputs()"
,
event
.
getQueryStr
());
}
}
...
@@ -711,6 +713,13 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
...
@@ -711,6 +713,13 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
}
}
}
}
private
void
addEntityUpdateNotificationMessagess
(
final
HiveEventContext
event
,
final
Collection
<
Referenceable
>
entities
)
{
// process each entity as separate message to avoid running into OOM errors
for
(
Referenceable
entity
:
entities
)
{
event
.
addMessage
(
new
HookNotification
.
EntityUpdateRequest
(
event
.
getUser
(),
entity
));
}
}
private
<
T
extends
Entity
>
void
processHiveEntity
(
HiveMetaStoreBridge
dgiBridge
,
HiveEventContext
event
,
T
entity
,
Set
<
String
>
dataSetsProcessed
,
private
<
T
extends
Entity
>
void
processHiveEntity
(
HiveMetaStoreBridge
dgiBridge
,
HiveEventContext
event
,
T
entity
,
Set
<
String
>
dataSetsProcessed
,
SortedMap
<
T
,
Referenceable
>
dataSets
,
Set
<
Referenceable
>
entities
)
throws
AtlasHookException
{
SortedMap
<
T
,
Referenceable
>
dataSets
,
Set
<
Referenceable
>
entities
)
throws
AtlasHookException
{
try
{
try
{
...
@@ -801,7 +810,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
...
@@ -801,7 +810,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
entities
.
addAll
(
tables
.
values
());
entities
.
addAll
(
tables
.
values
());
entities
.
add
(
processReferenceable
);
entities
.
add
(
processReferenceable
);
event
.
addMessage
(
new
HookNotification
.
EntityUpdateRequest
(
event
.
getUser
(),
entities
));
addEntityUpdateNotificationMessagess
(
event
,
entities
);
}
}
}
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment