Skip to content

A
atlas
Commit a2801f0e by Madhan Neethiraj Committed by Suma Shivaprasad
Options

ATLAS-1089: fix Storm hook to handle cyclic references in topology object

parent 9eafb165
Showing with 73 additions and 63 deletions
addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
......@@ -179,7 +179,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
private Referenceable createDataSet(String name, String topologyOwner,
Serializable instance,
Map stormConf, List<Referenceable> dependentEntities) throws IllegalAccessException {
Map<String, String> config = StormTopologyUtil.getFieldValues(instance, true);
Map<String, String> config = StormTopologyUtil.getFieldValues(instance, true, null);
String clusterName = null;
Referenceable dataSetReferenceable;
......@@ -298,7 +298,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
stormSpout.get_spout_object().get_serialized_java(), Serializable.class);
spoutReferenceable.set("driverClass", instance.getClass().getName());
Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true);
Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);
spoutReferenceable.set("conf", flatConfigMap);
return spoutReferenceable;
......@@ -322,7 +322,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
stormBolt.get_bolt_object().get_serialized_java(), Serializable.class);
boltReferenceable.set("driverClass", instance.getClass().getName());
Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true);
Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);
boltReferenceable.set("conf", flatConfigMap);
return boltReferenceable;
......
addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormTopologyUtil.java
......@@ -127,86 +127,95 @@ public final class StormTopologyUtil {
}
public static Map<String, String> getFieldValues(Object instance,
boolean prependClassName)
boolean prependClassName,
Set<Object> objectsToSkip)
throws IllegalAccessException {
Class clazz = instance.getClass();
if (objectsToSkip == null) {
objectsToSkip = new HashSet<Object>();
}
Map<String, String> output = new HashMap<>();
for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
Field[] fields = c.getDeclaredFields();
for (Field field : fields) {
if (java.lang.reflect.Modifier.isStatic(field.getModifiers())) {
continue;
}
String key;
if (prependClassName) {
key = String.format("%s.%s", clazz.getSimpleName(), field.getName());
} else {
key = field.getName();
}
if (objectsToSkip.add(instance)) {
Class clazz = instance.getClass();
for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
Field[] fields = c.getDeclaredFields();
for (Field field : fields) {
if (java.lang.reflect.Modifier.isStatic(field.getModifiers())) {
continue;
}
boolean accessible = field.isAccessible();
if (!accessible) {
field.setAccessible(true);
}
Object fieldVal = field.get(instance);
if (fieldVal == null) {
continue;
} else if (fieldVal.getClass().isPrimitive() ||
isWrapperType(fieldVal.getClass())) {
if (toString(fieldVal, false).isEmpty()) continue;
output.put(key, toString(fieldVal, false));
} else if (isMapType(fieldVal.getClass())) {
//TODO: check if it makes more sense to just stick to json
// like structure instead of a flatten output.
Map map = (Map) fieldVal;
for (Object entry : map.entrySet()) {
Object mapKey = ((Map.Entry) entry).getKey();
Object mapVal = ((Map.Entry) entry).getValue();
String keyStr = getString(mapKey, false);
String valStr = getString(mapVal, false);
if ((valStr == null) || (valStr.isEmpty())) {
continue;
} else {
output.put(String.format("%s.%s", key, keyStr), valStr);
}
String key;
if (prependClassName) {
key = String.format("%s.%s", clazz.getSimpleName(), field.getName());
} else {
key = field.getName();
}
} else if (isCollectionType(fieldVal.getClass())) {
//TODO check if it makes more sense to just stick to
// json like structure instead of a flatten output.
Collection collection = (Collection) fieldVal;
if (collection.size()==0) continue;
String outStr = "";
for (Object o : collection) {
outStr += getString(o, false) + ",";
boolean accessible = field.isAccessible();
if (!accessible) {
field.setAccessible(true);
}
if (outStr.length() > 0) {
outStr = outStr.substring(0, outStr.length() - 1);
Object fieldVal = field.get(instance);
if (fieldVal == null) {
continue;
} else if (fieldVal.getClass().isPrimitive() ||
isWrapperType(fieldVal.getClass())) {
if (toString(fieldVal, false).isEmpty()) continue;
output.put(key, toString(fieldVal, false));
} else if (isMapType(fieldVal.getClass())) {
//TODO: check if it makes more sense to just stick to json
// like structure instead of a flatten output.
Map map = (Map) fieldVal;
for (Object entry : map.entrySet()) {
Object mapKey = ((Map.Entry) entry).getKey();
Object mapVal = ((Map.Entry) entry).getValue();
String keyStr = getString(mapKey, false, objectsToSkip);
String valStr = getString(mapVal, false, objectsToSkip);
if ((valStr == null) || (valStr.isEmpty())) {
continue;
} else {
output.put(String.format("%s.%s", key, keyStr), valStr);
}
}
} else if (isCollectionType(fieldVal.getClass())) {
//TODO check if it makes more sense to just stick to
// json like structure instead of a flatten output.
Collection collection = (Collection) fieldVal;
if (collection.size() == 0) continue;
String outStr = "";
for (Object o : collection) {
outStr += getString(o, false, objectsToSkip) + ",";
}
if (outStr.length() > 0) {
outStr = outStr.substring(0, outStr.length() - 1);
}
output.put(key, String.format("%s", outStr));
} else {
Map<String, String> nestedFieldValues = getFieldValues(fieldVal, false, objectsToSkip);
for (Map.Entry<String, String> entry : nestedFieldValues.entrySet()) {
output.put(String.format("%s.%s", key, entry.getKey()), entry.getValue());
}
}
output.put(key, String.format("%s", outStr));
} else {
Map<String, String> nestedFieldValues = getFieldValues(fieldVal, false);
for (Map.Entry<String, String> entry : nestedFieldValues.entrySet()) {
output.put(String.format("%s.%s", key, entry.getKey()), entry.getValue());
if (!accessible) {
field.setAccessible(false);
}
}
if (!accessible) {
field.setAccessible(false);
}
}
}
return output;
}
private static String getString(Object instance,
boolean wrapWithQuote) throws IllegalAccessException {
boolean wrapWithQuote,
Set<Object> objectsToSkip) throws IllegalAccessException {
if (instance == null) {
return null;
} else if (instance.getClass().isPrimitive() || isWrapperType(instance.getClass())) {
return toString(instance, wrapWithQuote);
} else {
return getString(getFieldValues(instance, false), wrapWithQuote);
return getString(getFieldValues(instance, false, objectsToSkip), wrapWithQuote);
}
}
......
release-log.txt
......@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
ATLAS-1089 Storm hook should handle cyclic references in topology object (mneethiraj via sumasai)
ATLAS-1086 Build failure in hive-bridge after security fixes in ATLAS-762 (sumasai)
ATLAS-1088 Fix /search api to default to fulltext on dsl failure (sumasai)
ATLAS-762 Assertion in NegativeSSLAndKerberosTest.testUnsecuredClient needs to be hardened (nixonrodrigues via sumasai)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment