From 7fc276f609280ea51b782426a72d979613ae6707 Mon Sep 17 00:00:00 2001
From: Shwetha GS <sshivalingamurthy@hortonworks.com>
Date: Fri, 8 Jul 2016 11:32:13 +0530
Subject: [PATCH] ATLAS-902 Atlas throws exception due to null definition in Hive create table statement (svimal2106 via shwethags)
---
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java | 9 +++++++++
release-log.txt | 1 +
typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java | 8 +++++---
typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java | 15 ++++++++++++++-
typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java | 23 +++++++++++++++++++++++
typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala | 8 ++++----
6 files changed, 56 insertions(+), 8 deletions(-)
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 8ca47d9..e7ecbb1 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -368,6 +368,15 @@ public class HiveHookIT {
}
@Test
+ public void testEmptyStringAsValue() throws Exception{
+ String tableName = tableName();
+ String command = "create table " + tableName + "(id int, name string) row format delimited lines terminated by '\n' null defined as ''";
+ runCommand(command);
+ assertTableIsRegistered(DEFAULT_DB, tableName);
+
+ }
+
+ @Test
public void testDropAndRecreateCTASOutput() throws Exception {
String tableName = createTable();
String ctasTableName = "table" + random();
diff --git a/release-log.txt b/release-log.txt
index 8c9c1eb..00a7bf6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-902 Atlas throws exception due to null definition in Hive create table statement (svimal2106 via shwethags)
ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags)
ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags)
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
index 41b3427..3783c23 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
@@ -449,6 +449,10 @@ public class DataTypes {
if (val != null && (!(val instanceof String) || StringUtils.isNotEmpty((CharSequence) val))) {
return val.toString();
}
+
+ if (m.nullAllowed() && val != null){
+ return val.toString();
+ }
return convertNull(m);
}
@@ -584,9 +588,7 @@ public class DataTypes {
b.put(keyType.convert(e.getKey(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED),
- valueType.convert(e.getValue(),
- TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
- Multiplicity.REQUIRED));
+ valueType.convert(e.getValue(), Multiplicity.OPTIONAL));
}
return b.build();
} else {
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
index 64dc751..95c99e4 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
@@ -39,6 +39,7 @@ public abstract class BaseTest {
public static final String STRUCT_TYPE_1 = "t1";
public static final String STRUCT_TYPE_2 = "t2";
+ public static final String STRUCT_TYPE_3 = "t3";
public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
public static Struct createStruct() throws AtlasException {
@@ -62,6 +63,11 @@ public abstract class BaseTest {
hm.put("a", 1.0);
hm.put("b", 2.0);
s.set("o", hm);
+ s.set("p", "");
+ s.setNull("q");
+ Map<String, String> hm2 = Maps.newHashMap();
+ hm2.put("a", "");
+ s.set("r", hm2);
return s;
}
@@ -89,13 +95,20 @@ public abstract class BaseTest {
TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
+ TypesUtil.createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ TypesUtil.createOptionalAttrDef("p", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("q", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("r", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)));
System.out.println("defined structType = " + structType);
StructType recursiveStructType =
ts.defineStructType(STRUCT_TYPE_2, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
System.out.println("defined recursiveStructType = " + recursiveStructType);
+
+ StructType invalidStructType =
+ ts.defineStructType(STRUCT_TYPE_3, true, TypesUtil.createRequiredAttrDef("a",DataTypes.STRING_TYPE));
+ System.out.println("defined structType = " + invalidStructType);
}
protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition<TraitType>... tDefs)
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
index b49e5ce..e52962b 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
+import com.sun.source.tree.AssertTree;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
@@ -28,16 +29,21 @@ import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
public class StructTest extends TypeUpdateBaseTest {
StructType structType;
StructType recursiveStructType;
+ StructType invalidStructType;
@BeforeMethod
public void setup() throws Exception {
super.setup();
structType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
recursiveStructType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_2);
+ invalidStructType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_3);
}
@Test
@@ -60,10 +66,27 @@ public class StructTest extends TypeUpdateBaseTest {
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{a=1.0, b=2.0}\n" +
+ "\tp : \t\n" +
+ "\tq : \t<null>\n"+
+ "\tr : \t{a=}\n" +
"}");
}
@Test
+ public void testStructWithEmptyString() throws AtlasException{
+ try {
+ assertTrue(getTypeSystem().getTypeNames().contains("t3"));
+ Struct s = new Struct(invalidStructType.getName());
+ s.set("a", "");
+ ITypedStruct ts = invalidStructType.convert(s, Multiplicity.REQUIRED);
+ }
+ catch (AtlasException e){
+ String err = "org.apache.atlas.typesystem.types.ValueConversionException: Cannot convert value 'org.apache.atlas.typesystem.Struct@1ba02' to datatype t3";
+ Assert.assertEquals(e.toString(), err);
+ }
+ }
+
+ @Test
public void testRecursive() throws AtlasException {
Struct s1 = new Struct(recursiveStructType.getName());
s1.set("a", 1);
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
index 92489eb..931773d 100755
--- a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
@@ -48,7 +48,7 @@ class SerializationTest extends BaseTest {
val s: Struct = BaseTest.createStruct()
val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
- Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{a=1.0, b=2.0}\n}")
+ Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{a=1.0, b=2.0}\n\tp : \t\n\tq : \t<null>\n\tr : \t{a=}\n}")
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new BigDecimalSerializer + new BigIntegerSerializer
@@ -56,10 +56,10 @@ class SerializationTest extends BaseTest {
//Json representation
val ser = swrite(ts)
val ser1 = swrite(ts.toString)
- Assert.assertEquals(ser1, "\"{\\n\\ta : \\t1\\n\\tb : \\ttrue\\n\\tc : \\t1\\n\\td : \\t2\\n\\te : \\t1\\n\\tf : \\t1\\n\\tg : \\t1\\n\\th : \\t1.0\\n\\ti : \\t1.0\\n\\tj : \\t1\\n\\tk : \\t1\\n\\tl : \\t" + BaseTest.TEST_DATE + "\\n\\tm : \\t[1, 1]\\n\\tn : \\t[1.1, 1.1]\\n\\to : \\t{a=1.0, b=2.0}\\n}\"");
+ Assert.assertEquals(ser1, "\"{\\n\\ta : \\t1\\n\\tb : \\ttrue\\n\\tc : \\t1\\n\\td : \\t2\\n\\te : \\t1\\n\\tf : \\t1\\n\\tg : \\t1\\n\\th : \\t1.0\\n\\ti : \\t1.0\\n\\tj : \\t1\\n\\tk : \\t1\\n\\tl : \\t" + BaseTest.TEST_DATE + "\\n\\tm : \\t[1, 1]\\n\\tn : \\t[1.1, 1.1]\\n\\to : \\t{a=1.0, b=2.0}\\n\\tp : \\t\\n\\tq : \\t<null>\\n\\tr : \\t{a=}\\n}\"");
// Typed Struct read back
val ts1 = read[StructInstance](ser)
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n}")
+ Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n\tp : \t\n\tq : \t<null>\n\tr : \t{a=}\n}")
}
@Test def test2 {
@@ -74,7 +74,7 @@ class SerializationTest extends BaseTest {
{"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
"c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"a":1.0,"b":2.0}}""")
// Typed Struct read from string
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n}")
+ Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n\tp : \t<null>\n\tq : \t<null>\n\tr : \t<null>\n}")
}
@Test def testTrait {
--
libgit2 0.27.1