ATLAS-902 Atlas throws exception due to null definition in Hive create table…

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java

@@ -368,6 +368,15 @@ public class HiveHookIT {
     }
 
     @Test
+    public void testEmptyStringAsValue() throws Exception{
+        String tableName = tableName();
+        String command = "create table " + tableName + "(id int, name string) row format delimited lines terminated by '\n' null defined as ''";
+        runCommand(command);
+        assertTableIsRegistered(DEFAULT_DB, tableName);
+
+    }
+
+    @Test
     public void testDropAndRecreateCTASOutput() throws Exception {
         String tableName = createTable();
         String ctasTableName = "table" + random();

release-log.txt

@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
 
 
 ALL CHANGES:
+ATLAS-902 Atlas throws exception due to null definition in Hive create table statement (svimal2106 via shwethags)
 ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
 ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags)
 ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags)

typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java

@@ -449,6 +449,10 @@ public class DataTypes {
             if (val != null && (!(val instanceof String) || StringUtils.isNotEmpty((CharSequence) val))) {
                 return val.toString();
             }
+
+            if (m.nullAllowed() && val != null){
+                return val.toString();
+            }
             return convertNull(m);
         }
 
@@ -584,9 +588,7 @@ public class DataTypes {
                         b.put(keyType.convert(e.getKey(),
                                         TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
                                                 Multiplicity.REQUIRED),
-                                        valueType.convert(e.getValue(),
-                                        TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
-                                                Multiplicity.REQUIRED));
+                                        valueType.convert(e.getValue(), Multiplicity.OPTIONAL));
                     }
                     return b.build();
                 } else {

typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java

@@ -39,6 +39,7 @@ public abstract class BaseTest {
 
     public static final String STRUCT_TYPE_1 = "t1";
     public static final String STRUCT_TYPE_2 = "t2";
+    public static final String STRUCT_TYPE_3 = "t3";
     public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
 
     public static Struct createStruct() throws AtlasException {
@@ -62,6 +63,11 @@ public abstract class BaseTest {
         hm.put("a", 1.0);
         hm.put("b", 2.0);
         s.set("o", hm);
+        s.set("p", "");
+        s.setNull("q");
+        Map<String, String> hm2 = Maps.newHashMap();
+        hm2.put("a", "");
+        s.set("r", hm2);
         return s;
     }
 
@@ -89,13 +95,20 @@ public abstract class BaseTest {
                         TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
                         TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
                         TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
-                        TypesUtil.createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
+                        TypesUtil.createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+                        TypesUtil.createOptionalAttrDef("p", DataTypes.STRING_TYPE),
+                        TypesUtil.createOptionalAttrDef("q", DataTypes.STRING_TYPE),
+                        TypesUtil.createOptionalAttrDef("r", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)));
         System.out.println("defined structType = " + structType);
 
         StructType recursiveStructType =
                 ts.defineStructType(STRUCT_TYPE_2, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
                         TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
         System.out.println("defined recursiveStructType = " + recursiveStructType);
+
+        StructType invalidStructType =
+                ts.defineStructType(STRUCT_TYPE_3, true, TypesUtil.createRequiredAttrDef("a",DataTypes.STRING_TYPE));
+        System.out.println("defined structType = " + invalidStructType);
     }
 
     protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition<TraitType>... tDefs)

typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java

@@ -19,6 +19,7 @@
 package org.apache.atlas.typesystem.types;
 
 import com.google.common.collect.ImmutableList;
+import com.sun.source.tree.AssertTree;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.typesystem.ITypedStruct;
 import org.apache.atlas.typesystem.Struct;
@@ -28,16 +29,21 @@ import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
 public class StructTest extends TypeUpdateBaseTest {
 
     StructType structType;
     StructType recursiveStructType;
+    StructType invalidStructType;
 
     @BeforeMethod
     public void setup() throws Exception {
         super.setup();
         structType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
         recursiveStructType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_2);
+        invalidStructType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_3);
     }
 
     @Test
@@ -60,10 +66,27 @@ public class StructTest extends TypeUpdateBaseTest {
                 "\tm : \t[1, 1]\n" +
                 "\tn : \t[1.1, 1.1]\n" +
                 "\to : \t{a=1.0, b=2.0}\n" +
+                "\tp : \t\n" +
+                "\tq : \t<null>\n"+
+                "\tr : \t{a=}\n" +
                 "}");
     }
 
     @Test
+    public void testStructWithEmptyString() throws AtlasException{
+        try {
+            assertTrue(getTypeSystem().getTypeNames().contains("t3"));
+            Struct s = new Struct(invalidStructType.getName());
+            s.set("a", "");
+            ITypedStruct ts = invalidStructType.convert(s, Multiplicity.REQUIRED);
+        }
+        catch (AtlasException e){
+            String err = "org.apache.atlas.typesystem.types.ValueConversionException: Cannot convert value 'org.apache.atlas.typesystem.Struct@1ba02' to datatype t3";
+            Assert.assertEquals(e.toString(), err);
+        }
+    }
+
+    @Test
     public void testRecursive() throws AtlasException {
         Struct s1 = new Struct(recursiveStructType.getName());
         s1.set("a", 1);

typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala

@@ -48,7 +48,7 @@ class SerializationTest extends BaseTest {
         val s: Struct = BaseTest.createStruct()
         val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
 
-        Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{a=1.0, b=2.0}\n}")
+        Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{a=1.0, b=2.0}\n\tp : \t\n\tq : \t<null>\n\tr : \t{a=}\n}")
 
         implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
             new BigDecimalSerializer + new BigIntegerSerializer
@@ -56,10 +56,10 @@ class SerializationTest extends BaseTest {
         //Json representation
         val ser = swrite(ts)
         val ser1 = swrite(ts.toString)
-        Assert.assertEquals(ser1, "\"{\\n\\ta : \\t1\\n\\tb : \\ttrue\\n\\tc : \\t1\\n\\td : \\t2\\n\\te : \\t1\\n\\tf : \\t1\\n\\tg : \\t1\\n\\th : \\t1.0\\n\\ti : \\t1.0\\n\\tj : \\t1\\n\\tk : \\t1\\n\\tl : \\t" + BaseTest.TEST_DATE + "\\n\\tm : \\t[1, 1]\\n\\tn : \\t[1.1, 1.1]\\n\\to : \\t{a=1.0, b=2.0}\\n}\"");
+        Assert.assertEquals(ser1, "\"{\\n\\ta : \\t1\\n\\tb : \\ttrue\\n\\tc : \\t1\\n\\td : \\t2\\n\\te : \\t1\\n\\tf : \\t1\\n\\tg : \\t1\\n\\th : \\t1.0\\n\\ti : \\t1.0\\n\\tj : \\t1\\n\\tk : \\t1\\n\\tl : \\t" + BaseTest.TEST_DATE + "\\n\\tm : \\t[1, 1]\\n\\tn : \\t[1.1, 1.1]\\n\\to : \\t{a=1.0, b=2.0}\\n\\tp : \\t\\n\\tq : \\t<null>\\n\\tr : \\t{a=}\\n}\"");
         // Typed Struct read back
         val ts1 = read[StructInstance](ser)
-        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n}")
+        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n\tp : \t\n\tq : \t<null>\n\tr : \t{a=}\n}")
     }
 
     @Test def test2 {
@@ -74,7 +74,7 @@ class SerializationTest extends BaseTest {
         {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
         "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"a":1.0,"b":2.0}}""")
         // Typed Struct read from string
-        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n}")
+        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n\tp : \t<null>\n\tq : \t<null>\n\tr : \t<null>\n}")
     }
 
     @Test def testTrait {