Skip to content

A
atlas
Commit bcfd2e7b by Venkatesh Seetharam
Options

ISSUE-32 Make TypeSystem a singleton. Contributed by Venkatesh Seetharam

parent 0c54f27e
Showing with 136 additions and 88 deletions
pom.xml
......@@ -440,6 +440,12 @@
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-throwingproviders</artifactId>
<version>3.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.5</version>
......
repository/pom.xml
......@@ -88,7 +88,6 @@
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-throwingproviders</artifactId>
<version>3.0</version>
</dependency>
<dependency>
......
repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
......@@ -53,7 +53,7 @@ public class DefaultMetadataService implements MetadataService {
@Inject
DefaultMetadataService(MetadataRepository repository) throws MetadataException {
this.typeSystem = new TypeSystem();
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
}
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepository.java
......@@ -78,7 +78,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
GraphBackedMetadataRepository(GraphService graphService) throws MetadataException {
this.instances = new ConcurrentHashMap<>();
this.graphService = graphService;
this.typeSystem = new TypeSystem();
this.typeSystem = TypeSystem.getInstance();
}
/**
......@@ -186,7 +186,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for (AttributeInfo attributeInfo : fields.values()) {
System.out.println("*** attributeInfo = " + attributeInfo);
final IDataType dataType = attributeInfo.dataType();
String attributeName = attributeInfo.name;
Object attributeValue = instance.get(attributeInfo.name);
switch (dataType.getTypeCategory()) {
......@@ -220,7 +219,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break;
case CLASS:
Id id = (Id) instance.get(attributeName);
Id id = (Id) instance.get(attributeInfo.name);
if (id != null) {
Vertex referenceVertex = idToVertexMap.get(id);
addEdge(entityVertex, referenceVertex, "references");
......
repository/src/test/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryTest.java
......@@ -49,7 +49,7 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
repositoryService = super.injector.getInstance(GraphBackedMetadataRepository.class);
repositoryService.start();
ts = new TypeSystem();
ts = TypeSystem.getInstance();
repo = new MemRepository(ts);
defineDeptEmployeeTypes(ts);
......@@ -197,4 +197,4 @@ public class GraphBackedMetadataRepositoryTest extends RepositoryModuleBaseTest
String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);
}
}
\ No newline at end of file
}
typesystem/pom.xml
......@@ -38,7 +38,7 @@
</prerequisites>
<properties>
<java.version>1.6</java.version>
<java.version>1.7</java.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.macros.version>2.0.1</scala.macros.version>
......@@ -183,6 +183,10 @@
<artifactId>fastutil</artifactId>
<version>${fastutil.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
</dependencies>
<build>
......
typesystem/src/main/java/org/apache/hadoop/metadata/MetadataService.java
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
......@@ -51,15 +50,6 @@ public class MetadataService {
return r;
}
public static TypeSystem getCurrentTypeSystem() throws MetadataException {
MetadataService m = currentSvc.get();
TypeSystem t = m == null ? null : m.getTypeSystem();
if ( t == null ) {
throw new MetadataException("No TypeSystem associated with current thread");
}
return t;
}
public MetadataService(IRepository repo, TypeSystem typeSystem) {
this.typeSystem = typeSystem;
this.repo = repo;
......
typesystem/src/main/java/org/apache/hadoop/metadata/storage/IRepository.java
......@@ -18,20 +18,20 @@
package org.apache.hadoop.metadata.storage;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.HierarchicalType;
import org.apache.hadoop.metadata.types.TraitType;
import java.text.DateFormat;
import java.util.List;
public interface IRepository {
/*
DateFormat getDateFormat();
DateFormat getTimestampFormat();
boolean allowNullsInCollections();
*/
ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException;
......
typesystem/src/main/java/org/apache/hadoop/metadata/storage/StructInstance.java
......@@ -105,7 +105,7 @@ public class StructInstance implements ITypedStruct {
if (val != null && val instanceof Id) {
ClassType clsType =
MetadataService.getCurrentTypeSystem().getDataType(ClassType.class, i.dataType().getName());
TypeSystem.getInstance().getDataType(ClassType.class, i.dataType().getName());
clsType.validateId((Id)val);
cVal = val;
} else {
......
typesystem/src/main/java/org/apache/hadoop/metadata/storage/memory/MemRepository.java
......@@ -21,18 +21,35 @@ package org.apache.hadoop.metadata.storage.memory;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.storage.*;
import org.apache.hadoop.metadata.types.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import org.apache.hadoop.metadata.storage.DiscoverInstances;
import org.apache.hadoop.metadata.storage.IRepository;
import org.apache.hadoop.metadata.storage.Id;
import org.apache.hadoop.metadata.storage.MapIds;
import org.apache.hadoop.metadata.storage.ReferenceableInstance;
import org.apache.hadoop.metadata.storage.RepositoryException;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.DataTypes;
import org.apache.hadoop.metadata.types.HierarchicalType;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.ObjectGraphWalker;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
public class MemRepository implements IRepository {
/*
public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd");
*/
final TypeSystem typeSystem;
/*
......@@ -43,9 +60,10 @@ public class MemRepository implements IRepository {
public MemRepository(TypeSystem typeSystem) {
this.typeSystem = typeSystem;
this.typeStores = new HashMap<String, HierarchicalTypeStore>();
this.typeStores = new HashMap<>();
}
/*
@Override
public DateFormat getDateFormat() {
return dateFormat;
......@@ -60,6 +78,7 @@ public class MemRepository implements IRepository {
public boolean allowNullsInCollections() {
return false;
}
*/
@Override
public Id newId(String typeName) {
......
typesystem/src/main/java/org/apache/hadoop/metadata/types/DataTypes.java
......@@ -19,24 +19,22 @@
package org.apache.hadoop.metadata.types;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableCollection.Builder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataService;
import org.apache.hadoop.metadata.storage.Id;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.storage.IRepository;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataService;
import org.apache.hadoop.metadata.storage.Id;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.text.ParseException;
import java.util.*;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
public class DataTypes {
......@@ -388,7 +386,7 @@ public class DataTypes {
return (Date) val;
} else if ( val instanceof String) {
try {
return MetadataService.getCurrentRepository().getDateFormat().parse((String)val);
return TypeSystem.getInstance().getDateFormat().parse((String) val);
} catch(ParseException ne) {
throw new ValueConversionException(this, val, ne);
}
......@@ -404,7 +402,7 @@ public class DataTypes {
@Override
public void output(Date val, Appendable buf, String prefix) throws MetadataException {
TypeUtils.outputVal(val == null ? "<null>" :
MetadataService.getCurrentRepository().getDateFormat().format(val), buf, prefix);
TypeSystem.getInstance().getDateFormat().format(val), buf, prefix);
}
public Date nullValue() {
......@@ -488,14 +486,16 @@ public class DataTypes {
ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder() : ImmutableList.builder();
while (it.hasNext() ) {
b.add(elemType.convert(it.next(),
r.allowNullsInCollections() ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
TypeSystem.getInstance().allowNullsInCollections()
? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
}
return b.build();
}
else {
try {
return ImmutableList.of(elemType.convert(val,
r.allowNullsInCollections() ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
TypeSystem.getInstance().allowNullsInCollections()
? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
} catch(Exception e) {
throw new ValueConversionException(this, val, e);
}
......@@ -592,9 +592,11 @@ public class DataTypes {
while (it.hasNext() ) {
Map.Entry e = it.next();
b.put(keyType.convert(e.getKey(),
r.allowNullsInCollections() ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED),
TypeSystem.getInstance().allowNullsInCollections()
? Multiplicity.OPTIONAL : Multiplicity.REQUIRED),
valueType.convert(e.getValue(),
r.allowNullsInCollections() ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
TypeSystem.getInstance().allowNullsInCollections()
? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
}
return b.build();
}
......
typesystem/src/main/java/org/apache/hadoop/metadata/types/FieldMapping.java
......@@ -116,7 +116,7 @@ public class FieldMapping {
outputFields(s, buf, fieldPrefix);
TypeSystem ts = MetadataService.getCurrentTypeSystem();
TypeSystem ts = TypeSystem.getInstance();
for(String sT : s.getTraits() ) {
TraitType tt = ts.getDataType(TraitType.class, sT);
......
typesystem/src/main/java/org/apache/hadoop/metadata/types/TypeSystem.java
......@@ -20,22 +20,40 @@ package org.apache.hadoop.metadata.types;
import com.google.common.collect.ImmutableList;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.TypesDef;
import javax.inject.Singleton;
import java.lang.reflect.Constructor;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
@Singleton
public class TypeSystem {
private Map<String, IDataType> types;
public TypeSystem() throws MetadataException {
types = new HashMap<String, IDataType>();
private TypeSystem() {
initialize();
}
/**
* This is only used for testing prurposes.
* @nonpublic
*/
public void reset() {
initialize();
}
private void initialize() {
types = new HashMap<>();
registerPrimitiveTypes();
}
private TypeSystem(TypeSystem ts) {
private static final TypeSystem INSTANCE = new TypeSystem();
public static TypeSystem getInstance() {
return INSTANCE;
}
public ImmutableList<String> getTypeNames() {
......@@ -87,27 +105,29 @@ public class TypeSystem {
boolean errorIfExists,
AttributeDefinition... attrDefs) throws MetadataException {
StructTypeDefinition structDef = new StructTypeDefinition(name, attrDefs);
Map<String, IDataType> newTypes = defineTypes(ImmutableList.<StructTypeDefinition>of(structDef),
defineTypes(ImmutableList.of(structDef),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
return getDataType(StructType.class, structDef.typeName);
}
public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef
) throws MetadataException {
Map<String, IDataType> newTypes = defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(traitDef),
public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef)
throws MetadataException {
defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(traitDef),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
return getDataType(TraitType.class, traitDef.typeName);
}
public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef
) throws MetadataException {
Map<String, IDataType> newTypes = defineTypes(ImmutableList.<StructTypeDefinition>of(),
public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef)
throws MetadataException {
defineTypes(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(classDef));
ImmutableList.of(classDef));
return getDataType(ClassType.class, classDef.typeName);
}
......@@ -115,7 +135,7 @@ public class TypeSystem {
public Map<String, IDataType> defineTraitTypes(HierarchicalTypeDefinition<TraitType>...traitDefs)
throws MetadataException {
TransientTypeSystem transientTypes = new TransientTypeSystem(ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>copyOf(traitDefs),
ImmutableList.copyOf(traitDefs),
ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
return transientTypes.defineTypes();
}
......@@ -180,11 +200,11 @@ public class TypeSystem {
final ImmutableList<StructTypeDefinition> structDefs;
final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs;
final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs;
Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<String, StructTypeDefinition>();
Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap =
new HashMap<String, HierarchicalTypeDefinition<TraitType>>();
new HashMap<>();
Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap =
new HashMap<String, HierarchicalTypeDefinition<ClassType>>();
new HashMap<>();
Set<String> transientTypes;
......@@ -197,20 +217,17 @@ public class TypeSystem {
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) {
super(TypeSystem.this);
this.structDefs = structDefs;
this.traitDefs = traitDefs;
this.classDefs = classDefs;
structNameToDefMap = new HashMap<String, StructTypeDefinition>();
traitNameToDefMap =
new HashMap<String, HierarchicalTypeDefinition<TraitType>>();
classNameToDefMap =
new HashMap<String, HierarchicalTypeDefinition<ClassType>>();
recursiveRefs = new ArrayList<AttributeInfo>();
recursiveArrayTypes = new ArrayList<DataTypes.ArrayType>();
recursiveMapTypes = new ArrayList<DataTypes.MapType>();
transientTypes = new LinkedHashSet<String>();
structNameToDefMap = new HashMap<>();
traitNameToDefMap = new HashMap<>();
classNameToDefMap = new HashMap<>();
recursiveRefs = new ArrayList<>();
recursiveArrayTypes = new ArrayList<>();
recursiveMapTypes = new ArrayList<>();
transientTypes = new LinkedHashSet<>();
}
private IDataType dataType(String name) {
......@@ -264,9 +281,10 @@ public class TypeSystem {
}
}
private <U extends HierarchicalType> void validateSuperTypes(Class<U> cls, HierarchicalTypeDefinition<U> def)
private <U extends HierarchicalType> void validateSuperTypes(Class<U> cls,
HierarchicalTypeDefinition<U> def)
throws MetadataException {
Set<String> s = new HashSet<String>();
Set<String> s = new HashSet<>();
ImmutableList<String> superTypes = def.superTypes;
for (String superTypeName : superTypes) {
......@@ -373,13 +391,13 @@ public class TypeSystem {
*/
private void step3() throws MetadataException {
List<TraitType> traitTypes = new ArrayList<TraitType>();
List<TraitType> traitTypes = new ArrayList<>();
for (String traitTypeName : traitNameToDefMap.keySet()) {
traitTypes.add(getDataType(TraitType.class, traitTypeName));
}
Collections.sort(traitTypes);
List<ClassType> classTypes = new ArrayList<ClassType>();
List<ClassType> classTypes = new ArrayList<>();
for (String classTypeName : classNameToDefMap.keySet()) {
classTypes.add(getDataType(ClassType.class, classTypeName));
}
......@@ -429,7 +447,7 @@ public class TypeSystem {
throw me;
}
Map<String, IDataType> newTypes = new HashMap<String, IDataType>();
Map<String, IDataType> newTypes = new HashMap<>();
for (String tName : transientTypes) {
newTypes.put(tName, dataType(tName));
......@@ -483,4 +501,13 @@ public class TypeSystem {
}
}
public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public DateFormat getDateFormat() {
return dateFormat;
}
public boolean allowNullsInCollections() {
return false;
}
}
typesystem/src/main/scala/org/apache/hadoop/metadata/cli/Console.scala
......@@ -44,7 +44,7 @@ class SampleILoop extends ILoop {
//intp = Console.in
val ts: TypeSystem = new TypeSystem
val ts: TypeSystem = TypeSystem.getInstance()
val mr: MemRepository = new MemRepository(ts)
val ms : MetadataService = new MetadataService(mr, ts)
MetadataService.setCurrentService(ms)
......
typesystem/src/main/scala/org/apache/hadoop/metadata/dsl/DynamicTypedStruct.scala
......@@ -32,5 +32,5 @@ class DynamicTypedStruct(val ts : ITypedStruct) extends Dynamic {
}
ts.set(name, value1)
}
def dataType = MetadataService.getCurrentTypeSystem.getDataType(classOf[StructType], ts.getTypeName)
def dataType = TypeSystem.getInstance().getDataType(classOf[StructType], ts.getTypeName)
}
typesystem/src/main/scala/org/apache/hadoop/metadata/dsl/package.scala
......@@ -42,7 +42,7 @@ package object dsl {
new BigDecimalSerializer + new BigIntegerSerializer
def service = MetadataService.getCurrentService
def ts = MetadataService.getCurrentTypeSystem
def ts = TypeSystem.getInstance
def repo = MetadataService.getCurrentRepository
val BOOLEAN_TYPE = DataTypes.BOOLEAN_TYPE
......
typesystem/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala
......@@ -51,9 +51,9 @@ object Main extends App {
implicit val system = ActorSystem("metadataservice")
val typSys = new TypeSystem
val typSys = TypeSystem.getInstance()
val memRepo = new MemRepository(typSys)
val api = system.actorOf(Props(new RestInterface(typSys, memRepo)), "httpInterface")
IO(Http) ! Http.Bind(listener = api, interface = host, port = port)
}
\ No newline at end of file
}
typesystem/src/test/java/org/apache/hadoop/metadata/BaseTest.java
......@@ -41,10 +41,15 @@ public abstract class BaseTest {
public static final String STRUCT_TYPE_1 = "t1";
public static final String STRUCT_TYPE_2 = "t2";
protected TypeSystem getTypeSystem() {
return TypeSystem.getInstance();
}
@Before
public void setup() throws MetadataException {
TypeSystem ts = new TypeSystem();
TypeSystem ts = TypeSystem.getInstance();
ts.reset();
MemRepository mr = new MemRepository(ts);
ms = new MetadataService(mr, ts);
MetadataService.setCurrentService(ms);
......
typesystem/src/test/java/org/apache/hadoop/metadata/ClassTest.java
package org.apache.hadoop.metadata;
import com.google.common.collect.ImmutableList;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.*;
import org.junit.Assert;
......@@ -17,7 +16,7 @@ public class ClassTest extends BaseTest {
@Test
public void test1() throws MetadataException {
TypeSystem ts = ms.getTypeSystem();
TypeSystem ts = getTypeSystem();
defineDeptEmployeeTypes(ts);
Referenceable hrDept = createDeptEg1(ts);
......
typesystem/src/test/scala/org/apache/hadoop/metadata/json/TypesSerializationTest.scala
......@@ -132,7 +132,8 @@ class TypesSerializationTest extends BaseTest with TypeHelpers {
val typesDef1 = TypesSerialization.fromJson(ser)
val ts1 = new TypeSystem()
val ts1 = TypeSystem.getInstance()
ts1.reset()
typesDef1.enumTypes.foreach( ts1.defineEnumType(_))
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java
......@@ -56,7 +56,7 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
public void setUp() throws Exception {
super.setUp();
typeSystem = new TypeSystem();
typeSystem = TypeSystem.getInstance();
typeDefinitions = createHiveTypes();
}
......@@ -183,4 +183,4 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
return typeDefinitions;
}
}
\ No newline at end of file
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment