Skip to content

A
atlas
Commit 774975c9 by Vimal Committed by Madhan Neethiraj
Options

ATLAS-916: Return System Attributes in get entity definition

parent 95083cb0
Showing with 367 additions and 22 deletions
catalog/src/main/java/org/apache/atlas/catalog/DefaultPropertyMapper.java
......@@ -138,5 +138,11 @@ public class DefaultPropertyMapper implements PropertyMapper {
m_qualifiedToCleanMap.put(Constants.STATE_PROPERTY_KEY, "state");
m_cleanToQualifiedMap.put("state", Constants.STATE_PROPERTY_KEY);
m_qualifiedToCleanMap.put(Constants.CREATED_BY_KEY, "created_by");
m_cleanToQualifiedMap.put("created_by", Constants.CREATED_BY_KEY);
m_qualifiedToCleanMap.put(Constants.MODIFIED_BY_KEY, "modified_by");
m_cleanToQualifiedMap.put("modified_by", Constants.MODIFIED_BY_KEY);
}
}
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -63,6 +63,8 @@ public final class Constants {
public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
public static final String STATE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "state";
public static final String CREATED_BY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "createdBy";
public static final String MODIFIED_BY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy";
public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-916 Return System Attributes in get entity definition (svimal2106)
ATLAS-1242 update TypesResource API implementation to use new v2 TypesREST API
ATLAS-1306 bootstrap type-load ignores model file contents if a type in the file already exists
ATLAS-1299 The project org.apache.atlas:atlas-hbase-client-shaded - build error (shwethags)
......
repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
......@@ -397,6 +397,7 @@ public abstract class DeleteHandler {
RequestContext requestContext = RequestContext.get();
GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime());
GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
requestContext.recordEntityUpdate(outId);
}
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -238,6 +238,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
} catch (RepositoryException e) {
throw e;
......@@ -293,6 +294,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
}
@Override
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -88,7 +88,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
/**
* Adds index for properties of a given type when its added before any instances are added.
*/
public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler,
public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler,
TypeDefChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
......@@ -154,17 +154,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createIndexes(management, Constants.GUID_PROPERTY_KEY, String.class, true,
AtlasCardinality.SINGLE, true, true);
// create a composite index for entity creation timestamp
createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, AtlasCardinality.SINGLE, true, true);
// Add creation_timestamp property to Vertex Index (mixed index)
createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, AtlasCardinality.SINGLE, false, false);
// Add modification_timestamp property to Vertex Index (mixed index)
createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false,
AtlasCardinality.SINGLE, false, false);
// create a mixed index for entity state. Set systemProperty flag deliberately to false
// so that it doesnt create a composite index which has issues with
// titan 0.5.4 - Refer https://groups.google.com/forum/#!searchin/aureliusgraphs/hemanth/aureliusgraphs/bx7T843mzXU/fjAsclx7GAAJ
createIndexes(management, Constants.STATE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE, false, false);
// create a composite index for entity modification timestamp
createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false,
AtlasCardinality.SINGLE, false, false);
// Create a composite and mixed index for created by property
createIndexes(management, Constants.CREATED_BY_KEY, String.class, false,
AtlasCardinality.SINGLE, true, true);
// Create a composite and mixed index for modified by property
createIndexes(management, Constants.MODIFIED_BY_KEY, String.class, false,
AtlasCardinality.SINGLE, true, true);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE,
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.UUID;
import java.util.Date;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
......@@ -159,6 +160,8 @@ public final class GraphHelper {
setProperty(vertexWithoutIdentity, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
setProperty(vertexWithoutIdentity, Constants.CREATED_BY_KEY, RequestContext.get().getUser());
setProperty(vertexWithoutIdentity, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
return vertexWithoutIdentity;
}
......@@ -169,6 +172,8 @@ public final class GraphHelper {
setProperty(edge, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
setProperty(edge, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
setProperty(edge, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
setProperty(edge, Constants.CREATED_BY_KEY, RequestContext.get().getUser());
setProperty(edge, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
LOG.debug("Added {}", string(edge));
return edge;
......@@ -517,6 +522,24 @@ public final class GraphHelper {
return element.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
}
//Added conditions in fetching system attributes to handle test failures in GremlinTest where these properties are not set
public static String getCreatedByAsString(AtlasElement element){
return element.getProperty(Constants.CREATED_BY_KEY, String.class);
}
public static String getModifiedByAsString(AtlasElement element){
return element.getProperty(Constants.MODIFIED_BY_KEY, String.class);
}
public static long getCreatedTime(AtlasElement element){
return element.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
}
public static long getModifiedTime(AtlasElement element){
return element.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
}
/**
* For the given type, finds an unique attribute and checks if there is an existing instance with the same
* unique value
......@@ -857,7 +880,9 @@ public final class GraphHelper {
switch (field) {
case Constants.STATE_PROPERTY_KEY:
case Constants.GUID_PROPERTY_KEY:
return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
case Constants.CREATED_BY_KEY:
case Constants.MODIFIED_BY_KEY:
return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
case Constants.TIMESTAMP_PROPERTY_KEY:
case Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY:
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
......@@ -36,7 +36,10 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
......@@ -70,6 +73,13 @@ public final class GraphToTypedInstanceMapper {
String typeName = GraphHelper.getSingleValuedProperty(instanceVertex, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
List<String> traits = GraphHelper.getTraitNames(instanceVertex);
String state = GraphHelper.getStateAsString(instanceVertex);
String createdBy = GraphHelper.getCreatedByAsString(instanceVertex);
String modifiedBy = GraphHelper.getModifiedByAsString(instanceVertex);
Date createdTime = new Date(GraphHelper.getCreatedTime(instanceVertex));
Date modifiedTime = new Date(GraphHelper.getModifiedTime(instanceVertex));
AtlasSystemAttributes systemAttributes = new AtlasSystemAttributes(createdBy, modifiedBy, createdTime, modifiedTime);
LOG.debug("Found createdBy : {} modifiedBy : {} createdTime: {} modifedTime: {}", createdBy, modifiedBy, createdTime, modifiedTime);
Id id = new Id(guid, (Integer) GraphHelper.getProperty(instanceVertex, Constants.VERSION_PROPERTY_KEY),
typeName, state);
......@@ -77,7 +87,7 @@ public final class GraphToTypedInstanceMapper {
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance typedInstance =
classType.createInstance(id, traits.toArray(new String[traits.size()]));
classType.createInstance(id, systemAttributes, traits.toArray(new String[traits.size()]));
mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
......
repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
......@@ -27,6 +27,7 @@ import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.TypeSystem;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
public class SoftDeleteHandler extends DeleteHandler {
......@@ -45,6 +46,7 @@ public class SoftDeleteHandler extends DeleteHandler {
GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, MODIFIED_BY_KEY, RequestContext.get().getUser());
}
}
}
......@@ -59,6 +61,7 @@ public class SoftDeleteHandler extends DeleteHandler {
GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper
.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
GraphHelper.setProperty(edge, MODIFIED_BY_KEY, RequestContext.get().getUser());
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -187,6 +187,8 @@ public final class TypedInstanceToGraphMapper {
}
GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
LOG.debug("Setting modifiedBy: {} and modifiedTime: {}", RequestContext.get().getUser(), RequestContext.get().getRequestTime());
}
void mapAttributeToVertex(ITypedInstance typedInstance, AtlasVertex instanceVertex,
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -59,6 +59,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Date;
import java.text.SimpleDateFormat;
import javax.inject.Inject;
......@@ -67,6 +69,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAt
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
......@@ -167,6 +170,24 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
assertNotNull(rows);
assertEquals(rows.length(), 1);
//Assert system attributes are not null
JSONObject sys_attributes = (JSONObject)rows.getJSONObject(0).get("$systemAttributes$");
assertNotNull(sys_attributes.get("createdBy"));
assertNotNull(sys_attributes.get("modifiedBy"));
assertNotNull(sys_attributes.get("createdTime"));
assertNotNull(sys_attributes.get("modifiedTime"));
//Assert that createdTime and modifiedTime are valid dates
String createdTime = (String) sys_attributes.get("createdTime");
String modifiedTime = (String) sys_attributes.get("modifiedTime");
final String outputFormat = "EEE MMM dd HH:mm:ss z yyyy";
SimpleDateFormat df = new SimpleDateFormat(outputFormat);
Date createdDate = df.parse(createdTime);
Date modifiedDate = df.parse(modifiedTime);
assertNotNull(createdDate);
assertNotNull(modifiedDate);
final String testTs = "\"2011-11-01T02:35:58.440Z\"";
dslQuery = "Department where " + Constants.TIMESTAMP_PROPERTY_KEY + " > " + testTs;
jsonResults = searchByDSL(dslQuery);
......@@ -191,6 +212,27 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
assertNotNull(rows);
assertEquals(rows.length(), 1);
dslQuery = "from Department select " + Constants.CREATED_BY_KEY;
jsonResults = searchByDSL(dslQuery);
assertNotNull(jsonResults);
results = new JSONObject(jsonResults);
assertEquals(results.length(), 3);
rows = results.getJSONArray("rows");
assertNotNull(rows);
assertEquals(rows.length(), 1);
dslQuery = "from Department select " + Constants.MODIFIED_BY_KEY;
jsonResults = searchByDSL(dslQuery);
assertNotNull(jsonResults);
results = new JSONObject(jsonResults);
assertEquals(results.length(), 3);
rows = results.getJSONArray("rows");
assertNotNull(rows);
assertEquals(rows.length(), 1);
}
@Test
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -43,6 +43,7 @@ import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TraitNotFoundException;
import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
......@@ -207,6 +208,11 @@ public class GraphBackedMetadataRepositoryTest {
//entity state should be active by default
Assert.assertEquals(entity.getId().getState(), Id.EntityState.ACTIVE);
//System attributes created time and modified time should not be null
AtlasSystemAttributes systemAttributes = entity.getSystemAttributes();
Assert.assertNotNull(systemAttributes.createdTime);
Assert.assertNotNull(systemAttributes.modifiedTime);
}
@Test(expectedExceptions = EntityNotFoundException.class)
......
typesystem/src/main/java/org/apache/atlas/typesystem/IReferenceableInstance.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes;
import org.apache.atlas.typesystem.persistence.Id;
/**
......@@ -32,4 +33,6 @@ public interface IReferenceableInstance extends IStruct {
Id getId();
IStruct getTrait(String typeName);
AtlasSystemAttributes getSystemAttributes();
}
typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
......@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes;
import org.apache.atlas.typesystem.persistence.Id;
import java.util.HashMap;
......@@ -36,6 +37,7 @@ public class Referenceable extends Struct implements IReferenceableInstance {
private Id id;
private final ImmutableMap<String, IStruct> traits;
private final ImmutableList<String> traitNames;
private AtlasSystemAttributes systemAttributes;
public Referenceable(String typeName, String... traitNames) {
super(typeName);
......@@ -46,6 +48,7 @@ public class Referenceable extends Struct implements IReferenceableInstance {
b.put(t, new Struct(t));
}
traits = b.build();
this.systemAttributes = new AtlasSystemAttributes();
}
public Referenceable(String typeName, Map<String, Object> values) {
......@@ -53,6 +56,7 @@ public class Referenceable extends Struct implements IReferenceableInstance {
id = new Id(typeName);
traitNames = ImmutableList.of();
traits = ImmutableMap.of();
this.systemAttributes = new AtlasSystemAttributes();
}
public Referenceable(String guid, String typeName, Map<String, Object> values) {
......@@ -60,6 +64,7 @@ public class Referenceable extends Struct implements IReferenceableInstance {
id = new Id(guid, 0, typeName);
traitNames = ImmutableList.of();
traits = ImmutableMap.of();
this.systemAttributes = new AtlasSystemAttributes();
}
/**
......@@ -75,6 +80,7 @@ public class Referenceable extends Struct implements IReferenceableInstance {
id = new Id(guid, 0, typeName);
traitNames = ImmutableList.copyOf(_traitNames);
traits = ImmutableMap.copyOf(_traits);
this.systemAttributes = new AtlasSystemAttributes();
}
/**
......@@ -90,6 +96,23 @@ public class Referenceable extends Struct implements IReferenceableInstance {
this.id = id;
traitNames = ImmutableList.copyOf(_traitNames);
traits = ImmutableMap.copyOf(_traits);
this.systemAttributes = new AtlasSystemAttributes();
}
/**
* Not public - only use during deserialization
* @param id entity id
* @param typeName the type name
* @param values the entity attribute values
*/
@InterfaceAudience.Private
public Referenceable(Id id, String typeName, Map<String, Object> values, List<String> _traitNames,
Map<String, IStruct> _traits, AtlasSystemAttributes systemAttributes) {
super(typeName, values);
this.id = id;
traitNames = ImmutableList.copyOf(_traitNames);
traits = ImmutableMap.copyOf(_traits);
this.systemAttributes = systemAttributes;
}
/**
......@@ -130,6 +153,11 @@ public class Referenceable extends Struct implements IReferenceableInstance {
return traits.get(typeName);
}
@Override
public AtlasSystemAttributes getSystemAttributes(){
return systemAttributes;
}
/**
* Matches traits, values associated with this Referenceable and skips the id match
* @param o The Referenceable which needs to be matched with
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/AtlasSystemAttributes.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.typesystem.persistence;
import org.apache.atlas.typesystem.types.TypeSystem;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class AtlasSystemAttributes {
public String createdBy;
public String modifiedBy;
public Date createdTime;
public Date modifiedTime;
public SimpleDateFormat simpleDateFormat = TypeSystem.getInstance().getDateFormat();
public AtlasSystemAttributes(String createdBy, String modifiedBy, Date createdTime, Date modifiedTime){
this.createdBy = createdBy;
this.modifiedBy = modifiedBy;
this.createdTime = createdTime;
this.modifiedTime = modifiedTime;
}
public AtlasSystemAttributes(){
super();
}
public AtlasSystemAttributes(String createdBy, String modifiedBy, String createdTime, String modifiedTime){
this.createdBy = createdBy;
this.modifiedBy = modifiedBy;
try{
this.createdTime = simpleDateFormat.parse(createdTime);
}catch (ParseException e){
//this.createdTime = new Date(0);
}
try{
this.modifiedTime = simpleDateFormat.parse(modifiedTime);
}catch (ParseException e){
//this.modifiedTime = new Date(0);
}
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
AtlasSystemAttributes sys_attr = (AtlasSystemAttributes) o;
if (!createdBy.equals(sys_attr.createdBy)) {
return false;
}
if (!modifiedBy.equals(sys_attr.modifiedBy)) {
return false;
}
if (!createdTime.equals(sys_attr.createdTime)) {
return false;
}
if(!modifiedTime.equals(sys_attr.modifiedTime)){
return false;
}
return true;
}
@Override
public int hashCode() {
int result = createdBy.hashCode();
result = 31 * result + modifiedBy.hashCode();
result = 31 * result + createdTime.hashCode();
result = 31 * result + modifiedTime.hashCode();
return result;
}
public String getCreatedBy(){
return createdBy;
}
public String getModifiedBy(){
return modifiedBy;
}
public Date getCreatedTime(){
return createdTime;
}
public Date getModifiedTime(){
return modifiedTime;
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
......@@ -45,6 +45,7 @@ public class Id implements ITypedReferenceableInstance {
public final int version;
public EntityState state;
private static AtomicLong s_nextId = new AtomicLong(System.nanoTime());
public final AtlasSystemAttributes systemAttributes;
public Id(String id, int version, String typeName, String state) {
id = ParamChecker.notEmpty(id, "id");
......@@ -58,6 +59,7 @@ public class Id implements ITypedReferenceableInstance {
} else {
this.state = EntityState.valueOf(state.toUpperCase());
}
this.systemAttributes = new AtlasSystemAttributes();
}
public Id(String id, int version, String typeName) {
......@@ -105,6 +107,11 @@ public class Id implements ITypedReferenceableInstance {
return String.format("id[type=%s guid=%s state=%s]", typeName, id, state);
}
@Override
public AtlasSystemAttributes getSystemAttributes(){
return systemAttributes;
}
public String getClassName() {
return typeName;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
......@@ -45,9 +45,10 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
private final ImmutableMap<String, ITypedStruct> traits;
private final ImmutableList<String> traitNames;
private Id id;
private AtlasSystemAttributes systemAttributes;
public ReferenceableInstance(Id id, String dataTypeName, FieldMapping fieldMapping, boolean[] nullFlags,
public ReferenceableInstance(Id id, String dataTypeName, AtlasSystemAttributes systemAttributes, FieldMapping fieldMapping, boolean[] nullFlags,
boolean[] bools, byte[] bytes, short[] shorts, int[] ints, long[] longs, float[] floats, double[] doubles,
BigDecimal[] bigDecimals, BigInteger[] bigIntegers, Date[] dates, String[] strings,
ImmutableList<Object>[] arrays, ImmutableMap<Object, Object>[] maps, StructInstance[] structs,
......@@ -61,6 +62,12 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
b.add(t);
}
this.traitNames = b.build();
if(systemAttributes == null){
this.systemAttributes = new AtlasSystemAttributes();
}
else{
this.systemAttributes = systemAttributes;
}
}
@Override
......@@ -78,6 +85,11 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
return traits.get(typeName);
}
@Override
public AtlasSystemAttributes getSystemAttributes(){
return systemAttributes;
}
/**
* @nopub
* @param id
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
......@@ -31,6 +31,7 @@ import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.persistence.StructInstance;
......@@ -132,7 +133,7 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
}
ITypedReferenceableInstance tr =
r != null ? createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0])) :
r != null ? createInstanceWithTraits(id, null, r, r.getTraits().toArray(new String[0])) :
createInstance(id);
if (id != null && id.isAssigned()) {
......@@ -180,10 +181,14 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
}
public ITypedReferenceableInstance createInstance(Id id, String... traitNames) throws AtlasException {
return createInstanceWithTraits(id, null, traitNames);
return createInstanceWithTraits(id, null, null, traitNames);
}
public ITypedReferenceableInstance createInstanceWithTraits(Id id, Referenceable r, String... traitNames)
public ITypedReferenceableInstance createInstance(Id id, AtlasSystemAttributes systemAttributes, String... traitNames) throws AtlasException{
return createInstanceWithTraits(id, systemAttributes, null, traitNames);
}
public ITypedReferenceableInstance createInstanceWithTraits(Id id, AtlasSystemAttributes systemAttributes, Referenceable r, String... traitNames)
throws AtlasException {
ImmutableMap.Builder<String, ITypedStruct> b = new ImmutableBiMap.Builder<String, ITypedStruct>();
......@@ -197,7 +202,7 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
}
}
return new ReferenceableInstance(id == null ? new Id(getName()) : id, getName(), fieldMapping,
return new ReferenceableInstance(id == null ? new Id(getName()) : id, getName(), systemAttributes, fieldMapping,
new boolean[fieldMapping.fields.size()],
fieldMapping.numBools == 0 ? null : new boolean[fieldMapping.numBools],
fieldMapping.numBytes == 0 ? null : new byte[fieldMapping.numBytes],
......
typesystem/src/main/scala/org/apache/atlas/typesystem/json/InstanceSerialization.scala
......@@ -21,23 +21,26 @@ package org.apache.atlas.typesystem.json
import java.text.SimpleDateFormat
import org.apache.atlas.typesystem._
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.persistence.{AtlasSystemAttributes, Id}
import org.apache.atlas.typesystem.types._
import org.json4s._
import org.json4s.native.Serialization._
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import java.util.Date
object InstanceSerialization {
case class _Id(id : String, version : Int, typeName : String, state : Option[String])
case class _AtlasSystemAttributes(createdBy: Option[String], modifiedBy: Option[String], createdTime: Option[Date], modifiedTime: Option[Date])
case class _Struct(typeName : String, values : Map[String, AnyRef])
case class _Reference(id : Option[_Id],
typeName : String,
values : Map[String, AnyRef],
traitNames : List[String],
traits : Map[String, _Struct])
traits : Map[String, _Struct],
systemAttributes : Option[_AtlasSystemAttributes])
def Try[B](x : => B) : Option[B] = {
try { Some(x) } catch { case _ : Throwable => None }
......@@ -71,6 +74,14 @@ object InstanceSerialization {
jsonMap.get("id").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[String]))
}
def createdBy: Option[String] = {
jsonMap.get("createdBy").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[String]))
}
def modifiedBy: Option[String] = {
jsonMap.get("modifiedBy").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[String]))
}
/**
* validate and extract 'state' attribute from Map
* @return
......@@ -91,6 +102,14 @@ object InstanceSerialization {
}
}
def createdTime: Option[Date] = {
jsonMap.get("createdTime").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[Date]))
}
def modifiedTime: Option[Date] = {
jsonMap.get("modifiedTime").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[Date]))
}
/**
* A Map is an Id if:
* - it has the correct [[format.typeHintFieldName]]
......@@ -109,6 +128,15 @@ object InstanceSerialization {
} yield _Id(i, v, typNm, s)
}
def convertSystemAttributes: Option[_AtlasSystemAttributes] = {
for {
c <- Some(createdBy)
m <- Some(modifiedBy)
c_t <- Some(createdTime)
m_t <- Some(modifiedTime)
} yield _AtlasSystemAttributes(c, m, c_t, m_t)
}
/**
* validate and extract 'typeName' attribute from Map
* @return
......@@ -232,7 +260,8 @@ object InstanceSerialization {
values <- valuesMap
traitNms <- traitNames
ts <- traits
} yield _Reference(i, typNm, values, traitNms.toList, ts)
s_attr <- Some(convertSystemAttributes)
} yield _Reference(i, typNm, values, traitNms.toList, ts, s_attr)
}
/**
......@@ -259,16 +288,22 @@ object InstanceSerialization {
def asJava(v : Any)(implicit format: Formats) : Any = v match {
case i : _Id => new Id(i.id, i.version, i.typeName, i.state.orNull)
case s : _Struct => new Struct(s.typeName, asJava(s.values).asInstanceOf[java.util.Map[String, Object]])
case s_attr : _AtlasSystemAttributes => new AtlasSystemAttributes(s_attr.createdBy.orNull, s_attr.modifiedBy.orNull, s_attr.createdTime.orNull, s_attr.modifiedTime.orNull)
case r : _Reference => {
val id = r.id match {
case Some(i) => new Id(i.id, i.version, i.typeName, i.state.orNull)
case None => new Id(r.typeName)
}
val s_attr = r.systemAttributes match {
case Some(s) => new AtlasSystemAttributes(s.createdBy.orNull, s.modifiedBy.orNull, s.createdTime.orNull, s.modifiedTime.orNull)
case None => new AtlasSystemAttributes()
}
new Referenceable(id,
r.typeName,
asJava(r.values).asInstanceOf[java.util.Map[String, Object]],
asJava(r.traitNames).asInstanceOf[java.util.List[String]],
asJava(r.traits).asInstanceOf[java.util.Map[String, IStruct]])
asJava(r.traits).asInstanceOf[java.util.Map[String, IStruct]], s_attr)
}
case l : List[_] => l.map(e => asJava(e)).toList.asJava
case m : Map[_, _] if Try{m.asInstanceOf[Map[String,_]]}.isDefined => {
......@@ -284,6 +319,7 @@ object InstanceSerialization {
def asScala(v : Any) : Any = v match {
case i : Id => _Id(i._getId(), i.getVersion, i.getClassName, Some(i.getStateAsString))
case s_attr: AtlasSystemAttributes => _AtlasSystemAttributes(Some(s_attr.createdBy), Some(s_attr.modifiedBy), Some(s_attr.createdTime), Some(s_attr.modifiedTime))
case r : IReferenceableInstance => {
val traits = r.getTraits.map { tName =>
val t = r.getTrait(tName).asInstanceOf[IStruct]
......@@ -292,7 +328,7 @@ object InstanceSerialization {
_Reference(Some(asScala(r.getId).asInstanceOf[_Id]),
r.getTypeName, asScala(r.getValuesMap).asInstanceOf[Map[String, AnyRef]],
asScala(r.getTraits).asInstanceOf[List[String]],
traits.asInstanceOf[Map[String, _Struct]])
traits.asInstanceOf[Map[String, _Struct]], Some(asScala(r.getSystemAttributes).asInstanceOf[_AtlasSystemAttributes]))
}
case s : IStruct => _Struct(s.getTypeName, asScala(s.getValuesMap).asInstanceOf[Map[String, AnyRef]])
case l : java.util.List[_] => l.asScala.map(e => asScala(e)).toList
......
typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
......@@ -19,15 +19,16 @@
package org.apache.atlas.typesystem.json
import org.apache.atlas.typesystem._
import org.apache.atlas.typesystem.persistence.{Id, ReferenceableInstance, StructInstance}
import org.apache.atlas.typesystem.persistence.{AtlasSystemAttributes, Id, ReferenceableInstance, StructInstance}
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
import org.apache.atlas.typesystem.types._
import org.json4s.JsonAST.JInt
import org.json4s._
import org.json4s.{JsonAST, _}
import org.json4s.native.Serialization._
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import java.util.Date
class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => (
{
......@@ -60,6 +61,7 @@ class IdSerializer extends CustomSerializer[Id](format => ( {
case JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(typeName)) ::
JField("id", JString(id)) ::
JField("version", JString(version)) :: Nil) => new Id(id, version.toInt, typeName)
}, {
case id: Id => JObject(JField("id", JString(id.id)),
JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(id.typeName)),
......@@ -117,12 +119,14 @@ class TypedReferenceableInstanceSerializer()
var typField: Option[JField] = None
var idField: Option[JField] = None
var traitsField: Option[JField] = None
var sysAttrField: Option[JField] = None
var fields: List[JField] = Nil
fs.foreach { f: JField => f._1 match {
case Serialization.STRUCT_TYPE_FIELD_NAME => typField = Some(f)
case Serialization.ID_TYPE_FIELD_NAME => idField = Some(f)
case Serialization.TRAIT_TYPE_FIELD_NAME => traitsField = Some(f)
case Serialization.SYSTEM_ATTR_FIELD_NAME => sysAttrField = Some(f)
case _ => fields = fields :+ f
}
}
......@@ -141,7 +145,8 @@ class TypedReferenceableInstanceSerializer()
val sT = typSystem.getDataType(
classOf[ClassType], typName).asInstanceOf[ClassType]
val id = Serialization.deserializeId(idField.get._2)
val s = sT.createInstance(id, traitNames: _*)
val s_attr = Serialization.deserializeSystemAttributes(sysAttrField.get._2)
val s = sT.createInstance(id, s_attr, traitNames: _*)
Serialization.deserializeFields(typSystem, sT, s, fields)
traitsField.map { t =>
......@@ -169,10 +174,11 @@ class TypedReferenceableInstanceSerializer()
case id: Id => Serialization.serializeId(id)
case e: ITypedReferenceableInstance =>
val idJ = JField(Serialization.ID_TYPE_FIELD_NAME, Serialization.serializeId(e.getId))
val s_attrJ = JField(Serialization.SYSTEM_ATTR_FIELD_NAME, Serialization.serializeSystemAttributes(e.getSystemAttributes))
var fields = Serialization.serializeFields(e)
val traitsJ: List[JField] = e.getTraits.map(tName => JField(tName, Extraction.decompose(e.getTrait(tName)))).toList
fields = idJ :: fields
fields = idJ :: s_attrJ :: fields
if (traitsJ.size > 0) {
fields = fields :+ JField(Serialization.TRAIT_TYPE_FIELD_NAME, JObject(traitsJ: _*))
}
......@@ -186,6 +192,7 @@ object Serialization {
val STRUCT_TYPE_FIELD_NAME = "$typeName$"
val ID_TYPE_FIELD_NAME = "$id$"
val TRAIT_TYPE_FIELD_NAME = "$traits$"
val SYSTEM_ATTR_FIELD_NAME = "$systemAttributes$"
def extractList(lT: ArrayType, value: JArray)(implicit format: Formats): Any = {
val dT = lT.getElemType
......@@ -218,6 +225,22 @@ object Serialization {
JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(id.typeName)),
JField("version", JInt(id.version)), JField("state", JString(id.state.name())))
//Handling serialization issues with null values
//See https://github.com/json4s/json4s/issues/358
def parseString(s: Any) = s match {
case s:String => JString(s)
case s:Date => JString(s.toString)
case _ => JString("")
}
def serializeSystemAttributes(s_attr: AtlasSystemAttributes) = JObject(
JField("createdBy", parseString(s_attr.modifiedBy)),
JField("modifiedBy", parseString(s_attr.modifiedBy)),
JField("createdTime", parseString(s_attr.createdTime)),
JField("modifiedTime", parseString(s_attr.modifiedTime))
)
def serializeFields(e: ITypedInstance)(implicit format: Formats) = e.fieldMapping.fields.map {
case (fName, info) => {
var v = e.get(fName)
......@@ -272,6 +295,13 @@ object Serialization {
JField("state", JString(state)) :: Nil) => new Id(id, version.toInt, typeName, state)
}
def deserializeSystemAttributes(value: JValue)(implicit format : Formats) = value match {
case JObject(JField("createdBy", JString(createdBy))::
JField("modifiedBy", JString(modifiedBy))::
JField("createdTime", JString(createdTime))::
JField("modifiedTime", JString(modifiedTime))::Nil) => new AtlasSystemAttributes(createdBy, modifiedBy, createdTime, modifiedTime)
}
def toJson(value: ITypedReferenceableInstance): String = {
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment