Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
98f4d40a
Commit
98f4d40a
authored
9 years ago
by
Suma Shivaprasad
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-571 Modify Atlas client for necessary changes in context of HA (yhemanth via sumasai)
parent
c1d4e7c9
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
323 additions
and
119 deletions
+323
-119
HiveMetaStoreBridge.java
...ava/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+9
-1
AtlasClient.java
client/src/main/java/org/apache/atlas/AtlasClient.java
+0
-0
AtlasServerEnsemble.java
...t/src/main/java/org/apache/atlas/AtlasServerEnsemble.java
+52
-0
ResourceCreator.java
client/src/main/java/org/apache/atlas/ResourceCreator.java
+7
-21
AtlasClientTest.java
client/src/test/java/org/apache/atlas/AtlasClientTest.java
+0
-0
pom.xml
common/pom.xml
+5
-0
HAConfiguration.java
...on/src/main/java/org/apache/atlas/ha/HAConfiguration.java
+42
-65
SecurityProperties.java
...in/java/org/apache/atlas/security/SecurityProperties.java
+46
-0
HAConfigurationTest.java
...rc/test/java/org/apache/atlas/ha/HAConfigurationTest.java
+67
-0
release-log.txt
release-log.txt
+1
-0
AtlasServerIdSelector.java
.../main/java/org/apache/atlas/ha/AtlasServerIdSelector.java
+82
-0
AtlasServerIdSelectorTest.java
...n/test/org/apache/atlas/ha/AtlasServerIdSelectorTest.java
+4
-26
QuickStart.java
...p/src/main/java/org/apache/atlas/examples/QuickStart.java
+2
-1
ActiveInstanceElectorService.java
...pache/atlas/web/service/ActiveInstanceElectorService.java
+2
-1
CuratorFactory.java
...ain/java/org/apache/atlas/web/service/CuratorFactory.java
+1
-1
NegativeSSLAndKerberosTest.java
...apache/atlas/web/security/NegativeSSLAndKerberosTest.java
+1
-1
SSLAndKerberosTest.java
...ava/org/apache/atlas/web/security/SSLAndKerberosTest.java
+1
-1
SSLTest.java
.../src/test/java/org/apache/atlas/web/security/SSLTest.java
+1
-1
No files found.
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
View file @
98f4d40a
...
@@ -96,7 +96,15 @@ public class HiveMetaStoreBridge {
...
@@ -96,7 +96,15 @@ public class HiveMetaStoreBridge {
UserGroupInformation
ugi
)
throws
Exception
{
UserGroupInformation
ugi
)
throws
Exception
{
this
(
hiveConf
.
get
(
HIVE_CLUSTER_NAME
,
DEFAULT_CLUSTER_NAME
),
this
(
hiveConf
.
get
(
HIVE_CLUSTER_NAME
,
DEFAULT_CLUSTER_NAME
),
Hive
.
get
(
hiveConf
),
Hive
.
get
(
hiveConf
),
new
AtlasClient
(
atlasConf
.
getString
(
ATLAS_ENDPOINT
,
DEFAULT_DGI_URL
),
ugi
,
doAsUser
));
atlasConf
,
doAsUser
,
ugi
);
}
HiveMetaStoreBridge
(
String
clusterName
,
Hive
hiveClient
,
Configuration
atlasConf
,
String
doAsUser
,
UserGroupInformation
ugi
)
{
this
.
clusterName
=
clusterName
;
this
.
hiveClient
=
hiveClient
;
String
baseUrls
=
atlasConf
.
getString
(
ATLAS_ENDPOINT
,
DEFAULT_DGI_URL
);
this
.
atlasClient
=
new
AtlasClient
(
ugi
,
doAsUser
,
baseUrls
.
split
(
","
));
}
}
HiveMetaStoreBridge
(
String
clusterName
,
Hive
hiveClient
,
AtlasClient
atlasClient
)
{
HiveMetaStoreBridge
(
String
clusterName
,
Hive
hiveClient
,
AtlasClient
atlasClient
)
{
...
...
This diff is collapsed.
Click to expand it.
client/src/main/java/org/apache/atlas/AtlasClient.java
View file @
98f4d40a
This diff is collapsed.
Click to expand it.
client/src/main/java/org/apache/atlas/AtlasServerEnsemble.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
;
import
com.google.common.base.Preconditions
;
import
org.apache.commons.lang.StringUtils
;
import
scala.actors.threadpool.Arrays
;
import
java.util.List
;
public
class
AtlasServerEnsemble
{
private
final
String
[]
urls
;
public
AtlasServerEnsemble
(
String
[]
baseUrls
)
{
Preconditions
.
checkArgument
((
baseUrls
!=
null
&&
baseUrls
.
length
>
0
),
"List of baseURLs cannot be null or empty."
);
for
(
String
baseUrl
:
baseUrls
)
{
Preconditions
.
checkArgument
(
StringUtils
.
isNotEmpty
(
baseUrl
),
"Base URL cannot be null or empty."
);
}
urls
=
baseUrls
;
}
public
boolean
hasSingleInstance
()
{
return
urls
.
length
==
1
;
}
public
String
firstURL
()
{
return
urls
[
0
];
}
public
List
<
String
>
getMembers
()
{
return
Arrays
.
asList
(
urls
);
}
}
This diff is collapsed.
Click to expand it.
client/src/main/java/org/apache/atlas/
security/SecurityProperties
.java
→
client/src/main/java/org/apache/atlas/
ResourceCreator
.java
View file @
98f4d40a
/*
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* distributed with this work for additional information
...
@@ -15,28 +15,14 @@
...
@@ -15,28 +15,14 @@
* See the License for the specific language governing permissions and
* See the License for the specific language governing permissions and
* limitations under the License.
* limitations under the License.
*/
*/
package
org
.
apache
.
atlas
.
security
;
import
java.util.Arrays
;
package
org
.
apache
.
atlas
;
import
java.util.List
;
import
com.sun.jersey.api.client.WebResource
;
/**
/**
*
*
An interface to capture the closure of how a WebResource is created.
*/
*/
public
interface
SecurityProperties
{
public
interface
ResourceCreator
{
String
TLS_ENABLED
=
"atlas.enableTLS"
;
WebResource
createResource
();
String
KEYSTORE_FILE_KEY
=
"keystore.file"
;
String
DEFAULT_KEYSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
String
KEYSTORE_PASSWORD_KEY
=
"keystore.password"
;
String
TRUSTSTORE_FILE_KEY
=
"truststore.file"
;
String
DEFATULT_TRUSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
String
TRUSTSTORE_PASSWORD_KEY
=
"truststore.password"
;
String
SERVER_CERT_PASSWORD_KEY
=
"password"
;
String
CLIENT_AUTH_KEY
=
"client.auth.enabled"
;
String
CERT_STORES_CREDENTIAL_PROVIDER_PATH
=
"cert.stores.credential.provider.path"
;
String
SSL_CLIENT_PROPERTIES
=
"ssl-client.xml"
;
String
BIND_ADDRESS
=
"atlas.server.bind.address"
;
String
ATLAS_SSL_EXCLUDE_CIPHER_SUITES
=
"atlas.ssl.exclude.cipher.suites"
;
List
<
String
>
DEFAULT_CIPHER_SUITES
=
Arrays
.
asList
(
".*NULL.*"
,
".*RC4.*"
,
".*MD5.*"
,
".*DES.*"
,
".*DSS.*"
);
}
}
This diff is collapsed.
Click to expand it.
client/src/test/java/org/apache/atlas/AtlasClientTest.java
View file @
98f4d40a
This diff is collapsed.
Click to expand it.
common/pom.xml
View file @
98f4d40a
...
@@ -51,5 +51,10 @@
...
@@ -51,5 +51,10 @@
<artifactId>
commons-configuration
</artifactId>
<artifactId>
commons-configuration
</artifactId>
</dependency>
</dependency>
<dependency>
<groupId>
org.mockito
</groupId>
<artifactId>
mockito-all
</artifactId>
</dependency>
</dependencies>
</dependencies>
</project>
</project>
This diff is collapsed.
Click to expand it.
server-api
/src/main/java/org/apache/atlas/ha/HAConfiguration.java
→
common
/src/main/java/org/apache/atlas/ha/HAConfiguration.java
View file @
98f4d40a
...
@@ -18,34 +18,36 @@
...
@@ -18,34 +18,36 @@
package
org
.
apache
.
atlas
.
ha
;
package
org
.
apache
.
atlas
.
ha
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.net.NetUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
java.net.InetSocketAddress
;
import
java.util.ArrayList
;
import
java.util.List
;
/**
/**
* A wrapper for getting configuration entries related to HighAvailability.
* A wrapper for getting configuration entries related to HighAvailability.
*/
*/
public
class
HAConfiguration
{
public
final
class
HAConfiguration
{
private
HAConfiguration
()
{
}
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
HAConfiguration
.
class
);
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
HAConfiguration
.
class
);
public
static
final
String
ATLAS_SERVER_HA_PREFIX
=
"atlas.server.ha"
;
public
static
final
String
ATLAS_SERVER_HA_PREFIX
=
"atlas.server.ha
.
"
;
public
static
final
String
ATLAS_SERVER_HA_ENABLED_KEY
=
ATLAS_SERVER_HA_PREFIX
+
"
.
enabled"
;
public
static
final
String
ATLAS_SERVER_HA_ENABLED_KEY
=
ATLAS_SERVER_HA_PREFIX
+
"enabled"
;
public
static
final
String
ATLAS_SERVER_ADDRESS_PREFIX
=
"atlas.server.address."
;
public
static
final
String
ATLAS_SERVER_ADDRESS_PREFIX
=
"atlas.server.address."
;
public
static
final
String
ATLAS_SERVER_IDS
=
"atlas.server.ids"
;
public
static
final
String
ATLAS_SERVER_IDS
=
"atlas.server.ids"
;
public
static
final
String
HA_ZOOKEEPER_CONNECT
=
ATLAS_SERVER_HA_PREFIX
+
"
.
zookeeper.connect"
;
public
static
final
String
HA_ZOOKEEPER_CONNECT
=
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.connect"
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS
=
1000
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS
=
1000
;
public
static
final
String
HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS
=
ATLAS_SERVER_HA_PREFIX
+
".zookeeper.retry.sleeptime.ms"
;
public
static
final
String
HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS
=
public
static
final
String
HA_ZOOKEEPER_NUM_RETRIES
=
ATLAS_SERVER_HA_PREFIX
+
".zookeeper.num.retries"
;
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.retry.sleeptime.ms"
;
public
static
final
String
HA_ZOOKEEPER_NUM_RETRIES
=
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.num.retries"
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES
=
3
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES
=
3
;
public
static
final
String
HA_ZOOKEEPER_SESSION_TIMEOUT_MS
=
ATLAS_SERVER_HA_PREFIX
+
".zookeeper.session.timeout.ms"
;
public
static
final
String
HA_ZOOKEEPER_SESSION_TIMEOUT_MS
=
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.session.timeout.ms"
;
public
static
final
int
DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS
=
20000
;
public
static
final
int
DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS
=
20000
;
/**
/**
...
@@ -58,53 +60,6 @@ public class HAConfiguration {
...
@@ -58,53 +60,6 @@ public class HAConfiguration {
}
}
/**
/**
* Return the ID corresponding to this Atlas instance.
*
* The match is done by looking for an ID configured in {@link HAConfiguration#ATLAS_SERVER_IDS} key
* that has a host:port entry for the key {@link HAConfiguration#ATLAS_SERVER_ADDRESS_PREFIX}+ID where
* the host is a local IP address and port is set in the system property
* {@link AtlasConstants#SYSTEM_PROPERTY_APP_PORT}.
*
* @param configuration
* @return
* @throws AtlasException if no ID is found that maps to a local IP Address or port
*/
public
static
String
getAtlasServerId
(
Configuration
configuration
)
throws
AtlasException
{
// ids are already trimmed by this method
String
[]
ids
=
configuration
.
getStringArray
(
ATLAS_SERVER_IDS
);
String
matchingServerId
=
null
;
int
appPort
=
Integer
.
parseInt
(
System
.
getProperty
(
AtlasConstants
.
SYSTEM_PROPERTY_APP_PORT
));
for
(
String
id
:
ids
)
{
String
hostPort
=
configuration
.
getString
(
ATLAS_SERVER_ADDRESS_PREFIX
+
id
);
if
(!
StringUtils
.
isEmpty
(
hostPort
))
{
InetSocketAddress
socketAddress
;
try
{
socketAddress
=
NetUtils
.
createSocketAddr
(
hostPort
);
}
catch
(
Exception
e
)
{
LOG
.
warn
(
"Exception while trying to get socket address for "
+
hostPort
,
e
);
continue
;
}
if
(!
socketAddress
.
isUnresolved
()
&&
NetUtils
.
isLocalAddress
(
socketAddress
.
getAddress
())
&&
appPort
==
socketAddress
.
getPort
())
{
LOG
.
info
(
"Found matched server id "
+
id
+
" with host port: "
+
hostPort
);
matchingServerId
=
id
;
break
;
}
}
else
{
LOG
.
info
(
"Could not find matching address entry for id: "
+
id
);
}
}
if
(
matchingServerId
==
null
)
{
String
msg
=
String
.
format
(
"Could not find server id for this instance. "
+
"Unable to find IDs matching any local host and port binding among %s"
,
StringUtils
.
join
(
ids
,
","
));
throw
new
AtlasException
(
msg
);
}
return
matchingServerId
;
}
/**
* Get the web server address that a server instance with the passed ID is bound to.
* Get the web server address that a server instance with the passed ID is bound to.
*
*
* This method uses the property {@link SecurityProperties#TLS_ENABLED} to determine whether
* This method uses the property {@link SecurityProperties#TLS_ENABLED} to determine whether
...
@@ -121,8 +76,17 @@ public class HAConfiguration {
...
@@ -121,8 +76,17 @@ public class HAConfiguration {
return
protocol
+
hostPort
;
return
protocol
+
hostPort
;
}
}
public
static
List
<
String
>
getServerInstances
(
Configuration
configuration
)
{
String
[]
serverIds
=
configuration
.
getStringArray
(
ATLAS_SERVER_IDS
);
List
<
String
>
serverInstances
=
new
ArrayList
<>(
serverIds
.
length
);
for
(
String
serverId
:
serverIds
)
{
serverInstances
.
add
(
getBoundAddressForId
(
configuration
,
serverId
));
}
return
serverInstances
;
}
/**
/**
* A collection of Zookeeper specific configuration that is used by High Availability code
* A collection of Zookeeper specific configuration that is used by High Availability code
.
*/
*/
public
static
class
ZookeeperProperties
{
public
static
class
ZookeeperProperties
{
private
String
connectString
;
private
String
connectString
;
...
@@ -156,14 +120,27 @@ public class HAConfiguration {
...
@@ -156,14 +120,27 @@ public class HAConfiguration {
@Override
@Override
public
boolean
equals
(
Object
o
)
{
public
boolean
equals
(
Object
o
)
{
if
(
this
==
o
)
return
true
;
if
(
this
==
o
)
{
if
(
o
==
null
||
getClass
()
!=
o
.
getClass
())
return
false
;
return
true
;
}
if
(
o
==
null
||
getClass
()
!=
o
.
getClass
())
{
return
false
;
}
ZookeeperProperties
that
=
(
ZookeeperProperties
)
o
;
ZookeeperProperties
that
=
(
ZookeeperProperties
)
o
;
if
(
retriesSleepTimeMillis
!=
that
.
retriesSleepTimeMillis
)
return
false
;
if
(
retriesSleepTimeMillis
!=
that
.
retriesSleepTimeMillis
)
{
if
(
numRetries
!=
that
.
numRetries
)
return
false
;
return
false
;
if
(
sessionTimeout
!=
that
.
sessionTimeout
)
return
false
;
}
if
(
numRetries
!=
that
.
numRetries
)
{
return
false
;
}
if
(
sessionTimeout
!=
that
.
sessionTimeout
)
{
return
false
;
}
return
!(
connectString
!=
null
?
!
connectString
.
equals
(
that
.
connectString
)
:
that
.
connectString
!=
null
);
return
!(
connectString
!=
null
?
!
connectString
.
equals
(
that
.
connectString
)
:
that
.
connectString
!=
null
);
}
}
...
...
This diff is collapsed.
Click to expand it.
common/src/main/java/org/apache/atlas/security/SecurityProperties.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
security
;
import
java.util.Arrays
;
import
java.util.List
;
/**
*
*/
public
final
class
SecurityProperties
{
private
SecurityProperties
()
{
}
public
static
final
String
TLS_ENABLED
=
"atlas.enableTLS"
;
public
static
final
String
KEYSTORE_FILE_KEY
=
"keystore.file"
;
public
static
final
String
DEFAULT_KEYSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
public
static
final
String
KEYSTORE_PASSWORD_KEY
=
"keystore.password"
;
public
static
final
String
TRUSTSTORE_FILE_KEY
=
"truststore.file"
;
public
static
final
String
DEFATULT_TRUSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
public
static
final
String
TRUSTSTORE_PASSWORD_KEY
=
"truststore.password"
;
public
static
final
String
SERVER_CERT_PASSWORD_KEY
=
"password"
;
public
static
final
String
CLIENT_AUTH_KEY
=
"client.auth.enabled"
;
public
static
final
String
CERT_STORES_CREDENTIAL_PROVIDER_PATH
=
"cert.stores.credential.provider.path"
;
public
static
final
String
SSL_CLIENT_PROPERTIES
=
"ssl-client.xml"
;
public
static
final
String
BIND_ADDRESS
=
"atlas.server.bind.address"
;
public
static
final
String
ATLAS_SSL_EXCLUDE_CIPHER_SUITES
=
"atlas.ssl.exclude.cipher.suites"
;
public
static
final
List
<
String
>
DEFAULT_CIPHER_SUITES
=
Arrays
.
asList
(
".*NULL.*"
,
".*RC4.*"
,
".*MD5.*"
,
".*DES.*"
,
".*DSS.*"
);
}
This diff is collapsed.
Click to expand it.
common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
ha
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.commons.configuration.Configuration
;
import
org.mockito.Mock
;
import
org.mockito.MockitoAnnotations
;
import
org.testng.annotations.BeforeMethod
;
import
org.testng.annotations.Test
;
import
java.util.List
;
import
static
org
.
mockito
.
Mockito
.
when
;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
assertTrue
;
public
class
HAConfigurationTest
{
@Mock
private
Configuration
configuration
;
@BeforeMethod
public
void
setup
()
{
MockitoAnnotations
.
initMocks
(
this
);
System
.
setProperty
(
AtlasConstants
.
SYSTEM_PROPERTY_APP_PORT
,
AtlasConstants
.
DEFAULT_APP_PORT_STR
);
}
@Test
public
void
testShouldReturnHTTPSBoundAddress
()
{
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21443"
);
when
(
configuration
.
getBoolean
(
SecurityProperties
.
TLS_ENABLED
)).
thenReturn
(
true
);
String
address
=
HAConfiguration
.
getBoundAddressForId
(
configuration
,
"id1"
);
assertEquals
(
address
,
"https://127.0.0.1:21443"
);
}
@Test
public
void
testShouldReturnListOfAddressesInConfig
()
{
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{
"id1"
,
"id2"
});
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id2"
)).
thenReturn
(
"127.0.0.1:31000"
);
List
<
String
>
serverInstances
=
HAConfiguration
.
getServerInstances
(
configuration
);
assertEquals
(
serverInstances
.
size
(),
2
);
assertTrue
(
serverInstances
.
contains
(
"http://127.0.0.1:21000"
));
assertTrue
(
serverInstances
.
contains
(
"http://127.0.0.1:31000"
));
}
}
This diff is collapsed.
Click to expand it.
release-log.txt
View file @
98f4d40a
...
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
...
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ALL CHANGES:
ATLAS-571 Modify Atlas client for necessary changes in context of HA (yhemanth via sumasai)
ATLAS-620 Disable hbase based entity audit (shwethags)
ATLAS-620 Disable hbase based entity audit (shwethags)
ATLAS-618 Fix assembly for hdfs-module (sumasai via yhemanth)
ATLAS-618 Fix assembly for hdfs-module (sumasai via yhemanth)
ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai)
ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai)
...
...
This diff is collapsed.
Click to expand it.
server-api/src/main/java/org/apache/atlas/ha/AtlasServerIdSelector.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
ha
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasException
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.net.NetUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.net.InetSocketAddress
;
public
class
AtlasServerIdSelector
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
AtlasServerIdSelector
.
class
);
/**
* Return the ID corresponding to this Atlas instance.
*
* The match is done by looking for an ID configured in {@link HAConfiguration#ATLAS_SERVER_IDS} key
* that has a host:port entry for the key {@link HAConfiguration#ATLAS_SERVER_ADDRESS_PREFIX}+ID where
* the host is a local IP address and port is set in the system property
* {@link AtlasConstants#SYSTEM_PROPERTY_APP_PORT}.
*
* @param configuration
* @return
* @throws AtlasException if no ID is found that maps to a local IP Address or port
*/
public
static
String
selectServerId
(
Configuration
configuration
)
throws
AtlasException
{
// ids are already trimmed by this method
String
[]
ids
=
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
);
String
matchingServerId
=
null
;
int
appPort
=
Integer
.
parseInt
(
System
.
getProperty
(
AtlasConstants
.
SYSTEM_PROPERTY_APP_PORT
));
for
(
String
id
:
ids
)
{
String
hostPort
=
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
id
);
if
(!
StringUtils
.
isEmpty
(
hostPort
))
{
InetSocketAddress
socketAddress
;
try
{
socketAddress
=
NetUtils
.
createSocketAddr
(
hostPort
);
}
catch
(
Exception
e
)
{
LOG
.
warn
(
"Exception while trying to get socket address for "
+
hostPort
,
e
);
continue
;
}
if
(!
socketAddress
.
isUnresolved
()
&&
NetUtils
.
isLocalAddress
(
socketAddress
.
getAddress
())
&&
appPort
==
socketAddress
.
getPort
())
{
LOG
.
info
(
"Found matched server id "
+
id
+
" with host port: "
+
hostPort
);
matchingServerId
=
id
;
break
;
}
}
else
{
LOG
.
info
(
"Could not find matching address entry for id: "
+
id
);
}
}
if
(
matchingServerId
==
null
)
{
String
msg
=
String
.
format
(
"Could not find server id for this instance. "
+
"Unable to find IDs matching any local host and port binding among %s"
,
StringUtils
.
join
(
ids
,
","
));
throw
new
AtlasException
(
msg
);
}
return
matchingServerId
;
}
}
This diff is collapsed.
Click to expand it.
server-api/src/main/test/org/apache/atlas/ha/
HAConfiguration
Test.java
→
server-api/src/main/test/org/apache/atlas/ha/
AtlasServerIdSelector
Test.java
View file @
98f4d40a
...
@@ -20,7 +20,6 @@ package org.apache.atlas.ha;
...
@@ -20,7 +20,6 @@ package org.apache.atlas.ha;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.configuration.Configuration
;
import
org.mockito.Mock
;
import
org.mockito.Mock
;
import
org.mockito.MockitoAnnotations
;
import
org.mockito.MockitoAnnotations
;
...
@@ -31,8 +30,7 @@ import static org.mockito.Mockito.when;
...
@@ -31,8 +30,7 @@ import static org.mockito.Mockito.when;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
fail
;
import
static
org
.
testng
.
Assert
.
fail
;
public
class
HAConfigurationTest
{
public
class
AtlasServerIdSelectorTest
{
@Mock
@Mock
private
Configuration
configuration
;
private
Configuration
configuration
;
...
@@ -48,14 +46,14 @@ public class HAConfigurationTest {
...
@@ -48,14 +46,14 @@ public class HAConfigurationTest {
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id2"
)).
thenReturn
(
"127.0.0.1:21000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id2"
)).
thenReturn
(
"127.0.0.1:21000"
);
String
atlasServerId
=
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
String
atlasServerId
=
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
assertEquals
(
atlasServerId
,
"id2"
);
assertEquals
(
atlasServerId
,
"id2"
);
}
}
@Test
(
expectedExceptions
=
AtlasException
.
class
)
@Test
(
expectedExceptions
=
AtlasException
.
class
)
public
void
testShouldFailIfNoIDsConfiguration
()
throws
AtlasException
{
public
void
testShouldFailIfNoIDsConfiguration
()
throws
AtlasException
{
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{});
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{});
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
fail
(
"Should not return any server id if IDs not found in configuration"
);
fail
(
"Should not return any server id if IDs not found in configuration"
);
}
}
...
@@ -64,27 +62,7 @@ public class HAConfigurationTest {
...
@@ -64,27 +62,7 @@ public class HAConfigurationTest {
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{
"id1"
,
"id2"
});
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{
"id1"
,
"id2"
});
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
fail
(
"Should not return any server id if no matching address found for any ID"
);
fail
(
"Should not return any server id if no matching address found for any ID"
);
}
}
@Test
public
void
testShouldReturnHTTPBoundAddress
()
{
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21000"
);
when
(
configuration
.
getBoolean
(
SecurityProperties
.
TLS_ENABLED
)).
thenReturn
(
false
);
String
address
=
HAConfiguration
.
getBoundAddressForId
(
configuration
,
"id1"
);
assertEquals
(
address
,
"http://127.0.0.1:21000"
);
}
@Test
public
void
testShouldReturnHTTPSBoundAddress
()
{
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21443"
);
when
(
configuration
.
getBoolean
(
SecurityProperties
.
TLS_ENABLED
)).
thenReturn
(
true
);
String
address
=
HAConfiguration
.
getBoundAddressForId
(
configuration
,
"id1"
);
assertEquals
(
address
,
"https://127.0.0.1:21443"
);
}
}
}
This diff is collapsed.
Click to expand it.
webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
View file @
98f4d40a
...
@@ -113,7 +113,8 @@ public class QuickStart {
...
@@ -113,7 +113,8 @@ public class QuickStart {
private
final
AtlasClient
metadataServiceClient
;
private
final
AtlasClient
metadataServiceClient
;
QuickStart
(
String
baseUrl
)
{
QuickStart
(
String
baseUrl
)
{
metadataServiceClient
=
new
AtlasClient
(
baseUrl
);
String
[]
urls
=
baseUrl
.
split
(
","
);
metadataServiceClient
=
new
AtlasClient
(
null
,
null
,
urls
);
}
}
void
createTypes
()
throws
Exception
{
void
createTypes
()
throws
Exception
{
...
...
This diff is collapsed.
Click to expand it.
webapp/src/main/java/org/apache/atlas/web/service/ActiveInstanceElectorService.java
View file @
98f4d40a
...
@@ -23,6 +23,7 @@ import com.google.inject.Provider;
...
@@ -23,6 +23,7 @@ import com.google.inject.Provider;
import
com.google.inject.Singleton
;
import
com.google.inject.Singleton
;
import
org.apache.atlas.ApplicationProperties
;
import
org.apache.atlas.ApplicationProperties
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.ha.AtlasServerIdSelector
;
import
org.apache.atlas.ha.HAConfiguration
;
import
org.apache.atlas.ha.HAConfiguration
;
import
org.apache.atlas.listener.ActiveStateChangeHandler
;
import
org.apache.atlas.listener.ActiveStateChangeHandler
;
import
org.apache.atlas.service.Service
;
import
org.apache.atlas.service.Service
;
...
@@ -101,7 +102,7 @@ public class ActiveInstanceElectorService implements Service, LeaderLatchListene
...
@@ -101,7 +102,7 @@ public class ActiveInstanceElectorService implements Service, LeaderLatchListene
return
;
return
;
}
}
cacheActiveStateChangeHandlers
();
cacheActiveStateChangeHandlers
();
serverId
=
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
serverId
=
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
joinElection
();
joinElection
();
}
}
...
...
This diff is collapsed.
Click to expand it.
webapp/src/main/java/org/apache/atlas/web/service/CuratorFactory.java
View file @
98f4d40a
...
@@ -85,7 +85,7 @@ public class CuratorFactory {
...
@@ -85,7 +85,7 @@ public class CuratorFactory {
* Create a new instance {@link LeaderLatch}
* Create a new instance {@link LeaderLatch}
* @param serverId the ID used to register this instance with curator.
* @param serverId the ID used to register this instance with curator.
* This ID should typically be obtained using
* This ID should typically be obtained using
* {@link
HAConfiguration#getAtlas
ServerId(Configuration)}
* {@link
org.apache.atlas.ha.AtlasServerIdSelector#select
ServerId(Configuration)}
* @return
* @return
*/
*/
public
LeaderLatch
leaderLatchInstance
(
String
serverId
)
{
public
LeaderLatch
leaderLatchInstance
(
String
serverId
)
{
...
...
This diff is collapsed.
Click to expand it.
webapp/src/test/java/org/apache/atlas/web/security/NegativeSSLAndKerberosTest.java
View file @
98f4d40a
...
@@ -86,7 +86,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
...
@@ -86,7 +86,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
dgiClient
=
new
AtlasClient
(
DGI_URL
)
{
dgiClient
=
new
AtlasClient
(
DGI_URL
)
{
@Override
@Override
protected
PropertiesConfiguration
getClientProperties
()
throws
AtlasException
{
protected
PropertiesConfiguration
getClientProperties
()
{
return
configuration
;
return
configuration
;
}
}
};
};
...
...
This diff is collapsed.
Click to expand it.
webapp/src/test/java/org/apache/atlas/web/security/SSLAndKerberosTest.java
View file @
98f4d40a
...
@@ -102,7 +102,7 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
...
@@ -102,7 +102,7 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
public
AtlasClient
run
()
throws
Exception
{
public
AtlasClient
run
()
throws
Exception
{
return
new
AtlasClient
(
DGI_URL
)
{
return
new
AtlasClient
(
DGI_URL
)
{
@Override
@Override
protected
PropertiesConfiguration
getClientProperties
()
throws
AtlasException
{
protected
PropertiesConfiguration
getClientProperties
()
{
return
configuration
;
return
configuration
;
}
}
};
};
...
...
This diff is collapsed.
Click to expand it.
webapp/src/test/java/org/apache/atlas/web/security/SSLTest.java
View file @
98f4d40a
...
@@ -78,7 +78,7 @@ public class SSLTest extends BaseSSLAndKerberosTest {
...
@@ -78,7 +78,7 @@ public class SSLTest extends BaseSSLAndKerberosTest {
dgiCLient
=
new
AtlasClient
(
DGI_URL
)
{
dgiCLient
=
new
AtlasClient
(
DGI_URL
)
{
@Override
@Override
protected
PropertiesConfiguration
getClientProperties
()
throws
AtlasException
{
protected
PropertiesConfiguration
getClientProperties
()
{
return
configuration
;
return
configuration
;
}
}
};
};
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment