Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
atlas
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dataplatform
atlas
Commits
98f4d40a
Commit
98f4d40a
authored
Apr 04, 2016
by
Suma Shivaprasad
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ATLAS-571 Modify Atlas client for necessary changes in context of HA (yhemanth via sumasai)
parent
c1d4e7c9
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
323 additions
and
119 deletions
+323
-119
HiveMetaStoreBridge.java
...ava/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+9
-1
AtlasClient.java
client/src/main/java/org/apache/atlas/AtlasClient.java
+0
-0
AtlasServerEnsemble.java
...t/src/main/java/org/apache/atlas/AtlasServerEnsemble.java
+52
-0
ResourceCreator.java
client/src/main/java/org/apache/atlas/ResourceCreator.java
+7
-21
AtlasClientTest.java
client/src/test/java/org/apache/atlas/AtlasClientTest.java
+0
-0
pom.xml
common/pom.xml
+5
-0
HAConfiguration.java
...on/src/main/java/org/apache/atlas/ha/HAConfiguration.java
+42
-65
SecurityProperties.java
...in/java/org/apache/atlas/security/SecurityProperties.java
+46
-0
HAConfigurationTest.java
...rc/test/java/org/apache/atlas/ha/HAConfigurationTest.java
+67
-0
release-log.txt
release-log.txt
+1
-0
AtlasServerIdSelector.java
.../main/java/org/apache/atlas/ha/AtlasServerIdSelector.java
+82
-0
AtlasServerIdSelectorTest.java
...n/test/org/apache/atlas/ha/AtlasServerIdSelectorTest.java
+4
-26
QuickStart.java
...p/src/main/java/org/apache/atlas/examples/QuickStart.java
+2
-1
ActiveInstanceElectorService.java
...pache/atlas/web/service/ActiveInstanceElectorService.java
+2
-1
CuratorFactory.java
...ain/java/org/apache/atlas/web/service/CuratorFactory.java
+1
-1
NegativeSSLAndKerberosTest.java
...apache/atlas/web/security/NegativeSSLAndKerberosTest.java
+1
-1
SSLAndKerberosTest.java
...ava/org/apache/atlas/web/security/SSLAndKerberosTest.java
+1
-1
SSLTest.java
.../src/test/java/org/apache/atlas/web/security/SSLTest.java
+1
-1
No files found.
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
View file @
98f4d40a
...
@@ -96,7 +96,15 @@ public class HiveMetaStoreBridge {
...
@@ -96,7 +96,15 @@ public class HiveMetaStoreBridge {
UserGroupInformation
ugi
)
throws
Exception
{
UserGroupInformation
ugi
)
throws
Exception
{
this
(
hiveConf
.
get
(
HIVE_CLUSTER_NAME
,
DEFAULT_CLUSTER_NAME
),
this
(
hiveConf
.
get
(
HIVE_CLUSTER_NAME
,
DEFAULT_CLUSTER_NAME
),
Hive
.
get
(
hiveConf
),
Hive
.
get
(
hiveConf
),
new
AtlasClient
(
atlasConf
.
getString
(
ATLAS_ENDPOINT
,
DEFAULT_DGI_URL
),
ugi
,
doAsUser
));
atlasConf
,
doAsUser
,
ugi
);
}
HiveMetaStoreBridge
(
String
clusterName
,
Hive
hiveClient
,
Configuration
atlasConf
,
String
doAsUser
,
UserGroupInformation
ugi
)
{
this
.
clusterName
=
clusterName
;
this
.
hiveClient
=
hiveClient
;
String
baseUrls
=
atlasConf
.
getString
(
ATLAS_ENDPOINT
,
DEFAULT_DGI_URL
);
this
.
atlasClient
=
new
AtlasClient
(
ugi
,
doAsUser
,
baseUrls
.
split
(
","
));
}
}
HiveMetaStoreBridge
(
String
clusterName
,
Hive
hiveClient
,
AtlasClient
atlasClient
)
{
HiveMetaStoreBridge
(
String
clusterName
,
Hive
hiveClient
,
AtlasClient
atlasClient
)
{
...
...
client/src/main/java/org/apache/atlas/AtlasClient.java
View file @
98f4d40a
This diff is collapsed.
Click to expand it.
client/src/main/java/org/apache/atlas/AtlasServerEnsemble.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
;
import
com.google.common.base.Preconditions
;
import
org.apache.commons.lang.StringUtils
;
import
scala.actors.threadpool.Arrays
;
import
java.util.List
;
public
class
AtlasServerEnsemble
{
private
final
String
[]
urls
;
public
AtlasServerEnsemble
(
String
[]
baseUrls
)
{
Preconditions
.
checkArgument
((
baseUrls
!=
null
&&
baseUrls
.
length
>
0
),
"List of baseURLs cannot be null or empty."
);
for
(
String
baseUrl
:
baseUrls
)
{
Preconditions
.
checkArgument
(
StringUtils
.
isNotEmpty
(
baseUrl
),
"Base URL cannot be null or empty."
);
}
urls
=
baseUrls
;
}
public
boolean
hasSingleInstance
()
{
return
urls
.
length
==
1
;
}
public
String
firstURL
()
{
return
urls
[
0
];
}
public
List
<
String
>
getMembers
()
{
return
Arrays
.
asList
(
urls
);
}
}
client/src/main/java/org/apache/atlas/
security/SecurityProperties
.java
→
client/src/main/java/org/apache/atlas/
ResourceCreator
.java
View file @
98f4d40a
/*
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* distributed with this work for additional information
...
@@ -15,28 +15,14 @@
...
@@ -15,28 +15,14 @@
* See the License for the specific language governing permissions and
* See the License for the specific language governing permissions and
* limitations under the License.
* limitations under the License.
*/
*/
package
org
.
apache
.
atlas
.
security
;
import
java.util.Arrays
;
package
org
.
apache
.
atlas
;
import
java.util.List
;
import
com.sun.jersey.api.client.WebResource
;
/**
/**
*
*
An interface to capture the closure of how a WebResource is created.
*/
*/
public
interface
SecurityProperties
{
public
interface
ResourceCreator
{
String
TLS_ENABLED
=
"atlas.enableTLS"
;
WebResource
createResource
();
String
KEYSTORE_FILE_KEY
=
"keystore.file"
;
String
DEFAULT_KEYSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
String
KEYSTORE_PASSWORD_KEY
=
"keystore.password"
;
String
TRUSTSTORE_FILE_KEY
=
"truststore.file"
;
String
DEFATULT_TRUSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
String
TRUSTSTORE_PASSWORD_KEY
=
"truststore.password"
;
String
SERVER_CERT_PASSWORD_KEY
=
"password"
;
String
CLIENT_AUTH_KEY
=
"client.auth.enabled"
;
String
CERT_STORES_CREDENTIAL_PROVIDER_PATH
=
"cert.stores.credential.provider.path"
;
String
SSL_CLIENT_PROPERTIES
=
"ssl-client.xml"
;
String
BIND_ADDRESS
=
"atlas.server.bind.address"
;
String
ATLAS_SSL_EXCLUDE_CIPHER_SUITES
=
"atlas.ssl.exclude.cipher.suites"
;
List
<
String
>
DEFAULT_CIPHER_SUITES
=
Arrays
.
asList
(
".*NULL.*"
,
".*RC4.*"
,
".*MD5.*"
,
".*DES.*"
,
".*DSS.*"
);
}
}
client/src/test/java/org/apache/atlas/AtlasClientTest.java
View file @
98f4d40a
This diff is collapsed.
Click to expand it.
common/pom.xml
View file @
98f4d40a
...
@@ -51,5 +51,10 @@
...
@@ -51,5 +51,10 @@
<artifactId>
commons-configuration
</artifactId>
<artifactId>
commons-configuration
</artifactId>
</dependency>
</dependency>
<dependency>
<groupId>
org.mockito
</groupId>
<artifactId>
mockito-all
</artifactId>
</dependency>
</dependencies>
</dependencies>
</project>
</project>
server-api
/src/main/java/org/apache/atlas/ha/HAConfiguration.java
→
common
/src/main/java/org/apache/atlas/ha/HAConfiguration.java
View file @
98f4d40a
...
@@ -18,34 +18,36 @@
...
@@ -18,34 +18,36 @@
package
org
.
apache
.
atlas
.
ha
;
package
org
.
apache
.
atlas
.
ha
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.net.NetUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
java.net.InetSocketAddress
;
import
java.util.ArrayList
;
import
java.util.List
;
/**
/**
* A wrapper for getting configuration entries related to HighAvailability.
* A wrapper for getting configuration entries related to HighAvailability.
*/
*/
public
class
HAConfiguration
{
public
final
class
HAConfiguration
{
private
HAConfiguration
()
{
}
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
HAConfiguration
.
class
);
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
HAConfiguration
.
class
);
public
static
final
String
ATLAS_SERVER_HA_PREFIX
=
"atlas.server.ha"
;
public
static
final
String
ATLAS_SERVER_HA_PREFIX
=
"atlas.server.ha
.
"
;
public
static
final
String
ATLAS_SERVER_HA_ENABLED_KEY
=
ATLAS_SERVER_HA_PREFIX
+
"
.
enabled"
;
public
static
final
String
ATLAS_SERVER_HA_ENABLED_KEY
=
ATLAS_SERVER_HA_PREFIX
+
"enabled"
;
public
static
final
String
ATLAS_SERVER_ADDRESS_PREFIX
=
"atlas.server.address."
;
public
static
final
String
ATLAS_SERVER_ADDRESS_PREFIX
=
"atlas.server.address."
;
public
static
final
String
ATLAS_SERVER_IDS
=
"atlas.server.ids"
;
public
static
final
String
ATLAS_SERVER_IDS
=
"atlas.server.ids"
;
public
static
final
String
HA_ZOOKEEPER_CONNECT
=
ATLAS_SERVER_HA_PREFIX
+
"
.
zookeeper.connect"
;
public
static
final
String
HA_ZOOKEEPER_CONNECT
=
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.connect"
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS
=
1000
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS
=
1000
;
public
static
final
String
HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS
=
ATLAS_SERVER_HA_PREFIX
+
".zookeeper.retry.sleeptime.ms"
;
public
static
final
String
HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS
=
public
static
final
String
HA_ZOOKEEPER_NUM_RETRIES
=
ATLAS_SERVER_HA_PREFIX
+
".zookeeper.num.retries"
;
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.retry.sleeptime.ms"
;
public
static
final
String
HA_ZOOKEEPER_NUM_RETRIES
=
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.num.retries"
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES
=
3
;
public
static
final
int
DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES
=
3
;
public
static
final
String
HA_ZOOKEEPER_SESSION_TIMEOUT_MS
=
ATLAS_SERVER_HA_PREFIX
+
".zookeeper.session.timeout.ms"
;
public
static
final
String
HA_ZOOKEEPER_SESSION_TIMEOUT_MS
=
ATLAS_SERVER_HA_PREFIX
+
"zookeeper.session.timeout.ms"
;
public
static
final
int
DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS
=
20000
;
public
static
final
int
DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS
=
20000
;
/**
/**
...
@@ -58,53 +60,6 @@ public class HAConfiguration {
...
@@ -58,53 +60,6 @@ public class HAConfiguration {
}
}
/**
/**
* Return the ID corresponding to this Atlas instance.
*
* The match is done by looking for an ID configured in {@link HAConfiguration#ATLAS_SERVER_IDS} key
* that has a host:port entry for the key {@link HAConfiguration#ATLAS_SERVER_ADDRESS_PREFIX}+ID where
* the host is a local IP address and port is set in the system property
* {@link AtlasConstants#SYSTEM_PROPERTY_APP_PORT}.
*
* @param configuration
* @return
* @throws AtlasException if no ID is found that maps to a local IP Address or port
*/
public
static
String
getAtlasServerId
(
Configuration
configuration
)
throws
AtlasException
{
// ids are already trimmed by this method
String
[]
ids
=
configuration
.
getStringArray
(
ATLAS_SERVER_IDS
);
String
matchingServerId
=
null
;
int
appPort
=
Integer
.
parseInt
(
System
.
getProperty
(
AtlasConstants
.
SYSTEM_PROPERTY_APP_PORT
));
for
(
String
id
:
ids
)
{
String
hostPort
=
configuration
.
getString
(
ATLAS_SERVER_ADDRESS_PREFIX
+
id
);
if
(!
StringUtils
.
isEmpty
(
hostPort
))
{
InetSocketAddress
socketAddress
;
try
{
socketAddress
=
NetUtils
.
createSocketAddr
(
hostPort
);
}
catch
(
Exception
e
)
{
LOG
.
warn
(
"Exception while trying to get socket address for "
+
hostPort
,
e
);
continue
;
}
if
(!
socketAddress
.
isUnresolved
()
&&
NetUtils
.
isLocalAddress
(
socketAddress
.
getAddress
())
&&
appPort
==
socketAddress
.
getPort
())
{
LOG
.
info
(
"Found matched server id "
+
id
+
" with host port: "
+
hostPort
);
matchingServerId
=
id
;
break
;
}
}
else
{
LOG
.
info
(
"Could not find matching address entry for id: "
+
id
);
}
}
if
(
matchingServerId
==
null
)
{
String
msg
=
String
.
format
(
"Could not find server id for this instance. "
+
"Unable to find IDs matching any local host and port binding among %s"
,
StringUtils
.
join
(
ids
,
","
));
throw
new
AtlasException
(
msg
);
}
return
matchingServerId
;
}
/**
* Get the web server address that a server instance with the passed ID is bound to.
* Get the web server address that a server instance with the passed ID is bound to.
*
*
* This method uses the property {@link SecurityProperties#TLS_ENABLED} to determine whether
* This method uses the property {@link SecurityProperties#TLS_ENABLED} to determine whether
...
@@ -121,8 +76,17 @@ public class HAConfiguration {
...
@@ -121,8 +76,17 @@ public class HAConfiguration {
return
protocol
+
hostPort
;
return
protocol
+
hostPort
;
}
}
public
static
List
<
String
>
getServerInstances
(
Configuration
configuration
)
{
String
[]
serverIds
=
configuration
.
getStringArray
(
ATLAS_SERVER_IDS
);
List
<
String
>
serverInstances
=
new
ArrayList
<>(
serverIds
.
length
);
for
(
String
serverId
:
serverIds
)
{
serverInstances
.
add
(
getBoundAddressForId
(
configuration
,
serverId
));
}
return
serverInstances
;
}
/**
/**
* A collection of Zookeeper specific configuration that is used by High Availability code
* A collection of Zookeeper specific configuration that is used by High Availability code
.
*/
*/
public
static
class
ZookeeperProperties
{
public
static
class
ZookeeperProperties
{
private
String
connectString
;
private
String
connectString
;
...
@@ -156,14 +120,27 @@ public class HAConfiguration {
...
@@ -156,14 +120,27 @@ public class HAConfiguration {
@Override
@Override
public
boolean
equals
(
Object
o
)
{
public
boolean
equals
(
Object
o
)
{
if
(
this
==
o
)
return
true
;
if
(
this
==
o
)
{
if
(
o
==
null
||
getClass
()
!=
o
.
getClass
())
return
false
;
return
true
;
}
if
(
o
==
null
||
getClass
()
!=
o
.
getClass
())
{
return
false
;
}
ZookeeperProperties
that
=
(
ZookeeperProperties
)
o
;
ZookeeperProperties
that
=
(
ZookeeperProperties
)
o
;
if
(
retriesSleepTimeMillis
!=
that
.
retriesSleepTimeMillis
)
return
false
;
if
(
retriesSleepTimeMillis
!=
that
.
retriesSleepTimeMillis
)
{
if
(
numRetries
!=
that
.
numRetries
)
return
false
;
return
false
;
if
(
sessionTimeout
!=
that
.
sessionTimeout
)
return
false
;
}
if
(
numRetries
!=
that
.
numRetries
)
{
return
false
;
}
if
(
sessionTimeout
!=
that
.
sessionTimeout
)
{
return
false
;
}
return
!(
connectString
!=
null
?
!
connectString
.
equals
(
that
.
connectString
)
:
that
.
connectString
!=
null
);
return
!(
connectString
!=
null
?
!
connectString
.
equals
(
that
.
connectString
)
:
that
.
connectString
!=
null
);
}
}
...
...
common/src/main/java/org/apache/atlas/security/SecurityProperties.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
security
;
import
java.util.Arrays
;
import
java.util.List
;
/**
*
*/
public
final
class
SecurityProperties
{
private
SecurityProperties
()
{
}
public
static
final
String
TLS_ENABLED
=
"atlas.enableTLS"
;
public
static
final
String
KEYSTORE_FILE_KEY
=
"keystore.file"
;
public
static
final
String
DEFAULT_KEYSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
public
static
final
String
KEYSTORE_PASSWORD_KEY
=
"keystore.password"
;
public
static
final
String
TRUSTSTORE_FILE_KEY
=
"truststore.file"
;
public
static
final
String
DEFATULT_TRUSTORE_FILE_LOCATION
=
"target/atlas.keystore"
;
public
static
final
String
TRUSTSTORE_PASSWORD_KEY
=
"truststore.password"
;
public
static
final
String
SERVER_CERT_PASSWORD_KEY
=
"password"
;
public
static
final
String
CLIENT_AUTH_KEY
=
"client.auth.enabled"
;
public
static
final
String
CERT_STORES_CREDENTIAL_PROVIDER_PATH
=
"cert.stores.credential.provider.path"
;
public
static
final
String
SSL_CLIENT_PROPERTIES
=
"ssl-client.xml"
;
public
static
final
String
BIND_ADDRESS
=
"atlas.server.bind.address"
;
public
static
final
String
ATLAS_SSL_EXCLUDE_CIPHER_SUITES
=
"atlas.ssl.exclude.cipher.suites"
;
public
static
final
List
<
String
>
DEFAULT_CIPHER_SUITES
=
Arrays
.
asList
(
".*NULL.*"
,
".*RC4.*"
,
".*MD5.*"
,
".*DES.*"
,
".*DSS.*"
);
}
common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
ha
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.commons.configuration.Configuration
;
import
org.mockito.Mock
;
import
org.mockito.MockitoAnnotations
;
import
org.testng.annotations.BeforeMethod
;
import
org.testng.annotations.Test
;
import
java.util.List
;
import
static
org
.
mockito
.
Mockito
.
when
;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
assertTrue
;
public
class
HAConfigurationTest
{
@Mock
private
Configuration
configuration
;
@BeforeMethod
public
void
setup
()
{
MockitoAnnotations
.
initMocks
(
this
);
System
.
setProperty
(
AtlasConstants
.
SYSTEM_PROPERTY_APP_PORT
,
AtlasConstants
.
DEFAULT_APP_PORT_STR
);
}
@Test
public
void
testShouldReturnHTTPSBoundAddress
()
{
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21443"
);
when
(
configuration
.
getBoolean
(
SecurityProperties
.
TLS_ENABLED
)).
thenReturn
(
true
);
String
address
=
HAConfiguration
.
getBoundAddressForId
(
configuration
,
"id1"
);
assertEquals
(
address
,
"https://127.0.0.1:21443"
);
}
@Test
public
void
testShouldReturnListOfAddressesInConfig
()
{
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{
"id1"
,
"id2"
});
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id2"
)).
thenReturn
(
"127.0.0.1:31000"
);
List
<
String
>
serverInstances
=
HAConfiguration
.
getServerInstances
(
configuration
);
assertEquals
(
serverInstances
.
size
(),
2
);
assertTrue
(
serverInstances
.
contains
(
"http://127.0.0.1:21000"
));
assertTrue
(
serverInstances
.
contains
(
"http://127.0.0.1:31000"
));
}
}
release-log.txt
View file @
98f4d40a
...
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
...
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ALL CHANGES:
ATLAS-571 Modify Atlas client for necessary changes in context of HA (yhemanth via sumasai)
ATLAS-620 Disable hbase based entity audit (shwethags)
ATLAS-620 Disable hbase based entity audit (shwethags)
ATLAS-618 Fix assembly for hdfs-module (sumasai via yhemanth)
ATLAS-618 Fix assembly for hdfs-module (sumasai via yhemanth)
ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai)
ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai)
...
...
server-api/src/main/java/org/apache/atlas/ha/AtlasServerIdSelector.java
0 → 100644
View file @
98f4d40a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org
.
apache
.
atlas
.
ha
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasException
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.hadoop.net.NetUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.net.InetSocketAddress
;
public
class
AtlasServerIdSelector
{
private
static
final
Logger
LOG
=
LoggerFactory
.
getLogger
(
AtlasServerIdSelector
.
class
);
/**
* Return the ID corresponding to this Atlas instance.
*
* The match is done by looking for an ID configured in {@link HAConfiguration#ATLAS_SERVER_IDS} key
* that has a host:port entry for the key {@link HAConfiguration#ATLAS_SERVER_ADDRESS_PREFIX}+ID where
* the host is a local IP address and port is set in the system property
* {@link AtlasConstants#SYSTEM_PROPERTY_APP_PORT}.
*
* @param configuration
* @return
* @throws AtlasException if no ID is found that maps to a local IP Address or port
*/
public
static
String
selectServerId
(
Configuration
configuration
)
throws
AtlasException
{
// ids are already trimmed by this method
String
[]
ids
=
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
);
String
matchingServerId
=
null
;
int
appPort
=
Integer
.
parseInt
(
System
.
getProperty
(
AtlasConstants
.
SYSTEM_PROPERTY_APP_PORT
));
for
(
String
id
:
ids
)
{
String
hostPort
=
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
id
);
if
(!
StringUtils
.
isEmpty
(
hostPort
))
{
InetSocketAddress
socketAddress
;
try
{
socketAddress
=
NetUtils
.
createSocketAddr
(
hostPort
);
}
catch
(
Exception
e
)
{
LOG
.
warn
(
"Exception while trying to get socket address for "
+
hostPort
,
e
);
continue
;
}
if
(!
socketAddress
.
isUnresolved
()
&&
NetUtils
.
isLocalAddress
(
socketAddress
.
getAddress
())
&&
appPort
==
socketAddress
.
getPort
())
{
LOG
.
info
(
"Found matched server id "
+
id
+
" with host port: "
+
hostPort
);
matchingServerId
=
id
;
break
;
}
}
else
{
LOG
.
info
(
"Could not find matching address entry for id: "
+
id
);
}
}
if
(
matchingServerId
==
null
)
{
String
msg
=
String
.
format
(
"Could not find server id for this instance. "
+
"Unable to find IDs matching any local host and port binding among %s"
,
StringUtils
.
join
(
ids
,
","
));
throw
new
AtlasException
(
msg
);
}
return
matchingServerId
;
}
}
server-api/src/main/test/org/apache/atlas/ha/
HAConfiguration
Test.java
→
server-api/src/main/test/org/apache/atlas/ha/
AtlasServerIdSelector
Test.java
View file @
98f4d40a
...
@@ -20,7 +20,6 @@ package org.apache.atlas.ha;
...
@@ -20,7 +20,6 @@ package org.apache.atlas.ha;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasConstants
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.security.SecurityProperties
;
import
org.apache.commons.configuration.Configuration
;
import
org.apache.commons.configuration.Configuration
;
import
org.mockito.Mock
;
import
org.mockito.Mock
;
import
org.mockito.MockitoAnnotations
;
import
org.mockito.MockitoAnnotations
;
...
@@ -31,8 +30,7 @@ import static org.mockito.Mockito.when;
...
@@ -31,8 +30,7 @@ import static org.mockito.Mockito.when;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
assertEquals
;
import
static
org
.
testng
.
Assert
.
fail
;
import
static
org
.
testng
.
Assert
.
fail
;
public
class
HAConfigurationTest
{
public
class
AtlasServerIdSelectorTest
{
@Mock
@Mock
private
Configuration
configuration
;
private
Configuration
configuration
;
...
@@ -48,14 +46,14 @@ public class HAConfigurationTest {
...
@@ -48,14 +46,14 @@ public class HAConfigurationTest {
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id2"
)).
thenReturn
(
"127.0.0.1:21000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id2"
)).
thenReturn
(
"127.0.0.1:21000"
);
String
atlasServerId
=
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
String
atlasServerId
=
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
assertEquals
(
atlasServerId
,
"id2"
);
assertEquals
(
atlasServerId
,
"id2"
);
}
}
@Test
(
expectedExceptions
=
AtlasException
.
class
)
@Test
(
expectedExceptions
=
AtlasException
.
class
)
public
void
testShouldFailIfNoIDsConfiguration
()
throws
AtlasException
{
public
void
testShouldFailIfNoIDsConfiguration
()
throws
AtlasException
{
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{});
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{});
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
fail
(
"Should not return any server id if IDs not found in configuration"
);
fail
(
"Should not return any server id if IDs not found in configuration"
);
}
}
...
@@ -64,27 +62,7 @@ public class HAConfigurationTest {
...
@@ -64,27 +62,7 @@ public class HAConfigurationTest {
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{
"id1"
,
"id2"
});
when
(
configuration
.
getStringArray
(
HAConfiguration
.
ATLAS_SERVER_IDS
)).
thenReturn
(
new
String
[]
{
"id1"
,
"id2"
});
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:31000"
);
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
fail
(
"Should not return any server id if no matching address found for any ID"
);
fail
(
"Should not return any server id if no matching address found for any ID"
);
}
}
@Test
public
void
testShouldReturnHTTPBoundAddress
()
{
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21000"
);
when
(
configuration
.
getBoolean
(
SecurityProperties
.
TLS_ENABLED
)).
thenReturn
(
false
);
String
address
=
HAConfiguration
.
getBoundAddressForId
(
configuration
,
"id1"
);
assertEquals
(
address
,
"http://127.0.0.1:21000"
);
}
@Test
public
void
testShouldReturnHTTPSBoundAddress
()
{
when
(
configuration
.
getString
(
HAConfiguration
.
ATLAS_SERVER_ADDRESS_PREFIX
+
"id1"
)).
thenReturn
(
"127.0.0.1:21443"
);
when
(
configuration
.
getBoolean
(
SecurityProperties
.
TLS_ENABLED
)).
thenReturn
(
true
);
String
address
=
HAConfiguration
.
getBoundAddressForId
(
configuration
,
"id1"
);
assertEquals
(
address
,
"https://127.0.0.1:21443"
);
}
}
}
webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
View file @
98f4d40a
...
@@ -113,7 +113,8 @@ public class QuickStart {
...
@@ -113,7 +113,8 @@ public class QuickStart {
private
final
AtlasClient
metadataServiceClient
;
private
final
AtlasClient
metadataServiceClient
;
QuickStart
(
String
baseUrl
)
{
QuickStart
(
String
baseUrl
)
{
metadataServiceClient
=
new
AtlasClient
(
baseUrl
);
String
[]
urls
=
baseUrl
.
split
(
","
);
metadataServiceClient
=
new
AtlasClient
(
null
,
null
,
urls
);
}
}
void
createTypes
()
throws
Exception
{
void
createTypes
()
throws
Exception
{
...
...
webapp/src/main/java/org/apache/atlas/web/service/ActiveInstanceElectorService.java
View file @
98f4d40a
...
@@ -23,6 +23,7 @@ import com.google.inject.Provider;
...
@@ -23,6 +23,7 @@ import com.google.inject.Provider;
import
com.google.inject.Singleton
;
import
com.google.inject.Singleton
;
import
org.apache.atlas.ApplicationProperties
;
import
org.apache.atlas.ApplicationProperties
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.AtlasException
;
import
org.apache.atlas.ha.AtlasServerIdSelector
;
import
org.apache.atlas.ha.HAConfiguration
;
import
org.apache.atlas.ha.HAConfiguration
;
import
org.apache.atlas.listener.ActiveStateChangeHandler
;
import
org.apache.atlas.listener.ActiveStateChangeHandler
;
import
org.apache.atlas.service.Service
;
import
org.apache.atlas.service.Service
;
...
@@ -101,7 +102,7 @@ public class ActiveInstanceElectorService implements Service, LeaderLatchListene
...
@@ -101,7 +102,7 @@ public class ActiveInstanceElectorService implements Service, LeaderLatchListene
return
;
return
;
}
}
cacheActiveStateChangeHandlers
();
cacheActiveStateChangeHandlers
();
serverId
=
HAConfiguration
.
getAtlas
ServerId
(
configuration
);
serverId
=
AtlasServerIdSelector
.
select
ServerId
(
configuration
);
joinElection
();
joinElection
();
}
}
...
...
webapp/src/main/java/org/apache/atlas/web/service/CuratorFactory.java
View file @
98f4d40a
...
@@ -85,7 +85,7 @@ public class CuratorFactory {
...
@@ -85,7 +85,7 @@ public class CuratorFactory {
* Create a new instance {@link LeaderLatch}
* Create a new instance {@link LeaderLatch}
* @param serverId the ID used to register this instance with curator.
* @param serverId the ID used to register this instance with curator.
* This ID should typically be obtained using
* This ID should typically be obtained using
* {@link
HAConfiguration#getAtlas
ServerId(Configuration)}
* {@link
org.apache.atlas.ha.AtlasServerIdSelector#select
ServerId(Configuration)}
* @return
* @return
*/
*/
public
LeaderLatch
leaderLatchInstance
(
String
serverId
)
{
public
LeaderLatch
leaderLatchInstance
(
String
serverId
)
{
...
...
webapp/src/test/java/org/apache/atlas/web/security/NegativeSSLAndKerberosTest.java
View file @
98f4d40a
...
@@ -86,7 +86,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
...
@@ -86,7 +86,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
dgiClient
=
new
AtlasClient
(
DGI_URL
)
{
dgiClient
=
new
AtlasClient
(
DGI_URL
)
{
@Override
@Override
protected
PropertiesConfiguration
getClientProperties
()
throws
AtlasException
{
protected
PropertiesConfiguration
getClientProperties
()
{
return
configuration
;
return
configuration
;
}
}
};
};
...
...
webapp/src/test/java/org/apache/atlas/web/security/SSLAndKerberosTest.java
View file @
98f4d40a
...
@@ -102,7 +102,7 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
...
@@ -102,7 +102,7 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
public
AtlasClient
run
()
throws
Exception
{
public
AtlasClient
run
()
throws
Exception
{
return
new
AtlasClient
(
DGI_URL
)
{
return
new
AtlasClient
(
DGI_URL
)
{
@Override
@Override
protected
PropertiesConfiguration
getClientProperties
()
throws
AtlasException
{
protected
PropertiesConfiguration
getClientProperties
()
{
return
configuration
;
return
configuration
;
}
}
};
};
...
...
webapp/src/test/java/org/apache/atlas/web/security/SSLTest.java
View file @
98f4d40a
...
@@ -78,7 +78,7 @@ public class SSLTest extends BaseSSLAndKerberosTest {
...
@@ -78,7 +78,7 @@ public class SSLTest extends BaseSSLAndKerberosTest {
dgiCLient
=
new
AtlasClient
(
DGI_URL
)
{
dgiCLient
=
new
AtlasClient
(
DGI_URL
)
{
@Override
@Override
protected
PropertiesConfiguration
getClientProperties
()
throws
AtlasException
{
protected
PropertiesConfiguration
getClientProperties
()
{
return
configuration
;
return
configuration
;
}
}
};
};
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment