Commit f31db4c3 by kevalbhatt Committed by nixonrodrigues

ATLAS-3386 : Atlas documentation website with new template

parent 05130e83
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
.DS_Store
.bower-*/
.idea/
node_modules/
.docz
target/
*.log
*.tgz
export default {
'~docz-theme-component': '../node_modules/theme/dist/components/ui/index.ts'
}
\ No newline at end of file
export default [
"Introduction",
{
name: "Documentation",
menu: [
{
name: "Features",
menu: [
"High Level Architecture",
"Type System",
"Glossary",
"Classification Propagation",
"Notifications",
"High Availability"
]
},
{
name: "Search",
menu: [
"Basic Search",
"Advance Search"
]
},
{
name: "Hooks",
menu: [
"HBase",
"Hive",
"Sqoop",
"Storm",
"Kafka",
"Falcon"
]
},
{
name: "Import/Export",
menu: [
"Import API",
"Import API Options",
"Import Entity Transforms",
"Import Export API",
"Export API",
"Export HDFS API",
"Export Import Audits",
"Incremental Export"
]
},
{
name: "Security",
menu: [
"Security Details",
"Authentication",
"Authorization Model",
"Atlas Simple Authorizer",
"Atlas Ranger Authorizer"
]
},
{
name: "Setup",
menu: [
"Quick Start",
"Configuration",
"Build Instruction",
"Installation Instruction"
]
},
"REST API",
{
name: "Tools",
menu: [
"Atlas Repair Index"
]
},
{
name: "Misc",
menu: [
"Atlas Server",
"Replicated Attributes",
"Soft Reference"
]
}
]
}, {
name: "Downloads",
menu: [
"Download",
{
name: "Whats New",
menu: ["WhatsNew-2.0", "WhatsNew-1.0"]
},
"Migration"
]
} ,"For Developers",
{
name: "Project Info", menu: [
"Project Information",
"Mailing Lists",
"Team List",
"Issue Tracking",
"License",
"Source Repository"
]
}, "ASF" ];
export default [
{ id: 1, href: "/Doc-test/", title: "Latest", label: "Latest" },
{ id: 2, href: "/2.0.0/index.html", title: "2.0.0", label: "2.0.0" },
{ id: 4, href: "/1.1.0/index.html", title: "1.1.0", label: "1.1.0" },
{ id: 5, href: "/1.0.0/index.html", title: "1.0.0", label: "1.0.0" },
{ id: 7, href: "/0.8.3/index.html", title: "0.8.3", label: "0.8.3" },
{ id: 8, href: "/0.8.2/index.html", title: "0.8.2", label: "0.8.2" },
{ id: 9, href: "/0.8.1/index.html", title: "0.8.1", label: "0.8.1" },
{
id: 10,
href: "/0.8.0-incubating/index.html",
title: "0.8-incubating",
label: "0.8-incubating"
},
{
id: 11,
href: "/0.7.1-incubating/index.html",
title: "0.7.1-incubating",
label: "0.7.1-incubating"
},
{
id: 12,
href: "/0.7.0-incubating/index.html",
title: "0.7-incubating",
label: "0.7-incubating"
},
{
id: 13,
href: "/0.6.0-incubating/index.html",
title: "0.6-incubating",
label: "0.6-incubating"
},
{
id: 14,
href: "/0.5.0-incubating/index.html",
title: "0.5-incubating",
label: "0.5-incubating"
}
];
import webpack from 'webpack';
export declare const build: (config: webpack.Configuration, dist: string, publicDir: string) => Promise<void>;
import { Configuration } from 'webpack';
import { ServerHooks as Hooks } from '../lib/Bundler';
import { Config as Args, Env } from '../config/argv';
export declare const createConfig: (args: Args, env: Env) => (hooks: Hooks) => Promise<Configuration>;
import { Config as Args } from '../config/argv';
import { ServerHooks } from '../lib/Bundler';
export declare const devServerConfig: (hooks: ServerHooks, args: Args) => {
publicPath: string;
compress: boolean;
logLevel: string;
clientLogLevel: string;
contentBase: string[];
watchContentBase: boolean;
hot: boolean;
quiet: boolean;
open: boolean;
watchOptions: {
ignored: any;
};
overlay: boolean;
host: string;
port: number;
historyApiFallback: {
disableDotRule: boolean;
};
disableHostCheck: boolean;
before(app: any, server: any): void;
after(app: any): void;
};
import { Configuration as CFG } from 'webpack';
import { Bundler } from '../lib/Bundler';
import { Config as Args, Env } from '../config/argv';
export declare const bundler: (args: Args, env: Env) => Bundler<CFG>;
import Config from 'webpack-chain';
import { Config as Args } from '../config/argv';
import { BabelRC } from '../config/babel';
export declare const sourceMaps: (config: Config, args: Args) => void;
export interface AddScriptLoaderOpts {
threadLoader?: boolean;
rule: Config.Rule;
babelrc: BabelRC;
args: Args;
}
export declare const js: (config: Config, args: Args, babelrc: BabelRC) => void;
export declare const ts: (config: Config, args: Args, babelrc: BabelRC) => void;
export declare const mdx: (config: Config, args: Args, babelrc: BabelRC) => void;
export declare const images: (config: Config) => void;
export declare const svg: (config: Config) => void;
export declare const media: (config: Config) => void;
export declare const fonts: (config: Config) => void;
import Config from 'webpack-chain';
import { Config as Args } from '../config/argv';
export declare const minifier: (config: Config, args: Args) => void;
import Config from 'webpack-chain';
import { Config as Args, Env } from '../config/argv';
export declare const assets: (config: Config, args: Args, env: Env) => void;
export declare const analyzer: (config: Config) => void;
export declare const injections: (config: Config, args: Args, env: Env) => void;
export declare const ignore: (config: Config) => void;
export declare const hot: (config: Config) => void;
export declare const html: (config: Config, args: Args, env: Env) => Promise<void>;
export declare const webpackBar: (config: Config, args: Args) => void;
export declare const watchNodeModulesPlugin: (config: Config) => void;
export declare const notFoundPlugin: (config: Config) => void;
import webpack from 'webpack';
import { Config as Args } from '../config/argv';
import { ServerHooks as Hooks } from '../lib/Bundler';
export declare const server: (args: Args) => (config: webpack.Configuration, hooks: Hooks) => Promise<{
start: () => Promise<any>;
}>;
export declare const cli: () => {
[x: string]: unknown;
_: string[];
$0: string;
};
export declare const build: (args: any) => Promise<void>;
export declare const dev: (args: any) => Promise<void>;
export { dev } from './dev';
export { build } from './build';
export { serve } from './serve';
export declare const serve: (args: any) => Promise<void>;
import { Argv as Yargs } from 'yargs';
import { Plugin } from '../lib/Plugin';
import { BabelRC } from '../config/babel';
export declare type Env = 'production' | 'development';
export declare type ThemeConfig = Record<string, any>;
export interface DocgenConfig {
handlers?: any[];
resolver?: (ast: any, recast: any) => any;
propFilter?: (prop: any) => boolean;
searchPath: string;
}
export interface Menu {
name: string;
route?: string;
href?: string;
menu?: Menu[];
}
export interface HtmlContext {
lang: string;
favicon?: string;
head?: {
meta: any[];
links: any[];
raw: string;
scripts: any[];
};
body?: {
raw: string;
scripts: any[];
};
}
export interface Argv {
base: string;
src: string;
files: string | string[];
ignore: string[];
watchIgnore: string;
public: string;
dest: string;
editBranch: string;
config: string;
debug: boolean;
clearConsole: boolean;
typescript: boolean;
propsParser: boolean;
host: string;
port: number;
websocketPort: number;
websocketHost: string;
native: boolean;
codeSandbox: boolean;
sourcemaps: boolean;
notUseSpecifiers: boolean;
title: string;
description: string;
theme: string;
wrapper?: string;
indexHtml?: string;
/** slugify separator */
separator: string;
}
export interface Config extends Argv {
paths: Record<string, any>;
plugins: Plugin[];
mdPlugins: any[];
hastPlugins: any[];
menu: Menu[];
htmlContext: HtmlContext;
themeConfig: ThemeConfig;
docgenConfig: DocgenConfig;
filterComponents: (files: string[]) => string[];
modifyBundlerConfig<C>(config: C, dev: boolean, args: Config): C;
modifyBabelRc(babelrc: BabelRC, args: Config): BabelRC;
onCreateWebpackChain<C>(c: C, dev: boolean, args: Config): void;
}
export declare const setArgs: (yargs: Yargs<{}>) => Yargs<{
base: any;
} & {
source: any;
} & {
files: any;
} & {
ignore: any;
} & {
public: any;
} & {
dest: any;
} & {
editBranch: any;
} & {
config: any;
} & {
title: any;
} & {
description: any;
} & {
theme: any;
} & {
typescript: any;
} & {
propsParser: any;
} & {
wrapper: any;
} & {
indexHtml: any;
} & {
debug: any;
} & {
clearConsole: any;
} & {
host: any;
} & {
port: any;
} & {
websocketHost: any;
} & {
websocketPort: any;
} & {
native: any;
} & {
codeSandbox: any;
} & {
sourcemaps: any;
} & {
separator: any;
} & {
open: boolean;
}>;
import { Config, Env } from '../config/argv';
export interface BabelRC {
presets: any[];
plugins: any[];
cacheDirectory?: boolean;
babelrc?: boolean;
}
export declare const getBabelConfig: (args: Config, env: Env, typescript?: boolean | undefined) => Promise<BabelRC>;
import { Arguments } from 'yargs';
import { BabelRC } from '../config/babel';
import { Config, Argv } from '../config/argv';
export declare const doczRcBaseConfig: {
htmlContext: {
lang: string;
favicon: string;
};
themeConfig: {};
docgenConfig: {};
filterComponents: (files: string[]) => string[];
modifyBundlerConfig: (config: any) => any;
modifyBabelRc: (babelrc: BabelRC) => BabelRC;
onCreateWebpackChain: () => null;
menu: never[];
plugins: never[];
mdPlugins: never[];
hastPlugins: never[];
ignore: string[];
};
export declare const getBaseConfig: (argv: Arguments<Argv>, custom?: Partial<Config> | undefined) => Config;
export declare const parseConfig: (argv: Arguments<Argv>, custom?: Partial<Config> | undefined) => Promise<Config>;
export declare const setEnv: (env: string) => void;
export interface RT {
[key: string]: any;
}
export declare const getClientEnvironment: (publicUrl: string) => {
raw: RT;
stringified: {
'process.env': {};
};
};
export declare const ensureSlash: (filepath: any, needsSlash: boolean) => any;
export declare const root: string;
export declare const resolveApp: (to: string) => string;
export declare const resolveOwn: (to: string) => string;
export interface Paths {
root: string;
templates: string;
packageJson: string;
servedPath: (base: string) => string;
docz: string;
app: string;
cache: string;
appPublic: string;
appNodeModules: string;
appPackageJson: string;
appYarnLock: string;
ownNodeModules: string;
getDist: (dest: string) => string;
distPublic: (dest: string) => string;
importsJs: string;
rootJs: string;
indexJs: string;
indexHtml: string;
db: string;
}
export declare const templates: string;
export declare const packageJson: string;
export declare const servedPath: (base: string) => any;
export declare const docz: string;
export declare const app: string;
export declare const cache: string;
export declare const appPublic: string;
export declare const appNodeModules: string;
export declare const appPackageJson: string;
export declare const appYarnLock: string;
export declare const ownNodeModules: string;
export declare const getDist: (dest: string) => string;
export declare const distPublic: (dest: string) => string;
export declare const importsJs: string;
export declare const rootJs: string;
export declare const indexJs: string;
export declare const indexHtml: string;
export declare const db: string;
/** cli exports */
export { cli } from './cli';
/** config exports */
export { parseConfig, getBaseConfig } from './config/docz';
export { Config, setArgs } from './config/argv';
export { BabelRC } from './config/babel';
/** states */
import * as states from './states';
export { states };
/** lib exports */
export { Plugin, createPlugin } from './lib/Plugin';
export { DataServer } from './lib/DataServer';
export { Entries } from './lib/Entries';
export { Entry } from './lib/Entry';
/// <reference types="node" />
import * as http from 'http';
import { Config as Args, Env } from '../config/argv';
export interface ServerHooks {
onCreateWebpackChain<C>(config: C, dev: boolean, args: Args): void;
onPreCreateApp<A>(app: A): void;
onCreateApp<A>(app: A): void;
onServerListening<S>(server: S): void;
}
export interface BundlerServer {
start(): Promise<http.Server>;
}
export declare type ConfigFn<C> = (hooks: ServerHooks) => Promise<C>;
export declare type BuildFn<C> = (config: C, dist: string, publicDir: string) => void;
export declare type ServerFn<C> = (config: C, hooks: ServerHooks) => BundlerServer | Promise<BundlerServer>;
export interface BundlerConstructor<Config> {
args: Args;
config: ConfigFn<Config>;
server: ServerFn<Config>;
build: BuildFn<Config>;
}
export interface ConfigObj {
[key: string]: any;
}
export declare class Bundler<C = ConfigObj> {
private readonly args;
private config;
private server;
private builder;
private hooks;
constructor(params: BundlerConstructor<C>);
mountConfig(env: Env): Promise<C>;
createApp(config: C): Promise<BundlerServer>;
build(config: C): Promise<void>;
}
export interface Params {
getState: () => Record<string, any>;
setState: (key: string, val: any) => void;
}
export interface State {
id: string;
start: (params: Params) => Promise<void>;
close: () => void;
}
export interface Action {
type: string;
payload: any;
}
export declare type Listener = (action: Action) => void;
export declare class DataServer {
private states;
private state;
private listeners;
constructor();
register(states: State[]): DataServer;
start(): Promise<void>;
close(): void;
onStateChange(listener: Listener): () => void;
getState(): Map<string, any>;
private setState;
private writeDbFile;
private mapToObject;
}
import { EntryObj } from './Entry';
import { Config } from '../config/argv';
export declare const fromTemplates: (file: string) => string;
export declare type EntryMap = Record<string, EntryObj>;
export declare class Entries {
static writeApp(config: Config, dev: boolean): Promise<void>;
static writeImports(map: EntryMap): Promise<void>;
all: Map<string, EntryObj>;
get: () => Promise<EntryMap>;
repoEditUrl: string | null;
constructor(config: Config);
private getMap;
}
import { Heading } from 'docz-utils/lib/mdast';
import { Config } from '../config/argv';
export interface EntryObj {
id: string;
filepath: string;
link: string | null;
slug: string;
name: string;
route: string;
menu: string | null;
headings: Heading[];
[key: string]: any;
}
export declare class Entry {
readonly [key: string]: any;
id: string;
filepath: string;
link: string | null;
slug: string;
route: string;
name: string;
menu: string | null;
headings: Heading[];
settings: {
[key: string]: any;
};
constructor(ast: any, file: string, src: string, config: Config);
setLink(url: string): void;
private getFilepath;
private getName;
private slugify;
private getRoute;
}
import WebpackChainConfig from 'webpack-chain';
import { Config } from '../config/argv';
import { BabelRC } from '../config/babel';
export declare type SetConfig = (config: Config) => Config | Promise<Config>;
export declare type ModifyBundlerConfig<C = any> = (config: C, dev: boolean, args: Config) => C;
export declare type ModifyBabelRC = (babelrc: BabelRC, args: Config) => BabelRC;
export declare type ModifyFiles = (files: string[], args: Config) => string[];
export declare type OnCreateWebpackChain = (config: WebpackChainConfig, dev: boolean, args: Config) => void;
export declare type onPreCreateApp = <A>(app: A) => void;
export declare type onCreateApp = <A>(app: A) => void;
export declare type OnServerListening = <S>(server: S) => void;
export declare type OnPreBuild = (args: Config) => void;
export declare type OnPostBuild = (args: Config) => void;
export declare type OnPreRender = () => void;
export declare type OnPostRender = () => void;
export interface PluginFactory {
setConfig?: SetConfig;
modifyBundlerConfig?: ModifyBundlerConfig;
modifyBabelRc?: ModifyBabelRC;
modifyFiles?: ModifyFiles;
onCreateWebpackChain?: OnCreateWebpackChain;
onPreCreateApp?: onPreCreateApp;
onCreateApp?: onCreateApp;
onServerListening?: OnServerListening;
onPreBuild?: OnPreBuild;
onPostBuild?: OnPostBuild;
onPreRender?: OnPreRender;
onPostRender?: OnPostRender;
}
export declare class Plugin<C = any> implements PluginFactory {
static runPluginsMethod(plugins: Plugin[] | undefined): (method: keyof Plugin, ...args: any[]) => void;
static propsOfPlugins(plugins: Plugin[]): (prop: keyof Plugin) => any[];
static reduceFromPlugins<C>(plugins: Plugin[] | undefined): (method: keyof Plugin, initial: C, ...args: any[]) => C;
static reduceFromPluginsAsync<C>(plugins: Plugin[] | undefined): (method: keyof Plugin, initial: C, ...args: any[]) => Promise<C>;
readonly setConfig?: SetConfig;
readonly modifyBundlerConfig?: ModifyBundlerConfig<C>;
readonly modifyBabelRc?: ModifyBabelRC;
readonly modifyFiles?: ModifyFiles;
readonly onCreateWebpackChain?: OnCreateWebpackChain;
readonly onPreCreateApp?: onPreCreateApp;
readonly onCreateApp?: onCreateApp;
readonly onServerListening?: OnServerListening;
readonly onPreBuild?: OnPreBuild;
readonly onPostBuild?: OnPostBuild;
readonly onPreRender?: OnPreRender;
readonly onPostRender?: OnPostRender;
constructor(p: PluginFactory);
}
export declare function createPlugin<C = any>(factory: PluginFactory): Plugin<C>;
import WS from 'ws';
export declare type Send = (type: string, payload: any) => void;
export declare type On = (type: string) => Promise<any>;
export declare class Socket {
private client?;
constructor(server?: any, host?: string, port?: number);
onConnection(listener: (socket: WS, emit: Send) => () => void): void;
}
import { State } from '../lib/DataServer';
import { Config } from '../config/argv';
export declare const state: (config: Config, dev?: boolean | undefined) => State;
import { State } from '../lib/DataServer';
import { Entries } from '../lib/Entries';
import { Config } from '../config/argv';
export declare const state: (entries: Entries, config: Config, dev?: boolean | undefined) => State;
export { state as entries } from './entries';
export { state as config } from './config';
export { state as props } from './props';
import { State } from '../lib/DataServer';
import { Config } from '../config/argv';
export declare const state: (config: Config, dev?: boolean | undefined) => State;
export const imports = {
<% entries.forEach(function(entry) { %>'<%- entry.filepath %>': () =>
import(/* webpackPrefetch: true, webpackChunkName: "<%- entry.slug %>" */ '<%- entry.filepath %>'),<% }) %>
}
<!DOCTYPE html>
<html lang="{{ lang }}">
<head>
<meta charset="UTF-8">
<meta name="description" content="{{ description }}">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>{{ title }}</title>
{{ head }}
</head>
<body>
<div id="root"></div>
{{ footer }}
</body>
</html>
import React from 'react'
import ReactDOM from 'react-dom'
import Root from './root'
const _onPreRenders = [<% if (onPreRenders) {%><%- onPreRenders %><%}%>]
const _onPostRenders = [<% if (onPostRenders) {%><%- onPostRenders %><%}%>]
const onPreRender = () => _onPreRenders.forEach(f => f && f())
const onPostRender = () => _onPostRenders.forEach(f => f && f())
const root = document.querySelector('#root')
const render = (Component = Root) => {
onPreRender()
ReactDOM.render(<Component />, root, onPostRender)
}
render(Root)
import React from 'react'
import { Link, Router, Routes<% if (!isProd) {%>, useDataServer<%}%> } from '../../docz-lib/docz/dist'
<% if (!isProd) {%>import { hot } from 'react-hot-loader'<%}%>
import Theme from '<%- theme %>'
import { imports } from './imports'
import database from './db.json'
<% if (wrapper) {%>import Wrapper from '<%- wrapper %>'<%}%>
const Root = () => {
<% if (!isProd && websocketUrl) {%>useDataServer('<%- websocketUrl %>')<%}%>
return (
<Theme
<% if (wrapper) {%>wrapper={Wrapper}<%}%>
linkComponent={Link}
db={database}
>
<Routes imports={imports} />
</Theme>
)
}
<% if (!isProd) {%>export default hot(module)(Root)<%}%>
<% if (isProd) {%>export default Root<%}%>
import { Config } from '../../config/argv';
export declare const docgen: (files: string[], config: Config) => Promise<any>;
import { Config } from '../../config/argv';
export declare const jsParser: (files: string[], config: Config) => ({
key: string;
value: any;
} | null)[];
import { Config } from '../../config/argv';
export interface TSFile {
text?: string;
version: number;
}
export declare const tsParser: (files: string[], config: Config, tsconfig?: string | undefined) => any;
export declare const onSignal: (cb: () => any) => void;
/**
* Copyright (c) 2015-present, Facebook, Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/**
* Reads the BROWSER environment variable and decides what to do with it. Returns
* true if it opened a browser or ran a node.js script, otherwise false.
*/
export declare function openBrowser(url: any): boolean;
import { Config } from '../config/argv';
export declare type tagsTemplate = (type: string) => string;
export declare const htmlTemplate: (indexHtml: string | undefined) => Promise<(data: any) => string>;
interface ParseHtmlParams {
config: Config;
ctx: Record<string, any>;
dev: boolean;
template: (props: Record<string, any>) => string;
}
export declare const parseHtml: ({ config, ctx, dev, template }: ParseHtmlParams) => any;
export {};
export declare const parseRepo: () => any;
export declare const getRepoUrl: () => any;
export declare const getRepoEditUrl: (src: string, branch: string) => string | null;
#!/usr/bin/env nodejs
require('../../docz-core/dist/index.js').cli()
import { createElement } from 'react';
import 'lodash/fp/get';
import { b as useComponents } from './chunk.esm.js';
import 'lodash/fp/omit';
import 'fast-deep-equal';
import 'lodash/fp/merge';
import 'array-sort';
import 'lodash/fp/unionBy';
import 'lodash/fp/flattenDepth';
import 'lodash/fp/pipe';
import 'ulid';
import 'match-sorter';
import 'lodash/fp/throttle';
const Playground = ({
className,
style,
wrapper: Wrapper,
children,
__scope,
__position,
__code,
__codesandbox
}) => {
const components = useComponents();
if (!components || !components.playground) return null;
const props = {
className,
style,
components
};
return createElement(components.playground, Object.assign({}, props, {
component: children,
wrapper: Wrapper,
scope: __scope,
position: __position,
code: __code,
codesandbox: __codesandbox
}));
};
export default Playground;
'use strict';
var React = require('react');
require('lodash/fp/get');
var __chunk_1 = require('./chunk.js');
require('lodash/fp/omit');
require('fast-deep-equal');
require('lodash/fp/merge');
require('array-sort');
require('lodash/fp/unionBy');
require('lodash/fp/flattenDepth');
require('lodash/fp/pipe');
require('ulid');
require('match-sorter');
require('lodash/fp/throttle');
const Playground = ({
className,
style,
wrapper: Wrapper,
children,
__scope,
__position,
__code,
__codesandbox
}) => {
const components = __chunk_1.useComponents();
if (!components || !components.playground) return null;
const props = {
className,
style,
components
};
return React.createElement(components.playground, Object.assign({}, props, {
component: children,
wrapper: Wrapper,
scope: __scope,
position: __position,
code: __code,
codesandbox: __codesandbox
}));
};
exports.default = Playground;
import { createContext, createElement, useContext, Fragment, Component, useEffect, useMemo, useRef, useState } from 'react';
import _get from 'lodash/fp/get';
import _omit from 'lodash/fp/omit';
import equal from 'fast-deep-equal';
import _merge from 'lodash/fp/merge';
import sort from 'array-sort';
import _unionBy from 'lodash/fp/unionBy';
import _flattenDepth from 'lodash/fp/flattenDepth';
import _pipe from 'lodash/fp/pipe';
import { ulid } from 'ulid';
import match from 'match-sorter';
import _throttle from 'lodash/fp/throttle';
const DefaultNotFound = () => createElement(Fragment, null, "Not found");
const DefaultLoading = () => createElement(Fragment, null, "Loading");
const DefaultPage = ({
children
}) => createElement(Fragment, null, children);
const DefaultPlayground = ({
component,
code
}) => createElement(Fragment, null, component, code);
const defaultComponents = {
loading: DefaultLoading,
playground: DefaultPlayground,
notFound: DefaultNotFound,
page: DefaultPage
};
const ctx = createContext({});
const ComponentsProvider = ({
components: themeComponents = {},
children
}) => createElement(ctx.Provider, {
value: Object.assign({}, defaultComponents, themeComponents)
}, children);
const useComponents = () => {
return useContext(ctx);
};
const isFn = value => typeof value === 'function';
function flatArrFromObject(arr, prop) {
const reducer = (arr, obj) => {
const value = _get(prop)(obj);
return value ? arr.concat([value]) : arr;
};
return Array.from(new Set(arr.reduce(reducer, [])));
}
function compare(a, b, reverse) {
if (a < b) return reverse ? 1 : -1;
if (a > b) return reverse ? -1 : 1;
return 0;
}
function create(initial) {
var _a;
const ctx = createContext(initial);
const listeners = new Set();
const dispatch = fn => {
listeners.forEach(listener => listener(fn));
};
return {
context: ctx,
set: fn => dispatch(fn),
Provider: (_a = class Provider extends Component {
constructor() {
super(...arguments);
this.state = this.props.initial || initial || {};
}
static getDerivedStateFromProps(props, state) {
if (!equal(props.initial, state)) return props.initial;
return null;
}
componentDidMount() {
listeners.add(fn => this.setState(fn));
}
componentWillUnmount() {
listeners.clear();
}
render() {
return createElement(ctx.Provider, {
value: this.state
}, this.props.children);
}
}, _a.displayName = 'DoczStateProvider', _a)
};
}
const doczState = create({});
const useConfig = () => {
const state = useContext(doczState.context);
const {
linkComponent,
transform,
config,
themeConfig = {}
} = state;
const newConfig = _merge(themeConfig, config ? config.themeConfig : {});
const transformed = transform ? transform(newConfig) : newConfig;
return Object.assign({}, config, {
linkComponent,
themeConfig: transformed
});
};
const updateState = ev => {
const {
type,
payload
} = JSON.parse(ev.data);
const prop = type.startsWith('state.') && type.split('.')[1];
if (prop) {
doczState.set(state => Object.assign({}, state, {
[prop]: payload
}));
}
};
const useDataServer = url => {
useEffect(() => {
if (!url) return;
const socket = new WebSocket(url);
socket.onmessage = updateState;
return () => socket.close();
}, []);
};
const useDocs = () => {
const {
entries = []
} = useContext(doczState.context);
const arr = entries.map(({
value
}) => value);
return sort(arr, (a, b) => compare(a.name, b.name));
};
const noMenu = entry => !entry.menu;
const fromMenu = menu => entry => entry.menu === menu;
const entryAsMenu = entry => ({
name: entry.name,
route: entry.route,
parent: entry.parent
});
const entriesOfMenu = (menu, entries) => entries.filter(fromMenu(menu)).map(entryAsMenu);
const parseMenu = entries => name => ({
name,
menu: entriesOfMenu(name, entries)
});
const menusFromEntries = entries => {
const entriesWithoutMenu = entries.filter(noMenu).map(entryAsMenu);
const menus = flatArrFromObject(entries, 'menu').map(parseMenu(entries));
return _unionBy('name', menus, entriesWithoutMenu);
};
const parseItemStr = item => typeof item === 'string' ? {
name: item
} : item;
const normalize = item => {
const selected = parseItemStr(item);
return Object.assign({}, selected, {
id: selected.id || ulid(),
parent: _get('parent', selected) || _get('parent', item),
menu: Array.isArray(selected.menu) ? selected.menu.map(normalize) : selected.menu
});
};
const clean = item => item.href || item.route ? _omit('menu', item) : item;
const normalizeAndClean = _pipe(normalize, clean);
const mergeMenus = (entriesMenu, configMenu) => {
const first = entriesMenu.map(normalizeAndClean);
const second = configMenu.map(normalizeAndClean);
const merged = _unionBy('name', first, second);
return merged.map(item => {
if (!item.menu) return item;
const found = second.find(i => i.name === item.name);
const foundMenu = found && found.menu;
return Object.assign({}, item, {
menu: foundMenu ? mergeMenus(item.menu, foundMenu) : item.menu || found.menu
});
});
};
const UNKNOWN_POS = Infinity;
const findPos = (item, orderedList = []) => {
const name = typeof item !== 'string' ? _get('name', item) : item;
const pos = orderedList.findIndex(item => item === name);
return pos !== -1 ? pos : UNKNOWN_POS;
};
const compareWithMenu = (to = []) => (a, b) => {
const list = to.map(i => i.name || i);
return compare(findPos(a, list), findPos(b, list));
};
const sortByName = (a, b) => {
return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
};
const sortMenus = (first, second = []) => {
const sorted = sort(first, compareWithMenu(second), sortByName);
return sorted.map(item => {
if (!item.menu) return item;
const found = second.find(menu => menu.name === item.name);
const foundMenu = found && found.menu;
return Object.assign({}, item, {
menu: foundMenu ? sortMenus(item.menu, foundMenu) : sort(item.menu, sortByName)
});
});
};
const search = (val, menu) => {
const items = menu.map(item => [item].concat(item.menu || []));
const flattened = _flattenDepth(2, items);
const flattenedDeduplicated = [...new Set(flattened)];
return match(flattenedDeduplicated, val, {
keys: ['name']
});
};
const filterMenus = (items, filter) => {
if (!filter) return items;
return items.filter(filter).map(item => {
if (!item.menu) return item;
return Object.assign({}, item, {
menu: item.menu.filter(filter)
});
});
};
const useMenus = opts => {
const {
query = ''
} = opts || {};
const {
entries,
config
} = useContext(doczState.context);
if (!entries) return null;
const arr = entries.map(({
value
}) => value);
const entriesMenu = menusFromEntries(arr);
const sorted = useMemo(() => {
const merged = mergeMenus(entriesMenu, config.menu);
const result = sortMenus(merged, config.menu);
return filterMenus(result, opts && opts.filter);
}, [entries, config]);
return query && query.length > 0 ? search(query, sorted) : sorted;
};
const usePrevious = (value, defaultValue) => {
const ref = useRef(defaultValue);
useEffect(() => {
ref.current = value;
});
return ref.current;
};
const isClient = typeof window === 'object';
const getSize = (initialWidth, initialHeight) => ({
innerHeight: isClient ? window.innerHeight : initialHeight,
innerWidth: isClient ? window.innerWidth : initialWidth,
outerHeight: isClient ? window.outerHeight : initialHeight,
outerWidth: isClient ? window.outerWidth : initialWidth
});
const useWindowSize = (throttleMs = 300, initialWidth = Infinity, initialHeight = Infinity) => {
const [windowSize, setWindowSize] = useState(getSize(initialHeight, initialHeight));
const tSetWindowResize = _throttle(throttleMs, () => setWindowSize(getSize(initialHeight, initialHeight)));
useEffect(() => {
window.addEventListener('resize', tSetWindowResize);
return () => void window.removeEventListener('resize', tSetWindowResize);
}, []);
return windowSize;
};
export { isFn as a, useComponents as b, doczState as c, useConfig as d, useDataServer as e, useDocs as f, useMenus as g, usePrevious as h, useWindowSize as i, ComponentsProvider as j };
'use strict';
function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }
var React = require('react');
var _get = _interopDefault(require('lodash/fp/get'));
var _omit = _interopDefault(require('lodash/fp/omit'));
var equal = _interopDefault(require('fast-deep-equal'));
var _merge = _interopDefault(require('lodash/fp/merge'));
var sort = _interopDefault(require('array-sort'));
var _unionBy = _interopDefault(require('lodash/fp/unionBy'));
var _flattenDepth = _interopDefault(require('lodash/fp/flattenDepth'));
var _pipe = _interopDefault(require('lodash/fp/pipe'));
var ulid = require('ulid');
var match = _interopDefault(require('match-sorter'));
var _throttle = _interopDefault(require('lodash/fp/throttle'));
const DefaultNotFound = () => React.createElement(React.Fragment, null, "Not found");
const DefaultLoading = () => React.createElement(React.Fragment, null, "Loading");
const DefaultPage = ({
children
}) => React.createElement(React.Fragment, null, children);
const DefaultPlayground = ({
component,
code
}) => React.createElement(React.Fragment, null, component, code);
const defaultComponents = {
loading: DefaultLoading,
playground: DefaultPlayground,
notFound: DefaultNotFound,
page: DefaultPage
};
const ctx = React.createContext({});
const ComponentsProvider = ({
components: themeComponents = {},
children
}) => React.createElement(ctx.Provider, {
value: Object.assign({}, defaultComponents, themeComponents)
}, children);
const useComponents = () => {
return React.useContext(ctx);
};
const isFn = value => typeof value === 'function';
function flatArrFromObject(arr, prop) {
const reducer = (arr, obj) => {
const value = _get(prop)(obj);
return value ? arr.concat([value]) : arr;
};
return Array.from(new Set(arr.reduce(reducer, [])));
}
function compare(a, b, reverse) {
if (a < b) return reverse ? 1 : -1;
if (a > b) return reverse ? -1 : 1;
return 0;
}
function create(initial) {
var _a;
const ctx = React.createContext(initial);
const listeners = new Set();
const dispatch = fn => {
listeners.forEach(listener => listener(fn));
};
return {
context: ctx,
set: fn => dispatch(fn),
Provider: (_a = class Provider extends React.Component {
constructor() {
super(...arguments);
this.state = this.props.initial || initial || {};
}
static getDerivedStateFromProps(props, state) {
if (!equal(props.initial, state)) return props.initial;
return null;
}
componentDidMount() {
listeners.add(fn => this.setState(fn));
}
componentWillUnmount() {
listeners.clear();
}
render() {
return React.createElement(ctx.Provider, {
value: this.state
}, this.props.children);
}
}, _a.displayName = 'DoczStateProvider', _a)
};
}
const doczState = create({});
const useConfig = () => {
const state = React.useContext(doczState.context);
const {
linkComponent,
transform,
config,
themeConfig = {}
} = state;
const newConfig = _merge(themeConfig, config ? config.themeConfig : {});
const transformed = transform ? transform(newConfig) : newConfig;
return Object.assign({}, config, {
linkComponent,
themeConfig: transformed
});
};
const updateState = ev => {
const {
type,
payload
} = JSON.parse(ev.data);
const prop = type.startsWith('state.') && type.split('.')[1];
if (prop) {
doczState.set(state => Object.assign({}, state, {
[prop]: payload
}));
}
};
const useDataServer = url => {
React.useEffect(() => {
if (!url) return;
const socket = new WebSocket(url);
socket.onmessage = updateState;
return () => socket.close();
}, []);
};
const useDocs = () => {
const {
entries = []
} = React.useContext(doczState.context);
const arr = entries.map(({
value
}) => value);
return sort(arr, (a, b) => compare(a.name, b.name));
};
const noMenu = entry => !entry.menu;
const fromMenu = menu => entry => entry.menu === menu;
const entryAsMenu = entry => ({
name: entry.name,
route: entry.route,
parent: entry.parent
});
const entriesOfMenu = (menu, entries) => entries.filter(fromMenu(menu)).map(entryAsMenu);
const parseMenu = entries => name => ({
name,
menu: entriesOfMenu(name, entries)
});
const menusFromEntries = entries => {
const entriesWithoutMenu = entries.filter(noMenu).map(entryAsMenu);
const menus = flatArrFromObject(entries, 'menu').map(parseMenu(entries));
return _unionBy('name', menus, entriesWithoutMenu);
};
const parseItemStr = item => typeof item === 'string' ? {
name: item
} : item;
const normalize = item => {
const selected = parseItemStr(item);
return Object.assign({}, selected, {
id: selected.id || ulid.ulid(),
parent: _get('parent', selected) || _get('parent', item),
menu: Array.isArray(selected.menu) ? selected.menu.map(normalize) : selected.menu
});
};
const clean = item => item.href || item.route ? _omit('menu', item) : item;
const normalizeAndClean = _pipe(normalize, clean);
const mergeMenus = (entriesMenu, configMenu) => {
const first = entriesMenu.map(normalizeAndClean);
const second = configMenu.map(normalizeAndClean);
const merged = _unionBy('name', first, second);
return merged.map(item => {
if (!item.menu) return item;
const found = second.find(i => i.name === item.name);
const foundMenu = found && found.menu;
return Object.assign({}, item, {
menu: foundMenu ? mergeMenus(item.menu, foundMenu) : item.menu || found.menu
});
});
};
const UNKNOWN_POS = Infinity;
const findPos = (item, orderedList = []) => {
const name = typeof item !== 'string' ? _get('name', item) : item;
const pos = orderedList.findIndex(item => item === name);
return pos !== -1 ? pos : UNKNOWN_POS;
};
const compareWithMenu = (to = []) => (a, b) => {
const list = to.map(i => i.name || i);
return compare(findPos(a, list), findPos(b, list));
};
const sortByName = (a, b) => {
return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
};
const sortMenus = (first, second = []) => {
const sorted = sort(first, compareWithMenu(second), sortByName);
return sorted.map(item => {
if (!item.menu) return item;
const found = second.find(menu => menu.name === item.name);
const foundMenu = found && found.menu;
return Object.assign({}, item, {
menu: foundMenu ? sortMenus(item.menu, foundMenu) : sort(item.menu, sortByName)
});
});
};
const search = (val, menu) => {
const items = menu.map(item => [item].concat(item.menu || []));
const flattened = _flattenDepth(2, items);
const flattenedDeduplicated = [...new Set(flattened)];
return match(flattenedDeduplicated, val, {
keys: ['name']
});
};
const filterMenus = (items, filter) => {
if (!filter) return items;
return items.filter(filter).map(item => {
if (!item.menu) return item;
return Object.assign({}, item, {
menu: item.menu.filter(filter)
});
});
};
const useMenus = opts => {
const {
query = ''
} = opts || {};
const {
entries,
config
} = React.useContext(doczState.context);
if (!entries) return null;
const arr = entries.map(({
value
}) => value);
const entriesMenu = menusFromEntries(arr);
const sorted = React.useMemo(() => {
const merged = mergeMenus(entriesMenu, config.menu);
const result = sortMenus(merged, config.menu);
return filterMenus(result, opts && opts.filter);
}, [entries, config]);
return query && query.length > 0 ? search(query, sorted) : sorted;
};
const usePrevious = (value, defaultValue) => {
const ref = React.useRef(defaultValue);
React.useEffect(() => {
ref.current = value;
});
return ref.current;
};
const isClient = typeof window === 'object';
const getSize = (initialWidth, initialHeight) => ({
innerHeight: isClient ? window.innerHeight : initialHeight,
innerWidth: isClient ? window.innerWidth : initialWidth,
outerHeight: isClient ? window.outerHeight : initialHeight,
outerWidth: isClient ? window.outerWidth : initialWidth
});
const useWindowSize = (throttleMs = 300, initialWidth = Infinity, initialHeight = Infinity) => {
const [windowSize, setWindowSize] = React.useState(getSize(initialHeight, initialHeight));
const tSetWindowResize = _throttle(throttleMs, () => setWindowSize(getSize(initialHeight, initialHeight)));
React.useEffect(() => {
window.addEventListener('resize', tSetWindowResize);
return () => void window.removeEventListener('resize', tSetWindowResize);
}, []);
return windowSize;
};
exports.ComponentsProvider = ComponentsProvider;
exports.doczState = doczState;
exports.isFn = isFn;
exports.useComponents = useComponents;
exports.useConfig = useConfig;
exports.useDataServer = useDataServer;
exports.useDocs = useDocs;
exports.useMenus = useMenus;
exports.usePrevious = usePrevious;
exports.useWindowSize = useWindowSize;
import { SFC, ComponentType } from 'react';
interface Props {
as: ComponentType<any>;
getInitialProps?: (props: any) => any;
}
export declare const AsyncComponent: SFC<Props>;
export {};
import * as React from 'react';
export declare const Playground: React.SFC<any>;
import { SFC } from 'react';
import { Entry } from '../state';
import { ComponentsMap } from '../hooks/useComponents';
export declare type Imports = Record<string, () => Promise<any>>;
export declare const loadRoute: (path: string, imports: Record<string, () => Promise<any>>, components: ComponentsMap) => import("@loadable/component").LoadableComponent<any>;
interface AsyncRouteProps {
asyncComponent: any;
components: ComponentsMap;
path: string;
entry: Entry;
}
export declare const AsyncRoute: SFC<AsyncRouteProps>;
export {};
import * as React from 'react';
import { LinkProps } from '@reach/router';
export { LinkProps };
export declare const Link: React.ForwardRefExoticComponent<Pick<LinkProps<any>, "key" | "download" | "hrefLang" | "media" | "rel" | "target" | "type" | "referrerPolicy" | "defaultChecked" | "defaultValue" | "suppressContentEditableWarning" | "suppressHydrationWarning" | "accessKey" | "className" | "contentEditable" | "contextMenu" | "dir" | "draggable" | "hidden" | "id" | "lang" | "placeholder" | "slot" | "spellCheck" | "style" | "tabIndex" | "title" | "inputMode" | "is" | "radioGroup" | "role" | "about" | "datatype" | "inlist" | "prefix" | "property" | "resource" | "typeof" | "vocab" | "autoCapitalize" | "autoCorrect" | "autoSave" | "color" | "itemProp" | "itemScope" | "itemType" | "itemID" | "itemRef" | "results" | "security" | "unselectable" | "aria-activedescendant" | "aria-atomic" | "aria-autocomplete" | "aria-busy" | "aria-checked" | "aria-colcount" | "aria-colindex" | "aria-colspan" | "aria-controls" | "aria-current" | "aria-describedby" | "aria-details" | "aria-disabled" | "aria-dropeffect" | "aria-errormessage" | "aria-expanded" | "aria-flowto" | "aria-grabbed" | "aria-haspopup" | "aria-hidden" | "aria-invalid" | "aria-keyshortcuts" | "aria-label" | "aria-labelledby" | "aria-level" | "aria-live" | "aria-modal" | "aria-multiline" | "aria-multiselectable" | "aria-orientation" | "aria-owns" | "aria-placeholder" | "aria-posinset" | "aria-pressed" | "aria-readonly" | "aria-relevant" | "aria-required" | "aria-roledescription" | "aria-rowcount" | "aria-rowindex" | "aria-rowspan" | "aria-selected" | "aria-setsize" | "aria-sort" | "aria-valuemax" | "aria-valuemin" | "aria-valuenow" | "aria-valuetext" | "children" | "dangerouslySetInnerHTML" | "onCopy" | "onCopyCapture" | "onCut" | "onCutCapture" | "onPaste" | "onPasteCapture" | "onCompositionEnd" | "onCompositionEndCapture" | "onCompositionStart" | "onCompositionStartCapture" | "onCompositionUpdate" | "onCompositionUpdateCapture" | "onFocus" | "onFocusCapture" | "onBlur" | "onBlurCapture" | "onChange" | "onChangeCapture" | "onBeforeInput" | "onBeforeInputCapture" | "onInput" | "onInputCapture" | "onReset" | "onResetCapture" | "onSubmit" | "onSubmitCapture" | "onInvalid" | "onInvalidCapture" | "onLoad" | "onLoadCapture" | "onError" | "onErrorCapture" | "onKeyDown" | "onKeyDownCapture" | "onKeyPress" | "onKeyPressCapture" | "onKeyUp" | "onKeyUpCapture" | "onAbort" | "onAbortCapture" | "onCanPlay" | "onCanPlayCapture" | "onCanPlayThrough" | "onCanPlayThroughCapture" | "onDurationChange" | "onDurationChangeCapture" | "onEmptied" | "onEmptiedCapture" | "onEncrypted" | "onEncryptedCapture" | "onEnded" | "onEndedCapture" | "onLoadedData" | "onLoadedDataCapture" | "onLoadedMetadata" | "onLoadedMetadataCapture" | "onLoadStart" | "onLoadStartCapture" | "onPause" | "onPauseCapture" | "onPlay" | "onPlayCapture" | "onPlaying" | "onPlayingCapture" | "onProgress" | "onProgressCapture" | "onRateChange" | "onRateChangeCapture" | "onSeeked" | "onSeekedCapture" | "onSeeking" | "onSeekingCapture" | "onStalled" | "onStalledCapture" | "onSuspend" | "onSuspendCapture" | "onTimeUpdate" | "onTimeUpdateCapture" | "onVolumeChange" | "onVolumeChangeCapture" | "onWaiting" | "onWaitingCapture" | "onAuxClick" | "onAuxClickCapture" | "onClick" | "onClickCapture" | "onContextMenu" | "onContextMenuCapture" | "onDoubleClick" | "onDoubleClickCapture" | "onDrag" | "onDragCapture" | "onDragEnd" | "onDragEndCapture" | "onDragEnter" | "onDragEnterCapture" | "onDragExit" | "onDragExitCapture" | "onDragLeave" | "onDragLeaveCapture" | "onDragOver" | "onDragOverCapture" | "onDragStart" | "onDragStartCapture" | "onDrop" | "onDropCapture" | "onMouseDown" | "onMouseDownCapture" | "onMouseEnter" | "onMouseLeave" | "onMouseMove" | "onMouseMoveCapture" | "onMouseOut" | "onMouseOutCapture" | "onMouseOver" | "onMouseOverCapture" | "onMouseUp" | "onMouseUpCapture" | "onSelect" | "onSelectCapture" | "onTouchCancel" | "onTouchCancelCapture" | "onTouchEnd" | "onTouchEndCapture" | "onTouchMove" | "onTouchMoveCapture" | "onTouchStart" | "onTouchStartCapture" | "onPointerDown" | "onPointerDownCapture" | "onPointerMove" | "onPointerMoveCapture" | "onPointerUp" | "onPointerUpCapture" | "onPointerCancel" | "onPointerCancelCapture" | "onPointerEnter" | "onPointerEnterCapture" | "onPointerLeave" | "onPointerLeaveCapture" | "onPointerOver" | "onPointerOverCapture" | "onPointerOut" | "onPointerOutCapture" | "onGotPointerCapture" | "onGotPointerCaptureCapture" | "onLostPointerCapture" | "onLostPointerCaptureCapture" | "onScroll" | "onScrollCapture" | "onWheel" | "onWheelCapture" | "onAnimationStart" | "onAnimationStartCapture" | "onAnimationEnd" | "onAnimationEndCapture" | "onAnimationIteration" | "onAnimationIterationCapture" | "onTransitionEnd" | "onTransitionEndCapture" | "to" | "replace" | "getProps" | "state"> & React.RefAttributes<any>>;
import { ComponentType, SFC } from 'react';
export interface PlaygroundProps {
className?: string;
style?: any;
wrapper?: ComponentType<any>;
children: any;
__scope: Record<string, any>;
__position: number;
__code: string;
__codesandbox: string;
}
declare const Playground: SFC<PlaygroundProps>;
export default Playground;
import { SFC, ComponentType } from 'react';
export interface EnumValue {
value: string;
computed: boolean;
}
export interface FlowTypeElement {
name: string;
value: string;
}
export interface FlowTypeArgs {
name: string;
type: {
name: string;
};
}
export interface PropType {
name: string;
value?: any;
raw?: any;
computed?: boolean;
}
export interface FlowType extends PropType {
elements: FlowTypeElement[];
name: string;
raw: string;
type?: string;
computed?: boolean;
signature?: {
arguments: FlowTypeArgs[];
return: {
name: string;
};
};
}
export interface Prop {
required: boolean;
description?: string;
type: PropType;
defaultValue?: {
value: string;
computed: boolean;
};
flowType?: FlowType;
}
export declare type ComponentWithDocGenInfo = ComponentType & {
__docgenInfo: {
description?: string;
props?: Record<string, Prop>;
};
};
export interface PropsProps {
title?: Node;
isRaw?: boolean;
isToggle?: boolean;
of: ComponentWithDocGenInfo;
}
export declare const getPropType: (prop: Prop) => any;
export interface PropsComponentProps {
title?: Node;
isRaw?: boolean;
isToggle?: boolean;
props: Record<string, Prop>;
getPropType(prop: Prop): string;
}
export declare const Props: SFC<PropsProps>;
import { SFC } from 'react';
import { ComponentsMap } from '../hooks/useComponents';
import { Imports } from './AsyncRoute';
export interface RoutesProps {
components: ComponentsMap;
imports: Imports;
}
export declare const Routes: SFC<RoutesProps>;
export * from './useComponents';
export { useConfig, UseConfigObj } from './useConfig';
export { useDataServer } from './useDataServer';
export { useDocs } from './useDocs';
export { useMenus } from './useMenus';
export { usePrevious } from './usePrevious';
export { useWindowSize } from './useWindowSize';
import { SFC, ComponentType as CT } from 'react';
import { RouteComponentProps } from '@reach/router';
import { Entry } from '../state';
export declare type PageProps = RouteComponentProps<any> & {
doc: Entry;
};
export interface PlaygroundProps {
className?: string;
style?: any;
wrapper?: CT<any>;
components: ComponentsMap;
component: JSX.Element;
position: number;
code: string;
codesandbox: string;
scope: Record<string, any>;
}
export declare type PlaygroundComponent = CT<PlaygroundProps>;
export interface ComponentsMap {
loading?: CT;
page?: CT<PageProps>;
notFound?: CT<RouteComponentProps<any>>;
playground?: PlaygroundComponent;
h1?: CT<any> | string;
h2?: CT<any> | string;
h3?: CT<any> | string;
h4?: CT<any> | string;
h5?: CT<any> | string;
h6?: CT<any> | string;
span?: CT<any> | string;
a?: CT<any> | string;
ul?: CT<any> | string;
table?: CT<any> | string;
pre?: CT<any> | string;
code?: CT<any> | string;
inlineCode?: CT<any> | string;
[key: string]: any;
}
export declare type NotFoundComponent = CT<RouteComponentProps<any>>;
export interface ComponentsProviderProps {
components: ComponentsMap;
}
export declare const ComponentsProvider: SFC<ComponentsProviderProps>;
export declare const useComponents: () => ComponentsMap;
/// <reference types="react" />
import { ThemeConfig, Config } from '../state';
export interface UseConfigObj extends Config {
themeConfig: ThemeConfig;
linkComponent?: React.ComponentType<any>;
}
export declare const useConfig: () => UseConfigObj;
export declare const useDataServer: (url: string | undefined) => void;
import { Entry } from '../state';
export declare const useDocs: () => Entry[] | null;
import { MenuItem } from '../state';
declare type FilterFn = (item: MenuItem) => boolean;
export interface UseMenusParams {
query?: string;
filter?: FilterFn;
}
export declare const useMenus: (opts?: UseMenusParams | undefined) => MenuItem[] | null;
export {};
export declare const usePrevious: (value: any, defaultValue?: any) => any;
export declare const useWindowSize: (throttleMs?: number, initialWidth?: number, initialHeight?: number) => {
innerHeight: number;
innerWidth: number;
outerHeight: number;
outerWidth: number;
};
export { Playground } from './components/AsyncPlayground';
export { AsyncRoute, loadRoute } from './components/AsyncRoute';
export { Link, LinkProps } from './components/Link';
export { Props, PropsComponentProps } from './components/Props';
export { Routes } from './components/Routes';
export * from './hooks';
export { theme } from './theme';
export { doczState, Entry, MenuItem, ThemeConfig } from './state';
import * as React from 'react';
import { createElement, Suspense, useState, useEffect, forwardRef, useCallback, useContext, useMemo, memo, Fragment } from 'react';
import loadable from '@loadable/component';
import _get from 'lodash/fp/get';
import { __rest } from 'tslib';
import { a as isFn, b as useComponents, c as doczState } from './chunk.esm.js';
export { j as ComponentsProvider, c as doczState, b as useComponents, d as useConfig, e as useDataServer, f as useDocs, g as useMenus, h as usePrevious, i as useWindowSize } from './chunk.esm.js';
import _omit from 'lodash/fp/omit';
import { Link as Link$1, createHistory, LocationProvider, Router } from '@reach/router';
import _first from 'lodash/fp/first';
import capitalize from 'capitalize';
import 'fast-deep-equal';
import 'lodash/fp/merge';
import 'array-sort';
import 'lodash/fp/unionBy';
import 'lodash/fp/flattenDepth';
import 'lodash/fp/pipe';
import 'ulid';
import 'match-sorter';
import 'lodash/fp/throttle';
import { MDXProvider } from '@mdx-js/react';
import createHashSource from 'hash-source';
let source = createHashSource();
const BasePlayground = loadable(() => import('./Playground.esm.js'));
const Playground = props => typeof window !== 'undefined' ? createElement(Suspense, {
fallback: null
}, createElement(BasePlayground, Object.assign({}, props))) : null;
const AsyncComponent = defaultProps => {
const [loading, setLoading] = useState(true);
const [error, setError] = useState(null);
const [data, setData] = useState({});
useEffect(() => {
const {
getInitialProps
} = defaultProps;
if (getInitialProps && isFn(getInitialProps)) {
setLoading(true);
getInitialProps(defaultProps).then(data => {
setLoading(false);
setError(null);
setData(data);
}).catch(err => {
setLoading(false);
setError(err);
setData({});
});
}
}, []);
const {
as: Comp,
getInitialProps
} = defaultProps,
props = __rest(defaultProps, ["as", "getInitialProps"]);
return createElement(Comp, Object.assign({}, props, {
data: Object.assign({}, data, {
loading,
error
})
}));
};
const loadRoute = (path, imports, components) => {
const Loading = components.loading;
const fn = async () => {
const importFn = _get(path, imports);
const {
default: Component,
getInitialProps
} = await importFn();
const ExportedComponent = props => createElement(AsyncComponent, Object.assign({}, props, {
as: Component || 'div',
getInitialProps: getInitialProps
}));
return ExportedComponent;
};
return loadable(fn, {
fallback: createElement(Loading, null)
});
};
const AsyncRoute = defaultProps => {
const {
asyncComponent,
path,
entry
} = defaultProps,
routeProps = __rest(defaultProps, ["asyncComponent", "path", "entry"]);
const components = useComponents();
const Page = components.page;
const Component = asyncComponent;
const props = Object.assign({}, routeProps, {
doc: entry
});
return Page ? createElement(Page, Object.assign({}, props), createElement(Component, Object.assign({}, props))) : createElement(Component, Object.assign({}, props));
};
const Link = forwardRef((defaultProps, ref) => {
const props = _omit(['activeClassName', 'partiallyActive'], defaultProps);
const isActive = useCallback(({
isCurrent
}) => {
return isCurrent ? {
className: `${props.className} active`
} : {};
}, [props.className]);
return createElement(Link$1, Object.assign({}, props, {
getProps: isActive,
ref: ref
}));
});
Link.displayName = 'Link';
const RE_OBJECTOF = /(?:React\.)?(?:PropTypes\.)?objectOf\((?:React\.)?(?:PropTypes\.)?(\w+)\)/;
const getTypeStr = type => {
switch (type.name.toLowerCase()) {
case 'instanceof':
return `Class(${type.value})`;
case 'enum':
if (type.computed) return type.value;
return type.value ? type.value.map(v => `${v.value}`).join(' │ ') : type.raw;
case 'union':
return type.value ? type.value.map(t => `${getTypeStr(t)}`).join(' │ ') : type.raw;
case 'array':
return type.raw;
case 'arrayof':
return `Array<${getTypeStr(type.value)}>`;
case 'custom':
if (type.raw.indexOf('function') !== -1 || type.raw.indexOf('=>') !== -1) return 'Custom(Function)';else if (type.raw.toLowerCase().indexOf('objectof') !== -1) {
const m = type.raw.match(RE_OBJECTOF);
if (m && m[1]) return `ObjectOf(${capitalize(m[1])})`;
return 'ObjectOf';
}
return 'Custom';
case 'bool':
return 'Boolean';
case 'func':
return 'Function';
case 'shape':
const shape = type.value;
const rst = {};
Object.keys(shape).forEach(key => {
rst[key] = getTypeStr(shape[key]);
});
return JSON.stringify(rst, null, 2);
default:
return capitalize(type.name);
}
};
const humanize = type => getTypeStr(type);
const getPropType = prop => {
const propName = _get('name', prop.flowType || prop.type);
if (!propName) return null;
const isEnum = propName.startsWith('"') || propName === 'enum';
const name = capitalize(isEnum ? 'enum' : propName);
const value = _get('type.value', prop);
if (!name) return null;
if (isEnum && typeof value === 'string' || !prop.flowType && !isEnum && !value || prop.flowType && !prop.flowType.elements) {
return name;
}
return prop.flowType ? humanize(prop.flowType) : humanize(prop.type);
};
const Props = ({
title,
isToggle,
isRaw,
of: component
}) => {
const components = useComponents();
const {
props: stateProps
} = useContext(doczState.context);
const PropsComponent = components.props;
const filename = _get('__filemeta.filename', component);
const filemetaName = _get('__filemeta.name', component);
const componentName = filemetaName || component.displayName || component.name;
const found = stateProps && stateProps.length > 0 && stateProps.find(item => filename ? item.key === filename : item.key.includes(`${componentName}.`));
const value = _get('value', found) || [];
const firstDefinition = _first(value);
const definition = value.find(i => i.displayName === componentName);
const props = _get('props', definition || firstDefinition);
if (!props) return null;
if (!PropsComponent) return null;
return createElement(PropsComponent, {
title: title,
isRaw: isRaw,
isToggle: isToggle,
props: props,
getPropType: getPropType
});
};
const goToHash = ({
location
}) => {
setTimeout(() => {
if (location && location.hash) {
const decodedHash = decodeURI(location.hash);
const id = decodedHash.substring(1);
const el = document.getElementById(id);
if (el) el.scrollIntoView();
}
});
};
const Routes = ({
imports
}) => {
const components = useComponents();
const {
entries
} = useContext(doczState.context);
const NotFound = components.notFound;
const history = useMemo(() => createHistory(source), []);
useEffect(() => {
history.listen(goToHash);
}, []);
return createElement(MDXProvider, {
components: components
}, createElement(LocationProvider, {
history: history
}, createElement(Router, null, createElement(NotFound, {
default: true
}), entries && entries.map(({
key: path,
value: entry
}) => {
const props = {
path,
entries,
components
};
const component = loadRoute(path, imports, components);
return createElement(AsyncRoute, Object.assign({}, props, {
entry: entry,
key: entry.id,
path: entry.route,
asyncComponent: component
}));
}))));
};
function theme(themeConfig, transform = c => c) {
return WrappedComponent => {
const Theme = memo(props => {
const {
linkComponent
} = props;
const {
db,
children,
wrapper: Wrapper = Fragment
} = props;
const initial = Object.assign({}, db, {
themeConfig,
transform,
linkComponent
});
return createElement(doczState.Provider, {
initial: initial
}, createElement(Wrapper, null, createElement(WrappedComponent, null, children)));
});
Theme.displayName = WrappedComponent.displayName || 'DoczTheme';
return Theme;
};
}
export { AsyncRoute, Link, Playground, Props, Routes, loadRoute, theme };
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }
var React = require('react');
var loadable = _interopDefault(require('@loadable/component'));
var _get = _interopDefault(require('lodash/fp/get'));
var tslib_1 = require('tslib');
var __chunk_1 = require('./chunk.js');
var _omit = _interopDefault(require('lodash/fp/omit'));
var router = require('@reach/router');
var _first = _interopDefault(require('lodash/fp/first'));
var capitalize = _interopDefault(require('capitalize'));
require('fast-deep-equal');
require('lodash/fp/merge');
require('array-sort');
require('lodash/fp/unionBy');
require('lodash/fp/flattenDepth');
require('lodash/fp/pipe');
require('ulid');
require('match-sorter');
require('lodash/fp/throttle');
var react = require('@mdx-js/react');
var createHashSource = require('hash-source');
let source = createHashSource.default();;
const BasePlayground = loadable(() => Promise.resolve(require('./Playground.js')));
const Playground = props => typeof window !== 'undefined' ? React.createElement(React.Suspense, {
fallback: null
}, React.createElement(BasePlayground, Object.assign({}, props))) : null;
const AsyncComponent = defaultProps => {
const [loading, setLoading] = React.useState(true);
const [error, setError] = React.useState(null);
const [data, setData] = React.useState({});
React.useEffect(() => {
const {
getInitialProps
} = defaultProps;
if (getInitialProps && __chunk_1.isFn(getInitialProps)) {
setLoading(true);
getInitialProps(defaultProps).then(data => {
setLoading(false);
setError(null);
setData(data);
}).catch(err => {
setLoading(false);
setError(err);
setData({});
});
}
}, []);
const {
as: Comp,
getInitialProps
} = defaultProps,
props = tslib_1.__rest(defaultProps, ["as", "getInitialProps"]);
return React.createElement(Comp, Object.assign({}, props, {
data: Object.assign({}, data, {
loading,
error
})
}));
};
const loadRoute = (path, imports, components) => {
const Loading = components.loading;
const fn = async () => {
const importFn = _get(path, imports);
const {
default: Component,
getInitialProps
} = await importFn();
const ExportedComponent = props => React.createElement(AsyncComponent, Object.assign({}, props, {
as: Component || 'div',
getInitialProps: getInitialProps
}));
return ExportedComponent;
};
return loadable(fn, {
fallback: React.createElement(Loading, null)
});
};
const AsyncRoute = defaultProps => {
const {
asyncComponent,
path,
entry
} = defaultProps,
routeProps = tslib_1.__rest(defaultProps, ["asyncComponent", "path", "entry"]);
const components = __chunk_1.useComponents();
const Page = components.page;
const Component = asyncComponent;
const props = Object.assign({}, routeProps, {
doc: entry
});
return Page ? React.createElement(Page, Object.assign({}, props), React.createElement(Component, Object.assign({}, props))) : React.createElement(Component, Object.assign({}, props));
};
const Link = React.forwardRef((defaultProps, ref) => {
const props = _omit(['activeClassName', 'partiallyActive'], defaultProps);
const isActive = React.useCallback(({
isCurrent
}) => {
return isCurrent ? {
className: `${props.className} active`
} : {};
}, [props.className]);
return React.createElement(router.Link, Object.assign({}, props, {
getProps: isActive,
ref: ref
}));
});
Link.displayName = 'Link';
const RE_OBJECTOF = /(?:React\.)?(?:PropTypes\.)?objectOf\((?:React\.)?(?:PropTypes\.)?(\w+)\)/;
const getTypeStr = type => {
switch (type.name.toLowerCase()) {
case 'instanceof':
return `Class(${type.value})`;
case 'enum':
if (type.computed) return type.value;
return type.value ? type.value.map(v => `${v.value}`).join(' │ ') : type.raw;
case 'union':
return type.value ? type.value.map(t => `${getTypeStr(t)}`).join(' │ ') : type.raw;
case 'array':
return type.raw;
case 'arrayof':
return `Array<${getTypeStr(type.value)}>`;
case 'custom':
if (type.raw.indexOf('function') !== -1 || type.raw.indexOf('=>') !== -1) return 'Custom(Function)';else if (type.raw.toLowerCase().indexOf('objectof') !== -1) {
const m = type.raw.match(RE_OBJECTOF);
if (m && m[1]) return `ObjectOf(${capitalize(m[1])})`;
return 'ObjectOf';
}
return 'Custom';
case 'bool':
return 'Boolean';
case 'func':
return 'Function';
case 'shape':
const shape = type.value;
const rst = {};
Object.keys(shape).forEach(key => {
rst[key] = getTypeStr(shape[key]);
});
return JSON.stringify(rst, null, 2);
default:
return capitalize(type.name);
}
};
const humanize = type => getTypeStr(type);
const getPropType = prop => {
const propName = _get('name', prop.flowType || prop.type);
if (!propName) return null;
const isEnum = propName.startsWith('"') || propName === 'enum';
const name = capitalize(isEnum ? 'enum' : propName);
const value = _get('type.value', prop);
if (!name) return null;
if (isEnum && typeof value === 'string' || !prop.flowType && !isEnum && !value || prop.flowType && !prop.flowType.elements) {
return name;
}
return prop.flowType ? humanize(prop.flowType) : humanize(prop.type);
};
const Props = ({
title,
isToggle,
isRaw,
of: component
}) => {
const components = __chunk_1.useComponents();
const {
props: stateProps
} = React.useContext(__chunk_1.doczState.context);
const PropsComponent = components.props;
const filename = _get('__filemeta.filename', component);
const filemetaName = _get('__filemeta.name', component);
const componentName = filemetaName || component.displayName || component.name;
const found = stateProps && stateProps.length > 0 && stateProps.find(item => filename ? item.key === filename : item.key.includes(`${componentName}.`));
const value = _get('value', found) || [];
const firstDefinition = _first(value);
const definition = value.find(i => i.displayName === componentName);
const props = _get('props', definition || firstDefinition);
if (!props) return null;
if (!PropsComponent) return null;
return React.createElement(PropsComponent, {
title: title,
isRaw: isRaw,
isToggle: isToggle,
props: props,
getPropType: getPropType
});
};
const goToHash = ({
location
}) => {
setTimeout(() => {
if (location && location.hash) {
const decodedHash = decodeURI(location.hash);
const id = decodedHash.substring(1);
const el = document.getElementById(id);
if (el) el.scrollIntoView();
}
});
};
const Routes = ({
imports
}) => {
const components = __chunk_1.useComponents();
const {
entries
} = React.useContext(__chunk_1.doczState.context);
const NotFound = components.notFound;
const history = React.useMemo(() => router.createHistory(source), []);
React.useEffect(() => {
history.listen(goToHash);
}, []);
return React.createElement(react.MDXProvider, {
components: components
}, React.createElement(router.LocationProvider, {
history: history
}, React.createElement(router.Router, null, React.createElement(NotFound, {
default: true
}), entries && entries.map(({
key: path,
value: entry
}) => {
const props = {
path,
entries,
components
};
const component = loadRoute(path, imports, components);
return React.createElement(AsyncRoute, Object.assign({}, props, {
entry: entry,
key: entry.id,
path: entry.route,
asyncComponent: component
}));
}))));
};
function theme(themeConfig, transform = c => c) {
return WrappedComponent => {
const Theme = React.memo(props => {
const {
linkComponent
} = props;
const {
db,
children,
wrapper: Wrapper = React.Fragment
} = props;
const initial = Object.assign({}, db, {
themeConfig,
transform,
linkComponent
});
return React.createElement(__chunk_1.doczState.Provider, {
initial: initial
}, React.createElement(Wrapper, null, React.createElement(WrappedComponent, null, children)));
});
Theme.displayName = WrappedComponent.displayName || 'DoczTheme';
return Theme;
};
}
exports.ComponentsProvider = __chunk_1.ComponentsProvider;
exports.doczState = __chunk_1.doczState;
exports.useComponents = __chunk_1.useComponents;
exports.useConfig = __chunk_1.useConfig;
exports.useDataServer = __chunk_1.useDataServer;
exports.useDocs = __chunk_1.useDocs;
exports.useMenus = __chunk_1.useMenus;
exports.usePrevious = __chunk_1.usePrevious;
exports.useWindowSize = __chunk_1.useWindowSize;
exports.AsyncRoute = AsyncRoute;
exports.Link = Link;
exports.Playground = Playground;
exports.Props = Props;
exports.Routes = Routes;
exports.loadRoute = loadRoute;
exports.theme = theme;
import { ComponentType } from 'react';
export interface Heading {
depth: number;
slug: string;
value: string;
}
export interface Entry {
id: string;
filepath: string;
slug: string;
route: string;
name: string;
order: number;
menu: string | null;
headings: Heading[];
[key: string]: any;
}
export interface MenuItem {
id: string;
name: string;
route?: string;
href?: string;
menu?: MenuItem[];
order?: number;
parent?: string;
}
export declare type ThemeConfig = Record<string, any>;
export interface Config {
title: string;
description: string;
themeConfig: ThemeConfig;
menu: MenuItem[];
version: string | null;
repository: string | null;
native: boolean;
separator: string;
codeSandbox: boolean;
base?: string;
}
export declare type Entries = {
key: string;
value: Entry;
}[];
export declare type Props = {
key: string;
value: any;
}[];
export declare type TransformFn = (config: ThemeConfig) => ThemeConfig;
export interface Database {
config: Config;
props?: Props;
entries?: Entries;
}
export interface DoczState extends Database {
linkComponent: ComponentType<any>;
themeConfig?: ThemeConfig;
transform?: TransformFn;
}
export declare const doczState: import("./utils/createState").State<DoczState>;
import { ComponentType as CT } from 'react';
import { Database, ThemeConfig, TransformFn } from './state';
export interface ThemeProps {
db: Database;
wrapper?: CT;
linkComponent?: CT;
children(WrappedComponent: CT): JSX.Element;
}
export declare type ThemeReturn = (WrappedComponent: CT) => CT<ThemeProps>;
export declare function theme(themeConfig: ThemeConfig, transform?: TransformFn): ThemeReturn;
import * as React from 'react';
export interface ProviderProps<T> {
initial?: T;
}
export declare type PrevState<T> = (prevState: T) => T;
export declare type GetFn<T> = (state: T) => React.ReactNode;
export declare type Dispatch<T> = T | PrevState<T>;
export interface State<T> {
context: React.Context<T>;
set: (param: Dispatch<T>) => void;
Provider: React.ComponentType<ProviderProps<T>>;
}
export declare function create<T = any>(initial: T): State<T>;
export declare const isFn: (value: any) => boolean;
export declare function flatArrFromObject<T>(arr: T[], prop: string): string[];
export declare function compare<T>(a: T, b: T, reverse?: boolean): number;
import { PropType, FlowType } from '../components/Props';
export declare const humanize: (type: PropType | FlowType) => any;
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import menu from "./docz-lib/config/menu";
import versions from './docz-lib/config/versions';
module.exports = {
title: "Apache Atlas",
files: "**/*.{md,mdx}",
base: "/",
baseUrl:"./public",
src: "./src",
public: "./src/resources",
menu: menu,
atlasVersions: versions,
theme: "theme/",
modifyBundlerConfig: config => {
config.module.rules.push(
{
test: /\.(js)$/,
exclude: /node_modules/,
use: {
loader: "babel-loader",
query: {
presets: ["@babel/react"],
plugins: [
"@babel/plugin-proposal-class-properties",
"@babel/plugin-syntax-dynamic-import"
]
}
}
}
);
return config;
}
};
This source diff could not be displayed because it is too large. You can view the blob instead.
{
"name": "doc",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "node ./docz-lib/docz/bin/index.js dev",
"build": "node ./docz-lib/docz/bin/index.js build",
"predeploy": "npm install && npm run build",
"deploy": "gh-pages -d .docz/dist"
},
"author": "",
"license": "MIT",
"peerDependencies": {},
"devDependencies": {
"@babel/cli": "^7.4.4",
"@babel/core": "^7.4.5",
"@babel/plugin-syntax-dynamic-import": "^7.2.0",
"@babel/preset-react": "^7.0.0",
"axios": "^0.19.0",
"babel-loader": "^8.0.6",
"babel-plugin-lodash": "^3.3.4",
"babel-plugin-react-transform": "^3.0.0",
"gh-pages": "^2.0.1",
"react-dropdown-select": "^3.0.0",
"react-github-btn": "^1.0.5",
"react-syntax-highlighter": "^11.0.1",
"reactstrap": "^8.0.0",
"xml2js": "^0.4.19"
},
"dependencies": {
"docz": "^1.2.0",
"facepaint": "^1.2.1",
"hash-source": "^1.0.4",
"react-feather": "^1.1.6",
"styled-components": "^4.3.1",
"yargs": "^14.0.0"
}
}
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- <!--
~ Licensed to the Apache Software Foundation (ASF) under one ~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file ~ or more contributor license agreements. See the NOTICE file
...@@ -27,13 +26,12 @@ ...@@ -27,13 +26,12 @@
<artifactId>atlas-docs</artifactId> <artifactId>atlas-docs</artifactId>
<description>Apache Atlas Documentation</description> <description>Apache Atlas Documentation</description>
<name>Apache Atlas Documentation</name> <name>Apache Atlas Documentation</name>
<properties> <properties>
<skipTests>true</skipTests> <skipTests>true</skipTests>
<skipSite>false</skipSite> <skipSite>false</skipSite>
</properties> </properties>
<!-- TODO -->
<reporting> <!-- <reporting>
<excludeDefaults>true</excludeDefaults> <excludeDefaults>true</excludeDefaults>
<outputDirectory>${project.build.directory}/site</outputDirectory> <outputDirectory>${project.build.directory}/site</outputDirectory>
<plugins> <plugins>
...@@ -63,47 +61,81 @@ ...@@ -63,47 +61,81 @@
</reportSets> </reportSets>
</plugin> </plugin>
</plugins> </plugins>
</reporting> </reporting> -->
<build> <build>
<pluginManagement>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<configuration>
<phase>validate</phase>
<workingDirectory>${project.build.directory}</workingDirectory>
</configuration>
</plugin>
</plugins>
</pluginManagement>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId>
<artifactId>maven-site-plugin</artifactId> <version>2.7</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-module-twiki</artifactId>
<version>${doxia.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-module-markdown</artifactId>
<version>${doxia.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-module-markdown</artifactId>
<version>${doxia.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.doxia</groupId>
<artifactId>doxia-core</artifactId>
<version>${doxia.version}</version>
</dependency>
</dependencies>
<configuration>
<port>8888</port>
</configuration>
<executions> <executions>
<execution> <execution>
<id>copy-resources</id>
<goals>
<goal>copy-resources</goal>
</goals>
<phase>validate</phase>
<configuration>
<outputDirectory>${basedir}/target</outputDirectory>
<resources>
<resource>
<directory>${basedir}</directory>
<excludes>
<exclude>.docz/**</exclude>
<exclude>node_modules/**</exclude>
<exclude>target/**</exclude>
</excludes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node-for-v2.version}</nodeVersion>
<npmVersion>${npm-for-v2.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm build</id>
<goals> <goals>
<goal>site</goal> <goal>npm</goal>
</goals> </goals>
<phase>prepare-package</phase> <configuration>
<arguments>run build</arguments>
</configuration>
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</project> </project>
\ No newline at end of file
---
name: ASF
route: /asf
menu: ASF
---
import {CustomLink} from "theme/components/shared/common/CustomLink";
# ASF Infomation
1. <CustomLink href="http://www.apache.org/foundation/how-it-works.html">How Apache Works</CustomLink>
2. <CustomLink href="https://apachecon.com/?ref=atlas.apache.org">Events </CustomLink>
3. <CustomLink href="https://www.apache.org/licenses/">License </CustomLink>
4. <CustomLink href="http://www.apache.org/foundation/">Foundation </CustomLink>
5. <CustomLink href="http://www.apache.org/foundation/sponsorship.html">Sponsoring Apache </CustomLink>
6. <CustomLink href="http://www.apache.org/foundation/thanks.html">Thanks</CustomLink>
---
name: Classification Propagation
route: /ClassificationPropagation
menu: Documentation
submenu: Features
---
import Img from 'theme/components/shared/Img'
# Classification Propagation
* Classification propagation enables classifications associated to an entity to be automatically associated with other related entities of the entity. This is very useful in dealing with scenarios where a dataset derives it data from other datasets - like a table loaded with data in a file, a report generated from a table/view, etc.
* For example, when a table is classified as *PII*, tables or views that derive data from this table (via CTAS or ‘create view’ operation) will be automatically classified as *PII*.
## Use Cases
Consider the following lineage where data from a 'hdfs_path' entity is loaded into a table, which is further made available through views. We will go through various scenarios to understand the classification propagation feature.
<Img src={`/images/twiki/classification-propagation-1.png`}/>
## Add classification to an entity
When classification ‘PII’ is added to 'hdfs_path' entity, the classification is propagated to all impacted entities in the lineage path, including 'employees' table, views 'us_employees' and 'uk_employees' - as shown below.
<Img src={`/images/twiki/classification-propagation-2.png`}/>
## Update classification associated with an entity
Any updates to classifications associated with an entity will be seen in all entities the classification is propagated to as well.
<Img src={`/images/twiki/classification-propagation-3.png`}/>
## Remove classification associated with an entity
When a classification is deleted from an entity, the classification will be removed from all entities the classification is propagated to as well.
<Img src={`/images/twiki/classification-propagation-4.png`}/>
## Add lineage between entities
When lineage is added between entities, for example to capture loading of data in a file to a table, the classifications associated with the source entity are propagated to all impacted entities as well.
For example, when a view is created from a table, classifications associated with the table are propagated to the newly created view as well.
<Img src={`/images/twiki/classification-propagation-5.png`}/>
## Delete an entity
**Case 1:**
When an entity is deleted, classifications associated with this entity will be removed from all entities the classifications are propagated to.
For example. when _employees_ table is deleted, classifications associated with this table are removed from 'employees_view' view.
<Img src={`/images/twiki/classification-propagation-6.png`}/>
**Case 2:**
When an entity is deleted in the middle of a lineage path, the propagation link is broken and previously propagated classifications will be removed from all derived entities of the deleted entity.
For example. when 'us_employees' table is deleted, classifications propagating through this table (**PII**) are removed from 'ca_employees' table, since the only path of propagation is broken by entity deletion.
<Img src={`/images/twiki/classification-propagation-entity-delete-1.png"`}/>
<Img src={`/images/twiki/classification-propagation-entity-delete-2.png"`}/>
**Case 3:**
When an entity is deleted in the middle of a lineage path and if there exists alternate path for propagation, previously propagated classifications will be retained.
For example. when 'us_employees' table is deleted, classifications propagating (**PII**) through this table are retained in 'ca_employees' table, since there are two propagation paths available and only one of them is broken by entity deletion.
<Img src={`/images/twiki/classification-propagation-entity-delete-3.png"`}/>
<Img src={`/images/twiki/classification-propagation-entity-delete-4.png"`}/>
## Control Propagation
Apache Atlas provides few options to control whether/where a classification is propagated.
This section details available options.
## Propagate flag in classification
Each association of classification to an entity has a boolean flag that controls whether the classification is propagated or not.
When a classification is associated with an entity, this flag is set to ‘true’ i.e. the classification will be propagated to all impacted entities. This flag can be updated to desired value during initial association or later.
<Img src={`/images/twiki/classification-propagation-7.png`}/>
## Propagate flag in lineage edge
Apache Atlas supports a flag at lineage edge to enable/disable propagation of classifications through the edge. By default, the propagation is enabled for lineage edges.
When the flag is turned off, no classification will be propagated through this edge; and propagation of currently propagated classifications through the edge will be reevaluated, so that they can be removed from impacted entities.
When the flag is turned on, propagation of classifications of the source entity will be reevaluated, so that they can be propagated to all impacted entities.
## Block propagation of specific classifications in lineage edge
Apache Atlas supports blocking propagation of specific classifications in at lineage edges.
This can be useful, for example, to handle scenarios like: a column classified as PII is masked when creating a view; in such scenario, if corresponding column in the created view might not have PII, hence the propagation of PII classification should be blocked.
This can be done by updating the lineage edge to add the PII classification in _‘blocked propagated classifications’_ list.
Classifications in blocked propagated classifications will not be propagated in the derivative/downstream entities.
<Img src={`/images/twiki/classification-propagation-8.png`}/>
## Notifications and Audit
When propagated classifications are added/update/deleted, Apache Atlas sends notifications to 'ATLAS_ENTITIES' topic for each entity affected by the propagation.
## Glossary
When a classification is associated with a glossary term, the classification is automatically propagated to all entities associated with the term.
---+ Architecture ---
name: High Level Architecture
route: /Architecture
menu: Documentation
submenu: Features
---
import Img from 'theme/components/shared/Img'
---++ Introduction # Architecture
---++ Atlas High Level Architecture - Overview ## Introduction
<img src="images/twiki/architecture.png" height="400" width="600" />
## Atlas High Level Architecture - Overview
<Img src={`/images/twiki/architecture.png`} width="800" />
The components of Atlas can be grouped under the following major categories: The components of Atlas can be grouped under the following major categories:
---+++ Core ### Core
Atlas core includes the following components: Atlas core includes the following components:
*Type System*: Atlas allows users to define a model for the metadata objects they want to manage. The model is composed **Type System**: Atlas allows users to define a model for the metadata objects they want to manage. The model is composed
of definitions called ‘types’. Instances of ‘types’ called ‘entities’ represent the actual metadata objects that are of definitions called ‘types’. Instances of ‘types’ called ‘entities’ represent the actual metadata objects that are
managed. The Type System is a component that allows users to define and manage the types and entities. All metadata managed. The Type System is a component that allows users to define and manage the types and entities. All metadata
objects managed by Atlas out of the box (like Hive tables, for e.g.) are modelled using types and represented as objects managed by Atlas out of the box (like Hive tables, for e.g.) are modelled using types and represented as
...@@ -20,60 +28,53 @@ One key point to note is that the generic nature of the modelling in Atlas allow ...@@ -20,60 +28,53 @@ One key point to note is that the generic nature of the modelling in Atlas allow
define both technical metadata and business metadata. It is also possible to define rich relationships between the define both technical metadata and business metadata. It is also possible to define rich relationships between the
two using features of Atlas. two using features of Atlas.
*Graph Engine*: Internally, Atlas persists metadata objects it manages using a Graph model. This approach provides great **Graph Engine**: Internally, Atlas persists metadata objects it manages using a Graph model. This approach provides great
flexibility and enables efficient handling of rich relationships between the metadata objects. Graph engine component is flexibility and enables efficient handling of rich relationships between the metadata objects. Graph engine component is
responsible for translating between types and entities of the Atlas type system, and the underlying graph persistence model. responsible for translating between types and entities of the Atlas type system, and the underlying graph persistence model.
In addition to managing the graph objects, the graph engine also creates the appropriate indices for the metadata In addition to managing the graph objects, the graph engine also creates the appropriate indices for the metadata
objects so that they can be searched efficiently. Atlas uses the JanusGraph to store the metadata objects. objects so that they can be searched efficiently. Atlas uses the JanusGraph to store the metadata objects.
*Ingest / Export*: The Ingest component allows metadata to be added to Atlas. Similarly, the Export component exposes **Ingest / Export**: The Ingest component allows metadata to be added to Atlas. Similarly, the Export component exposes
metadata changes detected by Atlas to be raised as events. Consumers can consume these change events to react to metadata changes detected by Atlas to be raised as events. Consumers can consume these change events to react to
metadata changes in real time. metadata changes in real time.
---+++ Integration ### Integration
Users can manage metadata in Atlas using two methods: Users can manage metadata in Atlas using two methods:
*API*: All functionality of Atlas is exposed to end users via a REST API that allows types and entities to be created, **API**: All functionality of Atlas is exposed to end users via a REST API that allows types and entities to be created,
updated and deleted. It is also the primary mechanism to query and discover the types and entities managed by Atlas. updated and deleted. It is also the primary mechanism to query and discover the types and entities managed by Atlas.
*Messaging*: In addition to the API, users can choose to integrate with Atlas using a messaging interface that is **Messaging**: In addition to the API, users can choose to integrate with Atlas using a messaging interface that is
based on Kafka. This is useful both for communicating metadata objects to Atlas, and also to consume metadata change based on Kafka. This is useful both for communicating metadata objects to Atlas, and also to consume metadata change
events from Atlas using which applications can be built. The messaging interface is particularly useful if one wishes events from Atlas using which applications can be built. The messaging interface is particularly useful if one wishes
to use a more loosely coupled integration with Atlas that could allow for better scalability, reliability etc. Atlas to use a more loosely coupled integration with Atlas that could allow for better scalability, reliability etc. Atlas
uses Apache Kafka as a notification server for communication between hooks and downstream consumers of metadata uses Apache Kafka as a notification server for communication between hooks and downstream consumers of metadata
notification events. Events are written by the hooks and Atlas to different Kafka topics. notification events. Events are written by the hooks and Atlas to different Kafka topics.
---+++ Metadata sources ### Metadata sources
Atlas supports integration with many sources of metadata out of the box. More integrations will be added in future Atlas supports integration with many sources of metadata out of the box. More integrations will be added in future
as well. Currently, Atlas supports ingesting and managing metadata from the following sources: as well. Currently, Atlas supports ingesting and managing metadata from the following sources:
* [HBase](/Hook-HBase)
* [[Hook-HBase][HBase]] * [Hive](/Hook-Hive)
* [[Hook-Hive][Hive]] * [Sqoop](/Hook-Sqoop)
* [[Hook-Sqoop][Sqoop]] * [Storm](/Hook-Storm)
* [[Hook-Storm][Storm]] * [Kafka](/Hook-Kafka)
* [[Bridge-Kafka][Kafka]]
The integration implies two things: The integration implies two things:
There are metadata models that Atlas defines natively to represent objects of these components. There are metadata models that Atlas defines natively to represent objects of these components.
There are components Atlas provides to ingest metadata objects from these components There are components Atlas provides to ingest metadata objects from these components
(in real time or in batch mode in some cases). (in real time or in batch mode in some cases).
---+++ Applications ### Applications
The metadata managed by Atlas is consumed by a variety of applications for satisfying many governance use cases. The metadata managed by Atlas is consumed by a variety of applications for satisfying many governance use cases.
*Atlas Admin UI*: This component is a web based application that allows data stewards and scientists to discover **Atlas Admin UI**: This component is a web based application that allows data stewards and scientists to discover
and annotate metadata. Of primary importance here is a search interface and SQL like query language that can be and annotate metadata. Of primary importance here is a search interface and SQL like query language that can be
used to query the metadata types and objects managed by Atlas. The Admin UI uses the REST API of Atlas for used to query the metadata types and objects managed by Atlas. The Admin UI uses the REST API of Atlas for
building its functionality. building its functionality.
*Tag Based Policies*: [[http://ranger.apache.org/][Apache Ranger]] is an advanced security management solution **Tag Based Policies**: [Apache Ranger](http://ranger.apache.org/) is an advanced security management solution
for the Hadoop ecosystem having wide integration with a variety of Hadoop components. By integrating with Atlas, for the Hadoop ecosystem having wide integration with a variety of Hadoop components. By integrating with Atlas,
Ranger allows security administrators to define metadata driven security policies for effective governance. Ranger allows security administrators to define metadata driven security policies for effective governance.
Ranger is a consumer to the metadata change events notified by Atlas. Ranger is a consumer to the metadata change events notified by Atlas.
---+ Falcon Atlas Bridge ---
name: Falcon
route: /Hook-Falcon
menu: Documentation
submenu: Hooks
---
---++ Falcon Model import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Falcon Atlas Bridge
## Falcon Model
The default hive model includes the following types: The default hive model includes the following types:
* Entity types: * Entity types:
* falcon_cluster * falcon_cluster
...@@ -22,25 +33,28 @@ The default hive model includes the following types: ...@@ -22,25 +33,28 @@ The default hive model includes the following types:
One falcon_process entity is created for every cluster that the falcon process is defined for. One falcon_process entity is created for every cluster that the falcon process is defined for.
The entities are created and de-duped using unique qualifiedName attribute. They provide namespace and can be used for querying/lineage as well. The unique attributes are: The entities are created and de-duped using unique qualifiedName attribute. They provide namespace and can be used for querying/lineage as well. The unique attributes are:
* falcon_process.qualifiedName - <process name>@<cluster name> * falcon_process.qualifiedName - `<process name>@<cluster name>`
* falcon_cluster.qualifiedName - <cluster name> * falcon_cluster.qualifiedName - `<cluster name>`
* falcon_feed.qualifiedName - <feed name>@<cluster name> * falcon_feed.qualifiedName - `<feed name>@<cluster name>`
* falcon_feed_creation.qualifiedName - <feed name> * falcon_feed_creation.qualifiedName - `<feed name>`
* falcon_feed_replication.qualifiedName - <feed name> * falcon_feed_replication.qualifiedName - `<feed name>`
---++ Falcon Hook ## Falcon Hook
Falcon supports listeners on falcon entity submission. This is used to add entities in Atlas using the model detailed above. Falcon supports listeners on falcon entity submission. This is used to add entities in Atlas using the model detailed above.
Follow the instructions below to setup Atlas hook in Falcon: Follow the instructions below to setup Atlas hook in Falcon:
* Add 'org.apache.atlas.falcon.service.AtlasService' to application.services in <falcon-conf>/startup.properties * Add 'org.apache.atlas.falcon.service.AtlasService' to application.services in `<falcon-conf>`/startup.properties
* untar apache-atlas-${project.version}-falcon-hook.tar.gz * untar apache-atlas-${project.version}-falcon-hook.tar.gz
* cd apache-atlas-falcon-hook-${project.version} * cd apache-atlas-falcon-hook-${project.version}
* Copy entire contents of folder apache-atlas-falcon-hook-${project.version}/hook/falcon to <atlas-home>/hook/falcon * Copy entire contents of folder apache-atlas-falcon-hook-${project.version}/hook/falcon to `<atlas-home>`/hook/falcon
* Link Atlas hook jars in Falcon classpath - 'ln -s <atlas-home>/hook/falcon/* <falcon-home>/server/webapp/falcon/WEB-INF/lib/' * Link Atlas hook jars in Falcon classpath - 'ln -s `<atlas-home>`/hook/falcon/* `<falcon-home>`/server/webapp/falcon/WEB-INF/lib/'
* In <falcon_conf>/falcon-env.sh, set an environment variable as follows: * In `<falcon_conf>`/falcon-env.sh, set an environment variable as follows:
<verbatim>
export FALCON_SERVER_OPTS="<atlas_home>/hook/falcon/*:$FALCON_SERVER_OPTS"</verbatim> <SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
{`export FALCON_SERVER_OPTS="<atlas_home>/hook/falcon/*:$FALCON_SERVER_OPTS"`}
</SyntaxHighlighter>
The following properties in <atlas-conf>/atlas-application.properties control the thread pool and notification details: The following properties in `<atlas-conf>`/atlas-application.properties control the thread pool and notification details:
* atlas.hook.falcon.synchronous - boolean, true to run the hook synchronously. default false * atlas.hook.falcon.synchronous - boolean, true to run the hook synchronously. default false
* atlas.hook.falcon.numRetries - number of retries for notification failure. default 3 * atlas.hook.falcon.numRetries - number of retries for notification failure. default 3
* atlas.hook.falcon.minThreads - core number of threads. default 5 * atlas.hook.falcon.minThreads - core number of threads. default 5
...@@ -48,8 +62,8 @@ The following properties in <atlas-conf>/atlas-application.properties control th ...@@ -48,8 +62,8 @@ The following properties in <atlas-conf>/atlas-application.properties control th
* atlas.hook.falcon.keepAliveTime - keep alive time in msecs. default 10 * atlas.hook.falcon.keepAliveTime - keep alive time in msecs. default 10
* atlas.hook.falcon.queueSize - queue size for the threadpool. default 10000 * atlas.hook.falcon.queueSize - queue size for the threadpool. default 10000
Refer [[Configuration][Configuration]] for notification related configurations Refer [Configuration](Configuration) for notification related configurations
---++ NOTES ## NOTES
* In falcon cluster entity, cluster name used should be uniform across components like hive, falcon, sqoop etc. If used with ambari, ambari cluster name should be used for cluster entity * In falcon cluster entity, cluster name used should be uniform across components like hive, falcon, sqoop etc. If used with ambari, ambari cluster name should be used for cluster entity
---+ Apache Atlas Hook & Bridge for Apache HBase ---
name: HBase
route: /Hook-HBase
menu: Documentation
submenu: Hooks
---
---++ HBase Model import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Apache Atlas Hook & Bridge for Apache HBase
## HBase Model
HBase model includes the following types: HBase model includes the following types:
* Entity types: * Entity types:
* hbase_namespace * hbase_namespace
...@@ -14,60 +25,63 @@ HBase model includes the following types: ...@@ -14,60 +25,63 @@ HBase model includes the following types:
* attributes: qualifiedName, name, description, owner, columns, createTime, bloomFilterType, compressionType, compactionCompressionType, encryptionType, inMemoryCompactionPolicy, keepDeletedCells, maxversions, minVersions, datablockEncoding, storagePolicy, ttl, blockCachedEnabled, cacheBloomsOnWrite, cacheDataOnWrite, evictBlocksOnClose, prefetchBlocksOnOpen, newVersionsBehavior, isMobEnabled, mobCompactPartitionPolicy * attributes: qualifiedName, name, description, owner, columns, createTime, bloomFilterType, compressionType, compactionCompressionType, encryptionType, inMemoryCompactionPolicy, keepDeletedCells, maxversions, minVersions, datablockEncoding, storagePolicy, ttl, blockCachedEnabled, cacheBloomsOnWrite, cacheDataOnWrite, evictBlocksOnClose, prefetchBlocksOnOpen, newVersionsBehavior, isMobEnabled, mobCompactPartitionPolicy
HBase entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Note that namespaceName, tableName and columnFamilyName should be in lower case. HBase entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Note that namespaceName, tableName and columnFamilyName should be in lower case.
<verbatim>
hbase_namespace.qualifiedName: <namespaceName>@<clusterName>
hbase_table.qualifiedName: <namespaceName>:<tableName>@<clusterName>
hbase_column_family.qualifiedName: <namespaceName>:<tableName>.<columnFamilyName>@<clusterName>
</verbatim>
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
{`hbase_namespace.qualifiedName: <namespaceName>@<clusterName>
hbase_table.qualifiedName: <namespaceName>:<tableName>@<clusterName>
hbase_column_family.qualifiedName: <namespaceName>:<tableName>.<columnFamilyName>@<clusterName>`}
</SyntaxHighlighter>
---++ HBase Hook
## HBase Hook
Atlas HBase hook registers with HBase master as a co-processor. On detecting changes to HBase namespaces/tables/column-families, Atlas hook updates the metadata in Atlas via Kafka notifications. Atlas HBase hook registers with HBase master as a co-processor. On detecting changes to HBase namespaces/tables/column-families, Atlas hook updates the metadata in Atlas via Kafka notifications.
Follow the instructions below to setup Atlas hook in HBase: Follow the instructions below to setup Atlas hook in HBase:
* Register Atlas hook in hbase-site.xml by adding the following: * Register Atlas hook in hbase-site.xml by adding the following:
<verbatim>
<property> <SyntaxHighlighter wrapLines={true} language="xml" style={theme.dark}>
<name>hbase.coprocessor.master.classes</name> {`<property>
<value>org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor</value> <name>hbase.coprocessor.master.classes</name>
</property></verbatim> <value>org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor</value>
</property>`}
</SyntaxHighlighter>
* untar apache-atlas-${project.version}-hbase-hook.tar.gz * untar apache-atlas-${project.version}-hbase-hook.tar.gz
* cd apache-atlas-hbase-hook-${project.version} * cd apache-atlas-hbase-hook-${project.version}
* Copy entire contents of folder apache-atlas-hbase-hook-${project.version}/hook/hbase to <atlas package>/hook/hbase * Copy entire contents of folder apache-atlas-hbase-hook-${project.version}/hook/hbase to `<atlas package>`/hook/hbase
* Link Atlas hook jars in HBase classpath - 'ln -s <atlas package>/hook/hbase/* <hbase-home>/lib/' * Link Atlas hook jars in HBase classpath - 'ln -s `<atlas package>`/hook/hbase/* `<hbase-home>`/lib/'
* Copy <atlas-conf>/atlas-application.properties to the HBase conf directory. * Copy `<atlas-conf>`/atlas-application.properties to the HBase conf directory.
The following properties in atlas-application.properties control the thread pool and notification details: The following properties in atlas-application.properties control the thread pool and notification details:
<verbatim>
atlas.hook.hbase.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in HBase operations. Default: false <SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
{`atlas.hook.hbase.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in HBase operations. Default: false
atlas.hook.hbase.numRetries=3 # number of retries for notification failure. Default: 3 atlas.hook.hbase.numRetries=3 # number of retries for notification failure. Default: 3
atlas.hook.hbase.queueSize=10000 # queue size for the threadpool. Default: 10000 atlas.hook.hbase.queueSize=10000 # queue size for the threadpool. Default: 10000
atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary
atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181 atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181
atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000 atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000
atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000 atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20 atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}
</verbatim> </SyntaxHighlighter>
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.".
For list of configuration supported by Kafka producer, please refer to [[http://kafka.apache.org/documentation/#producerconfigs][Kafka Producer Configs]] For list of configuration supported by Kafka producer, please refer to [Kafka Producer Configs](http://kafka.apache.org/documentation/#producerconfigs)
---++ NOTES ## NOTES
* Only the namespace, table and column-family create/update/ delete operations are captured by Atlas HBase hook. Changes to columns are be captured. * Only the namespace, table and column-family create/update/ delete operations are captured by Atlas HBase hook. Changes to columns are be captured.
---++ Importing HBase Metadata ## Importing HBase Metadata
Apache Atlas provides a command-line utility, import-hbase.sh, to import metadata of Apache HBase namespaces and tables into Apache Atlas. Apache Atlas provides a command-line utility, import-hbase.sh, to import metadata of Apache HBase namespaces and tables into Apache Atlas.
This utility can be used to initialize Apache Atlas with namespaces/tables present in a Apache HBase cluster. This utility can be used to initialize Apache Atlas with namespaces/tables present in a Apache HBase cluster.
This utility supports importing metadata of a specific table, tables in a specific namespace or all tables. This utility supports importing metadata of a specific table, tables in a specific namespace or all tables.
<verbatim> <SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
Usage 1: <atlas package>/hook-bin/import-hbase.sh {`Usage 1: <atlas package>/hook-bin/import-hbase.sh
Usage 2: <atlas package>/hook-bin/import-hbase.sh [-n <namespace regex> OR --namespace <namespace regex>] [-t <table regex> OR --table <table regex>] Usage 2: <atlas package>/hook-bin/import-hbase.sh [-n <namespace regex> OR --namespace <namespace regex>] [-t <table regex> OR --table <table regex>]
Usage 3: <atlas package>/hook-bin/import-hbase.sh [-f <filename>] Usage 3: <atlas package>/hook-bin/import-hbase.sh [-f <filename>]
File Format: File Format:
namespace1:tbl1 namespace1:tbl1
namespace1:tbl2 namespace1:tbl2
namespace2:tbl1 namespace2:tbl1`}
</verbatim> </SyntaxHighlighter>
---+ Apache Atlas Hook & Bridge for Apache Hive ---
name: Hive
route: /Hook-Hive
menu: Documentation
submenu: Hooks
---
---++ Hive Model import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
import Img from 'theme/components/shared/Img'
# Apache Atlas Hook & Bridge for Apache Hive
## Hive Model
Hive model includes the following types: Hive model includes the following types:
* Entity types: * Entity types:
* hive_db * hive_db
* super-types: !Asset * super-types: !Asset
* attributes: qualifiedName, name, description, owner, clusterName, location, parameters, ownerName * attributes: qualifiedName, name, description, owner, clusterName, location, parameters, ownerName
* hive_table * hive_table
* super-types: !DataSet * super-types: !DataSet
* attributes: qualifiedName, name, description, owner, db, createTime, lastAccessTime, comment, retention, sd, partitionKeys, columns, aliases, parameters, viewOriginalText, viewExpandedText, tableType, temporary * attributes: qualifiedName, name, description, owner, db, createTime, lastAccessTime, comment, retention, sd, partitionKeys, columns, aliases, parameters, viewOriginalText, viewExpandedText, tableType, temporary
* hive_column * hive_column
...@@ -21,63 +34,69 @@ Hive model includes the following types: ...@@ -21,63 +34,69 @@ Hive model includes the following types:
* hive_column_lineage * hive_column_lineage
* super-types: Process * super-types: Process
* attributes: qualifiedName, name, description, owner, inputs, outputs, query, depenendencyType, expression * attributes: qualifiedName, name, description, owner, inputs, outputs, query, depenendencyType, expression
* Enum types: * Enum types:
* hive_principal_type * hive_principal_type
* values: USER, ROLE, GROUP * values: USER, ROLE, GROUP
* Struct types: * Struct types:
* hive_order * hive_order
* attributes: col, order * attributes: col, order
* hive_serde * hive_serde
* attributes: name, serializationLib, parameters * attributes: name, serializationLib, parameters
Hive entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Note that dbName, tableName and columnName should be in lower case. Hive entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Note that dbName, tableName and columnName should be in lower case.
<verbatim>
hive_db.qualifiedName: <dbName>@<clusterName>
hive_table.qualifiedName: <dbName>.<tableName>@<clusterName>
hive_column.qualifiedName: <dbName>.<tableName>.<columnName>@<clusterName>
hive_process.queryString: trimmed query string in lower case
</verbatim>
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`hive_db.qualifiedName: <dbName>@<clusterName>
hive_table.qualifiedName: <dbName>.<tableName>@<clusterName>
hive_column.qualifiedName: <dbName>.<tableName>.<columnName>@<clusterName>
hive_process.queryString: trimmed query string in lower case`}
</SyntaxHighlighter>
---++ Hive Hook
## Hive Hook
Atlas Hive hook registers with Hive to listen for create/update/delete operations and updates the metadata in Atlas, via Kafka notifications, for the changes in Hive. Atlas Hive hook registers with Hive to listen for create/update/delete operations and updates the metadata in Atlas, via Kafka notifications, for the changes in Hive.
Follow the instructions below to setup Atlas hook in Hive: Follow the instructions below to setup Atlas hook in Hive:
* Set-up Atlas hook in hive-site.xml by adding the following: * Set-up Atlas hook in hive-site.xml by adding the following:
<verbatim>
<property> <SyntaxHighlighter wrapLines={true} language="xml" style={theme.dark}>
<name>hive.exec.post.hooks</name> {`<property>
<name>hive.exec.post.hooks</name>
<value>org.apache.atlas.hive.hook.HiveHook</value> <value>org.apache.atlas.hive.hook.HiveHook</value>
</property></verbatim> </property>`}
</SyntaxHighlighter>
* untar apache-atlas-${project.version}-hive-hook.tar.gz * untar apache-atlas-${project.version}-hive-hook.tar.gz
* cd apache-atlas-hive-hook-${project.version} * cd apache-atlas-hive-hook-${project.version}
* Copy entire contents of folder apache-atlas-hive-hook-${project.version}/hook/hive to <atlas package>/hook/hive * Copy entire contents of folder apache-atlas-hive-hook-${project.version}/hook/hive to `<atlas package>`/hook/hive
* Add 'export HIVE_AUX_JARS_PATH=<atlas package>/hook/hive' in hive-env.sh of your hive configuration * Add 'export HIVE_AUX_JARS_PATH=`<atlas package>`/hook/hive' in hive-env.sh of your hive configuration
* Copy <atlas-conf>/atlas-application.properties to the hive conf directory. * Copy `<atlas-conf>`/atlas-application.properties to the hive conf directory.
The following properties in atlas-application.properties control the thread pool and notification details: The following properties in atlas-application.properties control the thread pool and notification details:
<verbatim>
atlas.hook.hive.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Hive query completion. Default: false <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`atlas.hook.hive.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Hive query completion. Default: false
atlas.hook.hive.numRetries=3 # number of retries for notification failure. Default: 3 atlas.hook.hive.numRetries=3 # number of retries for notification failure. Default: 3
atlas.hook.hive.queueSize=10000 # queue size for the threadpool. Default: 10000 atlas.hook.hive.queueSize=10000 # queue size for the threadpool. Default: 10000
atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary
atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181 atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181
atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000 atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000
atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000 atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20 atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}
</verbatim> </SyntaxHighlighter>
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [[http://kafka.apache.org/documentation/#producerconfigs][Kafka Producer Configs]] Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [Kafka Producer Configs](http://kafka.apache.org/documentation/#producerconfigs)
---++ Column Level Lineage ## Column Level Lineage
Starting from 0.8-incubating version of Atlas, Column level lineage is captured in Atlas. Below are the details Starting from 0.8-incubating version of Atlas, Column level lineage is captured in Atlas. Below are the details
---+++ Model ### Model
* !ColumnLineageProcess type is a subtype of Process * !ColumnLineageProcess type is a subtype of Process
* This relates an output Column to a set of input Columns or the Input Table * This relates an output Column to a set of input Columns or the Input Table
...@@ -91,23 +110,24 @@ Starting from 0.8-incubating version of Atlas, Column level lineage is captured ...@@ -91,23 +110,24 @@ Starting from 0.8-incubating version of Atlas, Column level lineage is captured
* Since Process links input and output !DataSets, Column is a subtype of !DataSet * Since Process links input and output !DataSets, Column is a subtype of !DataSet
---+++ Examples ### Examples
For a simple CTAS below: For a simple CTAS below:
<verbatim>
create table t2 as select id, name from T1</verbatim> <SyntaxHighlighter wrapLines={true} language="sql" style={theme.dark}>
create table t2 as select id, name from T1
</SyntaxHighlighter>
The lineage is captured as The lineage is captured as
<img src="images/column_lineage_ex1.png" height="200" width="400" /> <Img src={`/images/column_lineage_ex1.png`} height="200" width="400" />
---+++ Extracting Lineage from Hive commands ### Extracting Lineage from Hive commands
* The !HiveHook maps the !LineageInfo in the !HookContext to Column lineage instances * The !HiveHook maps the !LineageInfo in the !HookContext to Column lineage instances
* The !LineageInfo in Hive provides column-level lineage for the final !FileSinkOperator, linking them to the input columns in the Hive Query * The !LineageInfo in Hive provides column-level lineage for the final !FileSinkOperator, linking them to the input columns in the Hive Query
---++ NOTES ## NOTES
* Column level lineage works with Hive version 1.2.1 after the patch for <a href="https://issues.apache.org/jira/browse/HIVE-13112">HIVE-13112</a> is applied to Hive source * Column level lineage works with Hive version 1.2.1 after the patch for <a href="https://issues.apache.org/jira/browse/HIVE-13112">HIVE-13112</a> is applied to Hive source
* Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names * Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names
* The following hive operations are captured by hive hook currently * The following hive operations are captured by hive hook currently
...@@ -120,17 +140,17 @@ The lineage is captured as ...@@ -120,17 +140,17 @@ The lineage is captured as
* alter view * alter view
---++ Importing Hive Metadata ## Importing Hive Metadata
Apache Atlas provides a command-line utility, import-hive.sh, to import metadata of Apache Hive databases and tables into Apache Atlas. Apache Atlas provides a command-line utility, import-hive.sh, to import metadata of Apache Hive databases and tables into Apache Atlas.
This utility can be used to initialize Apache Atlas with databases/tables present in Apache Hive. This utility can be used to initialize Apache Atlas with databases/tables present in Apache Hive.
This utility supports importing metadata of a specific table, tables in a specific database or all databases and tables. This utility supports importing metadata of a specific table, tables in a specific database or all databases and tables.
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
Usage 1: <atlas package>/hook-bin/import-hive.sh {`Usage 1: <atlas package>/hook-bin/import-hive.sh
Usage 2: <atlas package>/hook-bin/import-hive.sh [-d <database regex> OR --database <database regex>] [-t <table regex> OR --table <table regex>] Usage 2: <atlas package>/hook-bin/import-hive.sh [-d <database regex> OR --database <database regex>] [-t <table regex> OR --table <table regex>]
Usage 3: <atlas package>/hook-bin/import-hive.sh [-f <filename>] Usage 3: <atlas package>/hook-bin/import-hive.sh [-f <filename>]
File Format: File Format:
database1:tbl1 database1:tbl1
database1:tbl2 database1:tbl2
database2:tbl1 database2:tbl1`}
</verbatim> </SyntaxHighlighter>
---
name: Kafka
route: /Hook-Kafka
menu: Documentation
submenu: Hooks
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
import Img from 'theme/components/shared/Img'
# Apache Atlas Hook for Apache Kafka
## Kafka Model
Kafka model includes the following types:
* Entity types:
* kafka_topic
* super-types: !DataSet
* attributes: qualifiedName, name, description, owner, topic, uri, partitionCount
Kafka entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below.
Note that qualifiedName will have topic name in lower case.
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`topic.qualifiedName: <topic>@<clusterName>`}
</SyntaxHighlighter>
## Setup
Binary files are present in apache-atlas-<release-version/>-kafka-hook.tar.gz
Copy apache-atlas-kafka-hook-<release-version/>/hook/kafka folder to <atlas package/>/hook/ directory
Copy apache-atlas-kafka-hook-<release-version/>/hook-bin folder to <atlas package/>/hook-bin directory
## Importing Kafka Metadata
Apache Atlas provides a command-line utility, import-kafka.sh, to import metadata of Apache Kafka topics into Apache Atlas.
This utility can be used to initialize Apache Atlas with topics present in Apache Kafka.
This utility supports importing metadata of a specific topic or all topics.
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`sage 1: <atlas package>/hook-bin/import-kafka.sh
Usage 2: <atlas package>/hook-bin/import-kafka.sh [-t <topic prefix> OR --topic <topic prefix>]
Usage 3: <atlas package>/hook-bin/import-kafka.sh [-f <filename>]
File Format:
topic1
topic2
topic3`}
</SyntaxHighlighter>
---+ Apache Atlas Hook for Apache Sqoop ---
name: Sqoop
route: /Hook-Sqoop
menu: Documentation
submenu: Hooks
---
---++ Sqoop Model import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Apache Atlas Hook for Apache Sqoop
## Sqoop Model
Sqoop model includes the following types: Sqoop model includes the following types:
* Entity types: * Entity types:
* sqoop_process * sqoop_process
...@@ -9,7 +20,6 @@ Sqoop model includes the following types: ...@@ -9,7 +20,6 @@ Sqoop model includes the following types:
* sqoop_dbdatastore * sqoop_dbdatastore
* super-types: !DataSet * super-types: !DataSet
* attributes: qualifiedName, name, description, owner, dbStoreType, storeUse, storeUri, source * attributes: qualifiedName, name, description, owner, dbStoreType, storeUse, storeUri, source
* Enum types: * Enum types:
* sqoop_operation_type * sqoop_operation_type
* values: IMPORT, EXPORT, EVAL * values: IMPORT, EXPORT, EVAL
...@@ -17,48 +27,52 @@ Sqoop model includes the following types: ...@@ -17,48 +27,52 @@ Sqoop model includes the following types:
* values: TABLE, QUERY, PROCEDURE, OTHER * values: TABLE, QUERY, PROCEDURE, OTHER
Sqoop entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Sqoop entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below.
<verbatim>
sqoop_process.qualifiedName: sqoop <operation> --connect <url> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>]
sqoop_dbdatastore.qualifiedName: <storeType> --url <storeUri> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>] --hive-<operation> --hive-database <databaseName> [--hive-table <tableName>] --hive-cluster <clusterName>
</verbatim>
---++ Sqoop Hook <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`sqoop_process.qualifiedName: sqoop <operation> --connect <url> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>]
sqoop_dbdatastore.qualifiedName: <storeType> --url <storeUri> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>] --hive-<operation> --hive-database <databaseName> [--hive-table <tableName>] --hive-cluster <clusterName>`}
</SyntaxHighlighter>
## Sqoop Hook
Sqoop added a !SqoopJobDataPublisher that publishes data to Atlas after completion of import Job. Today, only hiveImport is supported in !SqoopHook. Sqoop added a !SqoopJobDataPublisher that publishes data to Atlas after completion of import Job. Today, only hiveImport is supported in !SqoopHook.
This is used to add entities in Atlas using the model detailed above. This is used to add entities in Atlas using the model detailed above.
Follow the instructions below to setup Atlas hook in Hive: Follow the instructions below to setup Atlas hook in Hive:
Add the following properties to to enable Atlas hook in Sqoop: Add the following properties to to enable Atlas hook in Sqoop:
* Set-up Atlas hook in <sqoop-conf>/sqoop-site.xml by adding the following: * Set-up Atlas hook in `<sqoop-conf>`/sqoop-site.xml by adding the following:
<verbatim>
<property> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`<property>
<name>sqoop.job.data.publish.class</name> <name>sqoop.job.data.publish.class</name>
<value>org.apache.atlas.sqoop.hook.SqoopHook</value> <value>org.apache.atlas.sqoop.hook.SqoopHook</value>
</property></verbatim> </property>`}
</SyntaxHighlighter>
* untar apache-atlas-${project.version}-sqoop-hook.tar.gz * untar apache-atlas-${project.version}-sqoop-hook.tar.gz
* cd apache-atlas-sqoop-hook-${project.version} * cd apache-atlas-sqoop-hook-${project.version}
* Copy entire contents of folder apache-atlas-sqoop-hook-${project.version}/hook/sqoop to <atlas package>/hook/sqoop * Copy entire contents of folder apache-atlas-sqoop-hook-${project.version}/hook/sqoop to `<atlas package>`/hook/sqoop
* Copy <atlas-conf>/atlas-application.properties to to the sqoop conf directory <sqoop-conf>/ * Copy `<atlas-conf>`/atlas-application.properties to to the sqoop conf directory `<sqoop-conf>`/
* Link <atlas package>/hook/sqoop/*.jar in sqoop lib * Link `<atlas package>`/hook/sqoop/*.jar in sqoop lib
The following properties in atlas-application.properties control the thread pool and notification details: The following properties in atlas-application.properties control the thread pool and notification details:
<verbatim>
atlas.hook.sqoop.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Sqoop operation completion. Default: false <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`atlas.hook.sqoop.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Sqoop operation completion. Default: false
atlas.hook.sqoop.numRetries=3 # number of retries for notification failure. Default: 3 atlas.hook.sqoop.numRetries=3 # number of retries for notification failure. Default: 3
atlas.hook.sqoop.queueSize=10000 # queue size for the threadpool. Default: 10000 atlas.hook.sqoop.queueSize=10000 # queue size for the threadpool. Default: 10000
atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary
atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181 atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181
atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000 atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000
atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000 atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20 atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}
</verbatim> </SyntaxHighlighter>
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [[http://kafka.apache.org/documentation/#producerconfigs][Kafka Producer Configs]] Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [Kafka Producer Configs](http://kafka.apache.org/documentation/#producerconfigs)
---++ NOTES ## NOTES
* Only the following sqoop operations are captured by sqoop hook currently * Only the following sqoop operations are captured by sqoop hook currently
* hiveImport * hiveImport
---+ Apache Atlas Hook for Apache Storm ---
name: Storm
route: /Hook-Storm
menu: Documentation
submenu: Hooks
---
---++ Introduction import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Apache Atlas Hook for Apache Storm
## Introduction
Apache Storm is a distributed real-time computation system. Storm makes it Apache Storm is a distributed real-time computation system. Storm makes it
easy to reliably process unbounded streams of data, doing for real-time easy to reliably process unbounded streams of data, doing for real-time
...@@ -20,7 +31,7 @@ There are 2 parts in this process detailed below: ...@@ -20,7 +31,7 @@ There are 2 parts in this process detailed below:
* Storm Atlas Hook to update metadata in Atlas * Storm Atlas Hook to update metadata in Atlas
---++ Storm Data Model ## Storm Data Model
A data model is represented as Types in Atlas. It contains the descriptions A data model is represented as Types in Atlas. It contains the descriptions
of various nodes in the topology graph, such as spouts and bolts and the of various nodes in the topology graph, such as spouts and bolts and the
...@@ -39,7 +50,7 @@ if it finds that these are not known to the Atlas server. ...@@ -39,7 +50,7 @@ if it finds that these are not known to the Atlas server.
The data model for each of the types is described in The data model for each of the types is described in
the class definition at org.apache.atlas.storm.model.StormDataModel. the class definition at org.apache.atlas.storm.model.StormDataModel.
---++ Storm Atlas Hook ## Storm Atlas Hook
Atlas is notified when a new topology is registered successfully in Atlas is notified when a new topology is registered successfully in
Storm. Storm provides a hook, backtype.storm.ISubmitterHook, at the Storm client used to Storm. Storm provides a hook, backtype.storm.ISubmitterHook, at the Storm client used to
...@@ -50,7 +61,7 @@ topology and updates Atlas using the types defined. Atlas implements the ...@@ -50,7 +61,7 @@ topology and updates Atlas using the types defined. Atlas implements the
Storm client hook interface in org.apache.atlas.storm.hook.StormAtlasHook. Storm client hook interface in org.apache.atlas.storm.hook.StormAtlasHook.
---++ Limitations ## Limitations
The following apply for the first version of the integration. The following apply for the first version of the integration.
...@@ -59,7 +70,7 @@ The following apply for the first version of the integration. ...@@ -59,7 +70,7 @@ The following apply for the first version of the integration.
* The Hook currently does not support capturing lineage for custom spouts and bolts. * The Hook currently does not support capturing lineage for custom spouts and bolts.
---++ Installation ## Installation
The Storm Atlas Hook needs to be manually installed in Storm on the client side. The Storm Atlas Hook needs to be manually installed in Storm on the client side.
* untar apache-atlas-${project.version}-storm-hook.tar.gz * untar apache-atlas-${project.version}-storm-hook.tar.gz
...@@ -72,16 +83,16 @@ Replace STORM_HOME with storm installation path. ...@@ -72,16 +83,16 @@ Replace STORM_HOME with storm installation path.
Restart all daemons after you have installed the atlas hook into Storm. Restart all daemons after you have installed the atlas hook into Storm.
---++ Configuration ## Configuration
---+++ Storm Configuration ### Storm Configuration
The Storm Atlas Hook needs to be configured in Storm client config The Storm Atlas Hook needs to be configured in Storm client config
in *$STORM_HOME/conf/storm.yaml* as: in *$STORM_HOME/conf/storm.yaml* as:
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
storm.topology.submission.notifier.plugin.class: "org.apache.atlas.storm.hook.StormAtlasHook" {`storm.topology.submission.notifier.plugin.class: "org.apache.atlas.storm.hook.StormAtlasHook"`}
</verbatim> </SyntaxHighlighter>
Also set a 'cluster name' that would be used as a namespace for objects registered in Atlas. Also set a 'cluster name' that would be used as a namespace for objects registered in Atlas.
This name would be used for namespacing the Storm topology, spouts and bolts. This name would be used for namespacing the Storm topology, spouts and bolts.
...@@ -94,23 +105,23 @@ the client and the cluster name is defined there. This happens similarly for HBa ...@@ -94,23 +105,23 @@ the client and the cluster name is defined there. This happens similarly for HBa
data sets. In case this configuration is not available, the cluster name set in the Storm data sets. In case this configuration is not available, the cluster name set in the Storm
configuration will be used. configuration will be used.
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
atlas.cluster.name: "cluster_name" atlas.cluster.name: "cluster_name"
</verbatim> </SyntaxHighlighter>
In *$STORM_HOME/conf/storm_env.ini*, set an environment variable as follows: In *$STORM_HOME/conf/storm_env.ini*, set an environment variable as follows:
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
STORM_JAR_JVM_OPTS:"-Datlas.conf=$ATLAS_HOME/conf/" STORM_JAR_JVM_OPTS:"-Datlas.conf=$ATLAS_HOME/conf/"
</verbatim> </SyntaxHighlighter>
where ATLAS_HOME is pointing to where ATLAS is installed. where ATLAS_HOME is pointing to where ATLAS is installed.
You could also set this up programatically in Storm Config as: You could also set this up programatically in Storm Config as:
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
Config stormConf = new Config(); {`Config stormConf = new Config();
... ...
stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN, stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
org.apache.atlas.storm.hook.StormAtlasHook.class.getName()); org.apache.atlas.storm.hook.StormAtlasHook.class.getName());`}
</verbatim> </SyntaxHighlighter>
---+ Export API ---
name: Export API
route: /Export-API
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Export API
The general approach is: The general approach is:
* Consumer specifies the scope of data to be exported (details below). * Consumer specifies the scope of data to be exported (details below).
* The API if successful, will return the stream in the format specified. * The API if successful, will return the stream in the format specified.
* Error will be returned on failure of the call. * Error will be returned on failure of the call.
See [[Export-HDFS-API][here]] for details on exporting *hdfs_path* entities. See [here](http://atlas.apache.org/Export-HDFS-API.html) for details on exporting *hdfs_path* entities.
|*Title*|*Export API*| |**Title**|**Export API**|
| ------------ | ------------ |
| _Example_ | See Examples sections below. | | _Example_ | See Examples sections below. |
| _URL_ |_api/atlas/admin/export_ | | _URL_ |_api/atlas/admin/export_ |
| _Method_ |_POST_ | | _Method_ |_POST_ |
...@@ -17,13 +29,14 @@ See [[Export-HDFS-API][here]] for details on exporting *hdfs_path* entities. ...@@ -17,13 +29,14 @@ See [[Export-HDFS-API][here]] for details on exporting *hdfs_path* entities.
| _Notes_ | Consumer could choose to consume the output of the API by programmatically using _java.io.ByteOutputStream_ or by manually, save the contents of the stream to a file on the disk.| | _Notes_ | Consumer could choose to consume the output of the API by programmatically using _java.io.ByteOutputStream_ or by manually, save the contents of the stream to a file on the disk.|
__Method Signature__ __Method Signature__
<verbatim>
@POST <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`@POST
@Path("/export") @Path("/export")
@Consumes("application/json;charset=UTF-8") @Consumes("application/json;charset=UTF-8")`}
</verbatim> </SyntaxHighlighter>
---+++ Additional Options ### Additional Options
It is possible to specify additional parameters for the _Export_ operation. It is possible to specify additional parameters for the _Export_ operation.
Current implementation has 2 options. Both are optional: Current implementation has 2 options. Both are optional:
...@@ -36,17 +49,17 @@ Current implementation has 2 options. Both are optional: ...@@ -36,17 +49,17 @@ Current implementation has 2 options. Both are optional:
* _fetchType_ This option configures the approach used for fetching entities. It has following values: * _fetchType_ This option configures the approach used for fetching entities. It has following values:
* _FULL_: This fetches all the entities that are connected directly and indirectly to the starting entity. E.g. If a starting entity specified is a table, then this option will fetch the table, database and all the other tables within the database. * _FULL_: This fetches all the entities that are connected directly and indirectly to the starting entity. E.g. If a starting entity specified is a table, then this option will fetch the table, database and all the other tables within the database.
* _CONNECTED_: This fetches all the etnties that are connected directly to the starting entity. E.g. If a starting entity specified is a table, then this option will fetch the table and the database entity only. * _CONNECTED_: This fetches all the etnties that are connected directly to the starting entity. E.g. If a starting entity specified is a table, then this option will fetch the table and the database entity only.
* _INCREMENTAL_: See [[Incremental-Export][here]] for details. * _INCREMENTAL_: See [here](http://atlas.apache.org/Incremental-Export.html) for details.
If no _matchType_ is specified, exact match is used. Which means, that the entire string is used in the search criteria. If no _matchType_ is specified, exact match is used. Which means, that the entire string is used in the search criteria.
Searching using _matchType_ applies for all types of entities. It is particularly useful for matching entities of type hdfs_path (see [[Export-HDFS-API][here]]). Searching using _matchType_ applies for all types of entities. It is particularly useful for matching entities of type hdfs_path (see (here)[Export-HDFS-API]).
The _fetchType_ option defaults to _FULL_. The _fetchType_ option defaults to _FULL_.
For complete example see section below. For complete example see section below.
---+++ Contents of Exported ZIP File ### Contents of Exported ZIP File
The exported ZIP file has the following entries within it: The exported ZIP file has the following entries within it:
* _atlas-export-result.json_: * _atlas-export-result.json_:
...@@ -57,20 +70,22 @@ The exported ZIP file has the following entries within it: ...@@ -57,20 +70,22 @@ The exported ZIP file has the following entries within it:
* _atlas-export-order.json_: Order in which entities should be exported. * _atlas-export-order.json_: Order in which entities should be exported.
* _{guid}.json_: Individual entities are exported with file names that correspond to their id. * _{guid}.json_: Individual entities are exported with file names that correspond to their id.
---+++ Examples ### Examples
The _!AtlasExportRequest_ below shows filters that attempt to export 2 databases in cluster cl1: The _!AtlasExportRequest_ below shows filters that attempt to export 2 databases in cluster cl1:
<verbatim>
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"itemsToExport": [ "itemsToExport": [
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "accounts@cl1" } }, { "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "accounts@cl1" } },
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "hr@cl1" } } { "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "hr@cl1" } }
] ]
} }`}
</verbatim> </SyntaxHighlighter>
The _!AtlasExportRequest_ below specifies the _fetchType_ as _FULL_. The _matchType_ option will fetch _accounts@cl1_. The _!AtlasExportRequest_ below specifies the _fetchType_ as _FULL_. The _matchType_ option will fetch _accounts@cl1_.
<verbatim>
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"itemsToExport": [ "itemsToExport": [
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "accounts@" } }, { "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "accounts@" } },
], ],
...@@ -78,12 +93,13 @@ The _!AtlasExportRequest_ below specifies the _fetchType_ as _FULL_. The _matchT ...@@ -78,12 +93,13 @@ The _!AtlasExportRequest_ below specifies the _fetchType_ as _FULL_. The _matchT
"fetchType": "FULL", "fetchType": "FULL",
"matchType": "startsWith" "matchType": "startsWith"
} }
} }`}
</verbatim> </SyntaxHighlighter>
The _!AtlasExportRequest_ below specifies the _fetchType_ as _connected_. The _matchType_ option will fetch _accountsReceivable_, _accountsPayable_, etc present in the database. The _!AtlasExportRequest_ below specifies the _fetchType_ as _connected_. The _matchType_ option will fetch _accountsReceivable_, _accountsPayable_, etc present in the database.
<verbatim>
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"itemsToExport": [ "itemsToExport": [
{ "typeName": "hive_db", "uniqueAttributes": { "name": "accounts" } }, { "typeName": "hive_db", "uniqueAttributes": { "name": "accounts" } },
], ],
...@@ -91,15 +107,15 @@ The _!AtlasExportRequest_ below specifies the _fetchType_ as _connected_. The _m ...@@ -91,15 +107,15 @@ The _!AtlasExportRequest_ below specifies the _fetchType_ as _connected_. The _m
"fetchType": "CONNECTED", "fetchType": "CONNECTED",
"matchType": "startsWith" "matchType": "startsWith"
} }
} }`}
</verbatim> </SyntaxHighlighter>
Below is the _!AtlasExportResult_ JSON for the export of the _Sales_ DB present in the _!QuickStart_. Below is the _!AtlasExportResult_ JSON for the export of the _Sales_ DB present in the _!QuickStart_.
The _metrics_ contains the number of types and entities exported as part of the operation. The _metrics_ contains the number of types and entities exported as part of the operation.
<verbatim> <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"clientIpAddress": "10.0.2.15", "clientIpAddress": "10.0.2.15",
"hostName": "10.0.2.2", "hostName": "10.0.2.2",
"metrics": { "metrics": {
...@@ -128,26 +144,24 @@ The _metrics_ contains the number of types and entities exported as part of the ...@@ -128,26 +144,24 @@ The _metrics_ contains the number of types and entities exported as part of the
} }
], ],
"options": { "options": {
"fetchType": "FULL" "fetchType": "full"
} }
}, },
"userName": "admin" "userName": "admin"
} }`}
</verbatim> </SyntaxHighlighter>
---+++ CURL Calls ### CURL Calls
Below are sample CURL calls that demonstrate Export of _!QuickStart_ database. Below are sample CURL calls that demonstrate Export of _!QuickStart_ database.
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{ {`curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
"itemsToExport": [ "itemsToExport": [
{ "typeName": "DB", "uniqueAttributes": { "name": "Sales" } }, { "typeName": "DB", "uniqueAttributes": { "name": "Sales" }
{ "typeName": "DB", "uniqueAttributes": { "name": "Reporting" } }, { "typeName": "DB", "uniqueAttributes": { "name": "Reporting" }
{ "typeName": "DB", "uniqueAttributes": { "name": "Logging" } } { "typeName": "DB", "uniqueAttributes": { "name": "Logging" }
}
], ],
"options": { "options": { "full" }
"fetchType": "FULL" }' "http://localhost:21000/api/atlas/admin/export" > quickStartDB.zip`}
} </SyntaxHighlighter>
}' "http://localhost:21000/api/atlas/admin/export" > quickStartDB.zip
</verbatim>
---
name: Export HDFS API
route: /Export-HDFS-API
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Export & Import APIs for HDFS Path
### Introduction
The general approach for using the Import-Export APIs for HDFS Paths remain the same. There are minor variations caused how HDFS paths are handled within Atlas.
Unlike HIVE entities, HDFS entities within Atlas are created manually using the _Create Entity_ link within the Atlas Web UI.
Also, HDFS paths tend to be hierarchical, in the sense that users tend to model the same HDFS storage structure within Atlas.
__Sample HDFS Setup__
|**HDFS Path**|**Atlas Entity**|
| ------------ | ------------ |
|<em>/apps/warehouse/finance</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>Finance</em><br/>**QualifiedName: **<em>FinanceAll</em>|
|<em>/apps/warehouse/finance/accounts-receivable</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>FinanceReceivable</em><br/>**QualifiedName: **<em>FinanceReceivable</em><br/>**Path: **<em>/apps/warehouse/finance</em>|
|<em>/apps/warehouse/finance/accounts-payable</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>Finance-Payable</em><br/>**QualifiedName: **<em>FinancePayable</em><br/>**Path: **<em>/apps/warehouse/finance/accounts-payable</em>|
|<em>/apps/warehouse/finance/billing</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>FinanceBilling</em><br/>**QualifiedName: **<em>FinanceBilling</em><br/>**Path: **<em>/apps/warehouse/finance/billing</em>|
### Export API Using matchType
To export entities that represent HDFS path, use the Export API using the _matchType_ option. Details can be found [here](Export-API).
### Example Using CURL Calls
Below are sample CURL calls that performs export operation on the _Sample HDFS Setup_ shown above.
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
"itemsToExport": [
{ "typeName": "hdfs_path", "uniqueAttributes": { "name": "FinanceAll" }
}
],
"options": {
"fetchType": "full",
"matchType": "startsWith"
}
}' "http://localhost:21000/api/atlas/admin/export" > financeAll.zip`}
</SyntaxHighlighter>
### Automatic Creation of HDFS entities
Given that HDFS entity creation is a manual process. The Export API offers a mechanism for creation of requested HDFS entities.
<!-- ---
~ Licensed to the Apache Software Foundation (ASF) under one name: Export Import Audits
~ or more contributor license agreements. See the NOTICE file route: /ExportImportAudits
~ distributed with this work for additional information menu: Documentation
~ regarding copyright ownership. The ASF licenses this file submenu: Import/Export
~ to you under the Apache License, Version 2.0 (the ---
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at import themen from 'theme/styles/styled-colors';
~ import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
~ http://www.apache.org/licenses/LICENSE-2.0 import SyntaxHighlighter from 'react-syntax-highlighter';
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
# Export & Import Audits # Export & Import Audits
#### Background #### Background
...@@ -23,7 +17,7 @@ The new audits for Export and Import operations also have corresponding REST API ...@@ -23,7 +17,7 @@ The new audits for Export and Import operations also have corresponding REST API
#### REST APIs #### REST APIs
|Title | Replication Audits for a Cluster | |**Title** | **Replication Audits for a Cluster** |
|----------------|------------------------------------------------------------------| |----------------|------------------------------------------------------------------|
|Example | See below. | |Example | See below. |
|URL | api/atlas/admin/expimp/audit | |URL | api/atlas/admin/expimp/audit |
...@@ -44,8 +38,8 @@ The new audits for Export and Import operations also have corresponding REST API ...@@ -44,8 +38,8 @@ The new audits for Export and Import operations also have corresponding REST API
###### CURL ###### CURL
curl -X GET -u admin:admin -H "Content-Type: application/json" -H "Cache-Control: no-cache" 'http://localhost:21000/api/atlas/admin/expimp/audit?sourceClusterName=cl2' curl -X GET -u admin:admin -H "Content-Type: application/json" -H "Cache-Control: no-cache" 'http://localhost:21000/api/atlas/admin/expimp/audit?sourceClusterName=cl2'
```json <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"queryType": "BASIC", "queryType": "BASIC",
"searchParameters": { "searchParameters": {
"typeName": "ReplicationAuditEntry", "typeName": "ReplicationAuditEntry",
...@@ -89,5 +83,5 @@ curl -X GET -u admin:admin -H "Content-Type: application/json" -H "Cache-Control ...@@ -89,5 +83,5 @@ curl -X GET -u admin:admin -H "Content-Type: application/json" -H "Cache-Control
"displayText": "cl2", "displayText": "cl2",
"classificationNames": [] "classificationNames": []
}] }]
} }`}
``` </SyntaxHighlighter>
---+ Import API Options ---
name: Import API Options
route: /Import-API-Options
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Import API Options
Import API options are specified as _options_ JSON. Since the API accepts multi-part form data, it is possible to sepecify multipls input streams within the CURL call. Import API options are specified as _options_ JSON. Since the API accepts multi-part form data, it is possible to sepecify multipls input streams within the CURL call.
---+++ Examples Using CURL Calls ### Examples Using CURL Calls
<verbatim> <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data" {`curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
-H "Cache-Control: no-cache" -H "Cache-Control: no-cache"
-F request=@importOptions.json -F request=@importOptions.json
-F data=@quickStartDB.zip -F data=@quickStartDB.zip
"http://localhost:21000/api/atlas/admin/import" "http://localhost:21000/api/atlas/admin/import"`}
</verbatim> </SyntaxHighlighter>
To use the defaults, set the contents of _importOptions.json_ to: To use the defaults, set the contents of _importOptions.json_ to:
<verbatim>
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"options": { "options": {
} }
} }`}
</verbatim> </SyntaxHighlighter>
---+++ Options ### Options
Following options are supported for Import process: Following options are supported for Import process:
* Specify transforms during import operation. * Specify transforms during import operation.
...@@ -28,7 +40,7 @@ Following options are supported for Import process: ...@@ -28,7 +40,7 @@ Following options are supported for Import process:
* Optionally import type definition. * Optionally import type definition.
* Handling large imports. * Handling large imports.
---++++ Transforms #### Transforms
During the import process, the attribute value of the incoming entity can be changed. During the import process, the attribute value of the incoming entity can be changed.
...@@ -43,50 +55,51 @@ Example: ...@@ -43,50 +55,51 @@ Example:
The example below applies couple of transforms to the the _qualifiedName_ attribute of hive_table. It converts the value to lower case, then searches for 'cl1', if found, replaces it with 'cl2'. The example below applies couple of transforms to the the _qualifiedName_ attribute of hive_table. It converts the value to lower case, then searches for 'cl1', if found, replaces it with 'cl2'.
To use the option, set the contents of _importOptions.json_ to: To use the option, set the contents of _importOptions.json_ to:
<verbatim>
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"options": { "options": {
"transforms": "{ \"hive_table\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] }, \"hive_db\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] } }" "transforms": {"hive_table": { "qualifiedName": [ replace:@cl1:@cl2 ] }, "hive_db": { "qualifiedName": [ replace:@cl1:@cl2 ] } }
} }
} }`}
</verbatim> </SyntaxHighlighter>
Please refer to [[https://issues.apache.org/jira/browse/ATLAS-1825][ATLAS-1825]] for details scenarios when this option could be used. Please refer to [ATLAS-1825](https://issues.apache.org/jira/browse/ATLAS-1825) for details scenarios when this option could be used.
---++++ Start Guid or Start Index #### Start Guid or Start Index
When an import operation is in progress and the server goes down, it would be possible to resume import from the last successfully imported entity. This would allow the import to resume from where it left off. When an import operation is in progress and the server goes down, it would be possible to resume import from the last successfully imported entity. This would allow the import to resume from where it left off.
Server-side logging is improved to display the detail of the last successfully imported entity, this includes the index within the import list and the entity's guid. Either can be used specify the point to resume import. Server-side logging is improved to display the detail of the last successfully imported entity, this includes the index within the import list and the entity's guid. Either can be used specify the point to resume import.
To use the option, set the contents of _importOptions.json_ to: To use the option, set the contents of _importOptions.json_ to:
<verbatim>
{
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`{
"options": { "options": {
"startGuid": "bd97c78e-3fa5-4f9c-9f48-3683ca3d1fb1" "startGuid": "bd97c78e-3fa5-4f9c-9f48-3683ca3d1fb1"
} }
} }`}
</verbatim> </SyntaxHighlighter>
To use _startPosition_, use the following in the _importOptions.json_: To use _startPosition_, use the following in the _importOptions.json_:
<verbatim>
{
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"options": { "options": {
"startPosition": "332" "startPosition": "332"
} }
} }`}
</verbatim> </SyntaxHighlighter>
Steps to use the behavior: Steps to use the behavior:
* Start an import (using the CURL) that is fairly long, say about 1000+ entities. * Start an import (using the CURL) that is fairly long, say about 1000# entities.
* While the import is in progress, stop atlas server (using atlas_stop.py). * While the import is in progress, stop atlas server (using atlas_stop.py).
* From the log file located at _/var/log/atlas/application.log_ get the last successfully imported entity GUID or index position. * From the log file located at _/var/log/atlas/application.log_ get the last successfully imported entity GUID or index position.
* Update the _importOptions.json_ with the guid. * Update the _importOptions.json_ with the guid.
* Restart import. * Restart import.
---++++ Optional Importing Type Definition #### Optional Importing Type Definition
The output of Export has _atlas-typedef.json_ that contains the type definitions for the entities exported. The output of Export has _atlas-typedef.json_ that contains the type definitions for the entities exported.
...@@ -98,44 +111,45 @@ This option allows for optionally importing of type definition. The option is se ...@@ -98,44 +111,45 @@ This option allows for optionally importing of type definition. The option is se
Table below enumerates the conditions that get addressed as part of type definition import: Table below enumerates the conditions that get addressed as part of type definition import:
|*Condition*|*Action*| |**Condition**|**Action**|
| Incoming type does not exist in target system | Type is created. | | Incoming type does not exist in target system | Type is created. |
|Type to be imported and type in target system are same | No change | |Type to be imported and type in target system are same | No change |
|Type to be imported and type in target system differ by some attributes| Target system type is updated to the attributes present in the source. It is possible that the target system will have attributes in addition to the one present in the source. In that case, the target system's type attributes will be an union of the attributes. Attributes in target system will not be deleted to match the source. If the type of the attribute differ, import process will be aborted and exception logged.| |Type to be imported and type in target system differ by some attributes| Target system type is updated to the attributes present in the source. It is possible that the target system will have attributes in addition to the one present in the source. In that case, the target system's type attributes will be an union of the attributes. Attributes in target system will not be deleted to match the source. If the type of the attribute differ, import process will be aborted and exception logged.|
To use the option, set the contents of _importOptions.json_ to: To use the option, set the contents of _importOptions.json_ to:
<verbatim>
{
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"options": { "options": {
"updateTypeDefinition": true "updateTypeDefinition": true
} }
} }`}
</verbatim> </SyntaxHighlighter>
---++++ Specifying File to be Imported From Server Location #### Specifying File to be Imported From Server Location
In scenario where the file to be imported is present at a location on the server, the _importfile_ API can be used. It behaves like the Import API. In scenario where the file to be imported is present at a location on the server, the _importfile_ API can be used. It behaves like the Import API.
To use the option, set the contents of _importOptions.json_ to: To use the option, set the contents of _importOptions.json_ to:
<verbatim>
{
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"options": { "options": {
"fileName": "/root/fileToBeImported.zip" "fileName": "/root/fileToBeImported.zip"
} }
} }`}
</verbatim> </SyntaxHighlighter>
_CURL_ _CURL_
<verbatim>
curl -g -X POST -u adminuser:password -H "Content-Type: application/json" <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`curl -g -X POST -u adminuser:password -H "Content-Type: application/json"
-H "Cache-Control: no-cache" -H "Cache-Control: no-cache"
-d r@importOptions.json -d r@importOptions.json
"http://localhost:21000/api/atlas/admin/importfile" "http://localhost:21000/api/atlas/admin/importfile"`}
</verbatim> </SyntaxHighlighter>
---++++ Handling Large Imports #### Handling Large Imports
By default, the Import Service stores all of the data in memory. This may be limiting for ZIPs containing large amount of data. By default, the Import Service stores all of the data in memory. This may be limiting for ZIPs containing large amount of data.
......
---+ Import API ---
name: Import API
route: /Import-API
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Import API
The general approach is: The general approach is:
* Consumer makes a ZIP file available for import operation. See details below for the 2 flavors of the API. * Consumer makes a ZIP file available for import operation. See details below for the 2 flavors of the API.
* The API if successful, will return the results of the operation. * The API if successful, will return the results of the operation.
* Error will be returned on failure of the call. * Error will be returned on failure of the call.
---+++ Import ZIP File Using POST ### Import ZIP File Using POST
|*Title*|*Import API*| | **Title** | **Import API** |
| ------------ | ------------ |
| _Example_ | See Examples sections below. | | _Example_ | See Examples sections below. |
| _Description_|Provide the contents of the file to be imported in the request body.| | _Description_|Provide the contents of the file to be imported in the request body.|
| _URL_ |_api/atlas/admin/import_ | | _URL_ |_api/atlas/admin/import_ |
...@@ -17,9 +30,10 @@ The general approach is: ...@@ -17,9 +30,10 @@ The general approach is:
| _Success Response_ | _!AtlasImporResult_ is returned as JSON. See details below.| | _Success Response_ | _!AtlasImporResult_ is returned as JSON. See details below.|
|_Error Response_|Errors that are handled within the system will be returned as _!AtlasBaseException_. | |_Error Response_|Errors that are handled within the system will be returned as _!AtlasBaseException_. |
---+++ Import ZIP File Available on Server ### Import ZIP File Available on Server
|*Title*|*Import API*| |**Title**|**Import API**|
| ------------ | ------------ |
| _Example_ | See Examples sections below. | | _Example_ | See Examples sections below. |
| _Description_|Provide the path of the file to be imported.| | _Description_|Provide the path of the file to be imported.|
| _URL_ |_api/atlas/admin/importfile_ | | _URL_ |_api/atlas/admin/importfile_ |
...@@ -31,64 +45,69 @@ The general approach is: ...@@ -31,64 +45,69 @@ The general approach is:
|_Notes_| The file to be imported needs to be present on the server at the location specified by the _FILENAME_ parameter.| |_Notes_| The file to be imported needs to be present on the server at the location specified by the _FILENAME_ parameter.|
__Method Signature for Import__ __Method Signature for Import__
<verbatim>
@POST <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`@POST
@Path("/import") @Path("/import")
@Produces("application/json; charset=UTF-8") @Produces("application/json; charset=UTF-8")
@Consumes("multipart/form-data") @Consumes("multipart/form-data")`}
</verbatim> </SyntaxHighlighter>
__Method Signature for Import File__ __Method Signature for Import File__
<verbatim>
@POST <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`@POST
@Path("/importfile") @Path("/importfile")
@Produces("application/json; charset=UTF-8") @Produces("application/json; charset=UTF-8")
@Consumes("application/json") @Consumes("application/json")`}
</verbatim> </SyntaxHighlighter>
__Import Options__ __Import Options__
Please see __[[Import-API-Options][here]]__ for the available options during import process. Please see [here](Import-API-Options) for the available options during import process.
__!AtlasImportResult Response__ __AtlasImportResult Response__
The API will return the results of the import operation in the format defined by the _!AtlasImportResult_: The API will return the results of the import operation in the format defined by the _AtlasImportResult_:
* _!AtlasImportParameters_: This contains a collection of name value pair of the options that are applied during the import operation. * _AtlasImportParameters_: This contains a collection of name value pair of the options that are applied during the import operation.
* _Metrics_: Operation metrics. These include details on the number of types imported, number of entities imported, etc. * _Metrics_: Operation metrics. These include details on the number of types imported, number of entities imported, etc.
* _Processed Entities_: Contains list of GUIDs for the entities that were processed. * _Processed Entities_: Contains list of GUIDs for the entities that were processed.
* _Operation Status_: Overall status of the operation. Values are _SUCCESS_, PARTIAL_SUCCESS, _FAIL_. * _Operation Status_: Overall status of the operation. Values are _SUCCESS_, PARTIAL_SUCCESS, _FAIL_.
---+++ Examples Using CURL Calls ### Examples Using CURL Calls
The call below performs Import of _!QuickStart_ database using POST. The call below performs Import of _!QuickStart_ database using POST.
<verbatim>
curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data" <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
-H "Cache-Control: no-cache" -H "Cache-Control: no-cache"
-F request=@importOptions.json -F request=@importOptions.json
-F data=@quickStartDB.zip -F data=@quickStartDB.zip
"http://localhost:21000/api/atlas/admin/import" "http://localhost:21000/api/atlas/admin/import"`}
</verbatim> </SyntaxHighlighter>
The _request_ parameter is optional. If import has to be run without any options use: The _request_ parameter is optional. If import has to be run without any options use:
<verbatim>
curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data" <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
-H "Cache-Control: no-cache" -H "Cache-Control: no-cache"
-F data=@quickStartDB.zip -F data=@quickStartDB.zip
"http://localhost:21000/api/atlas/admin/import" "http://localhost:21000/api/atlas/admin/import"`}
</verbatim> </SyntaxHighlighter>
The call below performs Import of _QuickStart_ database using a ZIP file available on server.
The call below performs Import of _!QuickStart_ database using a ZIP file available on server. <SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
<verbatim> {`curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d ./importOptions.json
curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d ./importOptions.json "http://localhost:21000/api/atlas/admin/importFile" > quickStartDB-import-result.json`}
"http://localhost:21000/api/atlas/admin/importFile" > quickStartDB-import-result.json </SyntaxHighlighter>
</verbatim>
Below is the _!AtlasImportResult_ JSON for an import that contains _hive_db_. Below is the _AtlasImportResult_ JSON for an import that contains _hive_db_.
The _processedEntities_ contains the _guids_ of all the entities imported. The _processedEntities_ contains the _guids_ of all the entities imported.
The _metrics_ contain a breakdown of the types and entities imported along with the operation performed on them viz. _created_ or _updated_. The _metrics_ contain a breakdown of the types and entities imported along with the operation performed on them viz. _created_ or _updated_.
<verbatim> <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"request": { "request": {
"options": {} "options": {}
}, },
...@@ -112,12 +131,11 @@ The _metrics_ contain a breakdown of the types and entities imported along with ...@@ -112,12 +131,11 @@ The _metrics_ contain a breakdown of the types and entities imported along with
"processedEntities": [ "processedEntities": [
"2c4aa713-030b-4fb3-98b1-1cab23d9ac81", "2c4aa713-030b-4fb3-98b1-1cab23d9ac81",
"e4aa71ed-70fd-4fa7-9dfb-8250a573e293", "e4aa71ed-70fd-4fa7-9dfb-8250a573e293",
... ...
"ea0f9bdb-1dfc-4e48-9848-a006129929f9", "ea0f9bdb-1dfc-4e48-9848-a006129929f9",
"b5e2cb41-3e7d-4468-84e1-d87c320e75f9" "b5e2cb41-3e7d-4468-84e1-d87c320e75f9"
], ],
"operationStatus": "SUCCESS" "operationStatus": "SUCCESS"
} }`}
</verbatim> </SyntaxHighlighter>
---
name: Import Export API
route: /Import-Export-API
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# Export & Import REST APIs
### What's New
The release of 0.8.3 includes the following improvements to Export and Import APIs:
* Export: Support for [Incremental Export](Incremental-Export).
* Export & Import: Support for [replicated attributes](ReplicatedToFromAttributes) to entities made possible by [SoftReference](SoftReference) entity attribute option.
* Export option: [skipLineage](skipLineage).
* New entity transforms framework.
* New [AtlasServer](AtlasServer) entity type.
* Export: [Automatic creation of HDFS path](Export-HDFS-API) requested entities.
* New [ExportImportAudits](ExportImportAudits) for Export & Import operations.
### Background
The Import-Export APIs for Atlas facilitate transfer of data to and from a cluster that has Atlas provisioned.
The APIs when integrated with backup and/or disaster recovery process will ensure participation of Atlas.
### Introduction
There are 2 broad categories viz. Export & Import. The details of the APIs are discussed below.
The APIs are available only to _admin_ user.
Only a single import or export operation can be performed at a given time. The operations have a potential for generating large amount. They can also put pressure on resources. This restriction tries to alleviate this problem.
For Import-Export APIs relating to HDFS path, can be found [here](Import-Export-HDFS-Path).
For additional information please refer to the following:
* [ATLAS-1503](https://issues.apache.org/jira/browse/ATLAS-1503) Original Import-Export API requirements.
* [ATLAS-1618](https://issues.apache.org/jira/browse/ATLAS-1618) Export API Scope Specification.
### Errors
If an import or export operation is initiated while another is in progress, the consumer will receive this error:
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
{`"ATLAS5005E": "Another import or export is in progress. Please try again."`}
</SyntaxHighlighter>
Unhandled errors will be returned as Internal error code 500.
### REST API Reference
* [Export](Export-API)
* [Export HDFS](Export-HDFS-API)
* [Import](Import-API)
* [Import Options](Import-API-Options)
---
name: Import Entity Transforms
route: /ImportEntityTransforms
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
# (New) Entity Transforms Framework # (New) Entity Transforms Framework
#### Background #### Background
...@@ -30,7 +41,7 @@ The approach used by the new transformation framework creates a transformation b ...@@ -30,7 +41,7 @@ The approach used by the new transformation framework creates a transformation b
Following are built-in conditions. Following are built-in conditions.
Condition Types | Description | |**Condition Types** | **Description** |
-----------------------------------------|-----------------| -----------------------------------------|-----------------|
ENTITY_ALL | Any/every entity | ENTITY_ALL | Any/every entity |
ENTITY_TOP_LEVEL | Entity that is the top-level entity. This is also the entity present specified in _AtlasExportRequest_.| ENTITY_TOP_LEVEL | Entity that is the top-level entity. This is also the entity present specified in _AtlasExportRequest_.|
...@@ -43,7 +54,7 @@ HAS_VALUE | Entity attribute has value. | ...@@ -43,7 +54,7 @@ HAS_VALUE | Entity attribute has value. |
##### Actions ##### Actions
Action Type | Description | |**Action Type** | *Description** |
-------------------|----------------------------------------------| -------------------|----------------------------------------------|
ADD_CLASSIFICATION | Add classifiction | ADD_CLASSIFICATION | Add classifiction |
REPLACE_PREFIX | Replace value starting with another value. | REPLACE_PREFIX | Replace value starting with another value. |
...@@ -56,87 +67,91 @@ CLEAR | Clear value of an attribute | ...@@ -56,87 +67,91 @@ CLEAR | Clear value of an attribute |
###### Add Classification ###### Add Classification
During import, hive_db entity whose _qualifiedName_ is _stocks@cl1_ will get the classification _clSrcImported_. During import, hive_db entity whose _qualifiedName_ is _stocks@cl1_ will get the classification _clSrcImported_.
```json
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"conditions": { "conditions": {
"hive_db.qualifiedName": "stocks@cl1" "hive_db.qualifiedName": "stocks@cl1"
}, },
"action": { "action": {
"__entity": "ADD_CLASSIFICATION: clSrcImported" "__entity": "ADD_CLASSIFICATION: clSrcImported"
} }
} }`}
``` </SyntaxHighlighter>
Every imported entity will get the classification by simply changing the condition. The __entity is special condition which matches entity. Every imported entity will get the classification by simply changing the condition. The __entity is special condition which matches entity.
```json
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"conditions": { "conditions": {
"__entity": "" "__entity": ""
}, },
"action": { "action": {
"__entity": "ADD_CLASSIFICATION: clSrcImported" "__entity": "ADD_CLASSIFICATION: clSrcImported"
} }
} }`}
``` </SyntaxHighlighter>
To add classification to only the top-level entity (entity that is used as starting point for an export), use: To add classification to only the top-level entity (entity that is used as starting point for an export), use:
```json <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"conditions": { "conditions": {
"__entity": "topLevel:" "__entity": "topLevel:"
}, },
"action": { "action": {
"__entity": "ADD_CLASSIFICATION: clSrcImported" "__entity": "ADD_CLASSIFICATION: clSrcImported"
} }
} }`}
``` </SyntaxHighlighter>
###### Replace Prefix ###### Replace Prefix
This action works on string values. The first parameter is the prefix that is searched for a match, once matched, it is replaced with the provided replacement string. This action works on string values. The first parameter is the prefix that is searched for a match, once matched, it is replaced with the provided replacement string.
The sample below searches for _/aa/bb/_, once found replaces it with _/xx/yy/_. The sample below searches for _/aa/bb/_, once found replaces it with _/xx/yy/_.
```json
{ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{`{
"conditions": { "conditions": {
"hdfs_path.clusterName": "EQUALS: CL1" "hdfs_path.clusterName": "EQUALS: CL1"
}, },
"action": { "action": {
"hdfs_path.path": "REPLACE_PREFIX: = :/aa/bb/=/xx/yy/" "hdfs_path.path": "REPLACE_PREFIX: = :/aa/bb/=/xx/yy/"
} }
} }`}
``` </SyntaxHighlighter>
###### To Lower ###### To Lower
Entity whose hdfs_path.clusterName is CL1 will get its path attribute converted to lower case. Entity whose hdfs_path.clusterName is CL1 will get its path attribute converted to lower case.
```json <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"conditions": { "conditions": {
"hdfs_path.clusterName": "EQUALS: CL1" "hdfs_path.clusterName": "EQUALS: CL1"
}, },
"action": { "action": {
"hdfs_path.path": "TO_LOWER:" "hdfs_path.path": "TO_LOWER:"
} }
} }`}
``` </SyntaxHighlighter>
###### Clear ###### Clear
Entity whose hdfs_path.clusterName has value set, will get its _replicatedTo_ attribute value cleared. Entity whose hdfs_path.clusterName has value set, will get its _replicatedTo_ attribute value cleared.
```json <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"conditions": { "conditions": {
"hdfs_path.clusterName": "HAS_VALUE:" "hdfs_path.clusterName": "HAS_VALUE:"
}, },
"action": { "action": {
"hdfs_path.replicatedTo": "CLEAR:" "hdfs_path.replicatedTo": "CLEAR:"
} }
} }`}
``` </SyntaxHighlighter>
#### Additional Examples #### Additional Examples
......
---
name: Incremental Export
route: /Incremental-Export
menu: Documentation
submenu: Import/Export
---
import themen from 'theme/styles/styled-colors';
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
import SyntaxHighlighter from 'react-syntax-highlighter';
## Incremental Export ## Incremental Export
#### Background #### Background
...@@ -6,8 +17,8 @@ Incremental export allows for export of entities after a specified timestamp. Th ...@@ -6,8 +17,8 @@ Incremental export allows for export of entities after a specified timestamp. Th
#### Export Options #### Export Options
New _fetchType_ added to indicate incremental export. This option can be used with any _matchType_. When _fetchType_ is _incremental_, it is necessary to specify the _changeMarker_ option for incremental export to function, else full export will be performed. New _fetchType_ added to indicate incremental export. This option can be used with any _matchType_. When _fetchType_ is _incremental_, it is necessary to specify the _changeMarker_ option for incremental export to function, else full export will be performed.
```json <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
{ {`{
"itemsToExport": [ "itemsToExport": [
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks@cl1" } } { "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks@cl1" } }
], ],
...@@ -15,8 +26,8 @@ New _fetchType_ added to indicate incremental export. This option can be used wi ...@@ -15,8 +26,8 @@ New _fetchType_ added to indicate incremental export. This option can be used wi
"fetchType": "incremental", "fetchType": "incremental",
"changeMarker": 10000 "changeMarker": 10000
} }
} }`}
``` </SyntaxHighlighter>
#### Getting Change Marker #### Getting Change Marker
......
---+ Data Governance and Metadata framework for Hadoop ---
name: Introduction
route: /
menu: Introduction
---
# Introduction
---++ Overview Atlas is Data Governance and Metadata framework for Hadoop
## Overview
Atlas is a scalable and extensible set of core foundational governance services – enabling Atlas is a scalable and extensible set of core foundational governance services – enabling
enterprises to effectively and efficiently meet their compliance requirements within Hadoop and enterprises to effectively and efficiently meet their compliance requirements within Hadoop and
...@@ -11,79 +18,49 @@ Apache Atlas provides open metadata management and governance capabilities for o ...@@ -11,79 +18,49 @@ Apache Atlas provides open metadata management and governance capabilities for o
to build a catalog of their data assets, classify and govern these assets and provide collaboration to build a catalog of their data assets, classify and govern these assets and provide collaboration
capabilities around these data assets for data scientists, analysts and the data governance team. capabilities around these data assets for data scientists, analysts and the data governance team.
---++ Features
---+++ Metadata types & instances ## Features
### Metadata types & instances
* Pre-defined types for various Hadoop and non-Hadoop metadata * Pre-defined types for various Hadoop and non-Hadoop metadata
* Ability to define new types for the metadata to be managed * Ability to define new types for the metadata to be managed
* Types can have primitive attributes, complex attributes, object references; can inherit from other types * Types can have primitive attributes, complex attributes, object references; can inherit from other types
* Instances of types, called entities, capture metadata object details and their relationships * Instances of types, called entities, capture metadata object details and their relationships
* REST APIs to work with types and instances allow easier integration * REST APIs to work with types and instances allow easier integration
---+++ Classification ### Classification
* Ability to dynamically create classifications - like PII, EXPIRES_ON, DATA_QUALITY, SENSITIVE * Ability to dynamically create classifications - like PII, EXPIRES_ON, DATA_QUALITY, SENSITIVE
* Classifications can include attributes - like expiry_date attribute in EXPIRES_ON classification * Classifications can include attributes - like expiry_date attribute in EXPIRES_ON classification
* Entities can be associated with multiple classifications, enabling easier discovery and security enforcement * Entities can be associated with multiple classifications, enabling easier discovery and security enforcement
* Propagation of classifications via lineage - automatically ensures that classifications follow the data as it goes through various processing * Propagation of classifications via lineage - automatically ensures that classifications follow the data as it goes through various processing
---+++ Lineage ### Lineage
* Intuitive UI to view lineage of data as it moves through various processes * Intuitive UI to view lineage of data as it moves through various processes
* REST APIs to access and update lineage * REST APIs to access and update lineage
---+++ Search/Discovery ### Search/Discovery
* Intuitive UI to search entities by type, classification, attribute value or free-text * Intuitive UI to search entities by type, classification, attribute value or free-text
* Rich REST APIs to search by complex criteria * Rich REST APIs to search by complex criteria
* SQL like query language to search entities - Domain Specific Language (DSL) * SQL like query language to search entities - Domain Specific Language (DSL)
---+++ Security & Data Masking ### Security & Data Masking
* Fine grained security for metadata access, enabling controls on access to entity instances and operations like add/update/remove classifications * Fine grained security for metadata access, enabling controls on access to entity instances and operations like add/update/remove classifications
* Integration with Apache Ranger enables authorization/data-masking on data access based on classifications associated with entities in Apache Atlas. For example: * Integration with Apache Ranger enables authorization/data-masking on data access based on classifications associated with entities in Apache Atlas. For example:
* who can access data classified as PII, SENSITIVE * who can access data classified as PII, SENSITIVE
* customer-service users can only see last 4 digits of columns classified as NATIONAL_ID * customer-service users can only see last 4 digits of columns classified as NATIONAL_ID
---++ Getting Started ## Getting Started
* [[WhatsNew-2.0][What's new in Apache Atlas 2.0?]]
* [[InstallationSteps][Build & Install]]
* [[QuickStart][Quick Start]]
---++ Documentation
* [[Architecture][High Level Architecture]]
* [[TypeSystem][Type System]]
* [[Search - Basic][Search: Basic]]
* [[Search - Advanced][Search: Advanced]]
* [[Glossary][Glossary]]
* [[security][Security]]
* [[Atlas-Authentication][Authentication]]
* [[Atlas-Authorization-Model][Atlas Authorization Model]]
* [[Atlas-Authorization-Simple-Authorizer][Steps to configure Atlas Simple Authorizer]]
* [[Atlas-Authorization-Ranger-Authorizer][Steps to configure Atlas Ranger Authorizer]]
* [[ClassificationPropagation][Classification Propagation]]
* [[Configuration][Configuration]]
* [[Notifications][Notifications]]
* Hooks & Bridges
* [[Hook-HBase][HBase Hook & Bridge]]
* [[Hook-Hive][Hive Hook & Bridge]]
* [[Hook-Sqoop][Sqoop Hook]]
* [[Hook-Storm][Storm Hook]]
* [[Bridge-Kafka][Kafka Bridge]]
* [[HighAvailability][Fault Tolerance And High Availability Options]]
* [[Migration-0.8-to-1.0][Migration from Apache Atlas 0.8]]
* [[AtlasRepairIndex][Index repair tool]]
---++ API Documentation
* <a href="api/v2/index.html">REST API Documentation</a> * [What's new in Apache Atlas 2.0?](WhatsNew-2.0)
* [[Import-Export-API][Export & Import REST API Documentation]] * [Build & Install](InstallationSteps)
* <a href="../api/rest.html">Legacy API Documentation</a> * [Quick Start](QuickStart)
---++ Developer Setup Documentation ## API Documentation
* [[EclipseSetup][Developer Setup: Eclipse]]
#LicenseInfo * <a href="api/v2/index.html">REST API Documentation</a>
---+ Licensing Information * [Export & Import REST API Documentation](Import-Export-API)
* <a href="../api/rest.html">Legacy API Documentation</a>
Atlas is distributed under [[http://www.apache.org/licenses/][Apache License 2.0]]. ## Developer Setup Documentation
* [Developer Setup: Eclipse](EclipseSetup)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment