diff --git a/docs/oss/integrations/athena.mdx b/docs/oss/integrations/athena.mdx new file mode 100644 index 000000000..b015c3517 --- /dev/null +++ b/docs/oss/integrations/athena.mdx @@ -0,0 +1,19 @@ +--- +title: "Athena" +--- + +import AthenaCli from '/snippets/cli/athena-cli.mdx'; + + + + + + + + + +### Have a question? + +We are available +on [Slack](https://elementary-data.com/community), reach out +for any kind of help! diff --git a/docs/oss/integrations/duckdb.mdx b/docs/oss/integrations/duckdb.mdx new file mode 100644 index 000000000..97ccdb2cd --- /dev/null +++ b/docs/oss/integrations/duckdb.mdx @@ -0,0 +1,19 @@ +--- +title: "DuckDB" +--- + +import DuckdbCli from '/snippets/cli/duckdb-cli.mdx'; + + + + + + + + + +### Have a question? + +We are available +on [Slack](https://elementary-data.com/community), reach out +for any kind of help! diff --git a/docs/oss/integrations/spark.mdx b/docs/oss/integrations/spark.mdx new file mode 100644 index 000000000..f8e09dcd9 --- /dev/null +++ b/docs/oss/integrations/spark.mdx @@ -0,0 +1,19 @@ +--- +title: "Spark" +--- + +import SparkCli from '/snippets/cli/spark-cli.mdx'; + + + + + + + + + +### Have a question? + +We are available +on [Slack](https://elementary-data.com/community), reach out +for any kind of help! diff --git a/docs/oss/integrations/trino.mdx b/docs/oss/integrations/trino.mdx new file mode 100644 index 000000000..5ff62e7c6 --- /dev/null +++ b/docs/oss/integrations/trino.mdx @@ -0,0 +1,19 @@ +--- +title: "Trino" +--- + +import TrinoCli from '/snippets/cli/trino-cli.mdx'; + + + + + + + + + +### Have a question? + +We are available +on [Slack](https://elementary-data.com/community), reach out +for any kind of help! diff --git a/docs/snippets/cli/athena-cli.mdx b/docs/snippets/cli/athena-cli.mdx new file mode 100644 index 000000000..1a77eace7 --- /dev/null +++ b/docs/snippets/cli/athena-cli.mdx @@ -0,0 +1,33 @@ +### Athena connection profile + +After installing Elementary's dbt package upon running `dbt deps`, +you can generate Elementary's profile for usage with `edr` by running the following command within your project: + +```shell +dbt run-operation elementary.generate_elementary_cli_profile +``` + +The command will print to the terminal a partially filled template of the profile that's needed for `edr` to work. + +```yml Athena +## ATHENA ## +## By default, edr expects the profile name 'elementary'. ## +## Configure the database and schema of elementary models. ## +## Check where 'elementary_test_results' is to find it. ## + +elementary: + outputs: + default: + type: athena + work_group: [athena workgroup] + s3_staging_dir: [s3_staging_dir] # Location to store query results & metadata + s3_data_dir: [s3 data dir] # Location to store table data (if not specified, s3_staging_dir is used) + region_name: [aws region name] # AWS region, e.g. eu-west-1 + database: [database name] + schema: [schema name] # elementary schema, usually [schema name]_elementary + threads: [number of threads like 8] +``` + +We support the same format and connection methods as dbt. Please refer to +dbt's documentation of [Athena profile](https://docs.getdbt.com/reference/warehouse-setups/athena-setup) for +further details. diff --git a/docs/snippets/cli/dremio-cli.mdx b/docs/snippets/cli/dremio-cli.mdx new file mode 100644 index 000000000..a34a793bb --- /dev/null +++ b/docs/snippets/cli/dremio-cli.mdx @@ -0,0 +1,64 @@ +### Dremio connection profile + +After installing Elementary's dbt package upon running `dbt deps`, +you can generate Elementary's profile for usage with `edr` by running the following command within your project: + +```shell +dbt run-operation elementary.generate_elementary_cli_profile +``` + +The command will print to the terminal a partially filled template of the profile that's needed for `edr` to work. + + + + +```yml +## DREMIO CLOUD ## +## By default, edr expects the profile name 'elementary'. ## +## Configure the database and schema of elementary models. ## +## Check where 'elementary_test_results' is to find it. ## + +elementary: + outputs: + default: + type: dremio + cloud_host: api.dremio.cloud # or api.eu.dremio.cloud for EU + cloud_project_id: [project ID] + user: [email address] + pat: [personal access token] + use_ssl: true + object_storage_source: [name] # alias: datalake + object_storage_path: [path] # alias: root_path + dremio_space: [name] # alias: database + dremio_space_folder: [path] # alias: schema, usually [schema]_elementary + threads: [1 or more] +``` + + +```yml +## DREMIO SOFTWARE ## +## By default, edr expects the profile name 'elementary'. ## +## Configure the database and schema of elementary models. ## +## Check where 'elementary_test_results' is to find it. ## + +elementary: + outputs: + default: + type: dremio + software_host: [hostname or IP address] + port: 9047 + user: [username] + password: [password] # or use pat: [personal access token] + use_ssl: [true or false] + object_storage_source: [name] # alias: datalake + object_storage_path: [path] # alias: root_path + dremio_space: [name] # alias: database + dremio_space_folder: [path] # alias: schema, usually [schema]_elementary + threads: [1 or more] +``` + + + +We support the same format and connection methods as dbt. Please refer to +dbt's documentation of [Dremio profile](https://docs.getdbt.com/docs/core/connect-data-platform/dremio-setup) for +further details. diff --git a/docs/snippets/cli/duckdb-cli.mdx b/docs/snippets/cli/duckdb-cli.mdx new file mode 100644 index 000000000..0db80b0be --- /dev/null +++ b/docs/snippets/cli/duckdb-cli.mdx @@ -0,0 +1,29 @@ +### DuckDB connection profile + +After installing Elementary's dbt package upon running `dbt deps`, +you can generate Elementary's profile for usage with `edr` by running the following command within your project: + +```shell +dbt run-operation elementary.generate_elementary_cli_profile +``` + +The command will print to the terminal a partially filled template of the profile that's needed for `edr` to work. + +```yml DuckDB +## DUCKDB ## +## By default, edr expects the profile name 'elementary'. ## +## Configure the database and schema of elementary models. ## +## Check where 'elementary_test_results' is to find it. ## + +elementary: + outputs: + default: + type: duckdb + path: [path to your .duckdb file] + schema: [schema name] # elementary schema, usually [schema name]_elementary + threads: [1 or more] +``` + +We support the same format and connection methods as dbt. Please refer to +dbt's documentation of [DuckDB profile](https://docs.getdbt.com/reference/warehouse-setups/duckdb-setup) for +further details. diff --git a/docs/snippets/cli/spark-cli.mdx b/docs/snippets/cli/spark-cli.mdx new file mode 100644 index 000000000..af871ce9c --- /dev/null +++ b/docs/snippets/cli/spark-cli.mdx @@ -0,0 +1,33 @@ +### Spark connection profile + +After installing Elementary's dbt package upon running `dbt deps`, +you can generate Elementary's profile for usage with `edr` by running the following command within your project: + +```shell +dbt run-operation elementary.generate_elementary_cli_profile +``` + +The command will print to the terminal a partially filled template of the profile that's needed for `edr` to work. + +```yml Spark +## SPARK ## +## By default, edr expects the profile name 'elementary'. ## +## Configure the database and schema of elementary models. ## +## Check where 'elementary_test_results' is to find it. ## + +elementary: + outputs: + default: + type: spark + method: [thrift, http, or odbc] + host: [hostname] + port: [port] + user: [username] + schema: [schema name] # elementary schema, usually [schema name]_elementary + threads: [1 or more] + # token: [optional, used with http method] +``` + +We support the same format and connection methods as dbt. Please refer to +dbt's documentation of [Spark profile](https://docs.getdbt.com/reference/warehouse-setups/spark-setup) for +further details. diff --git a/docs/snippets/cli/trino-cli.mdx b/docs/snippets/cli/trino-cli.mdx new file mode 100644 index 000000000..82d9da22a --- /dev/null +++ b/docs/snippets/cli/trino-cli.mdx @@ -0,0 +1,35 @@ +### Trino connection profile + +After installing Elementary's dbt package upon running `dbt deps`, +you can generate Elementary's profile for usage with `edr` by running the following command within your project: + +```shell +dbt run-operation elementary.generate_elementary_cli_profile +``` + +The command will print to the terminal a partially filled template of the profile that's needed for `edr` to work. + +```yml Trino +## TRINO ## +## By default, edr expects the profile name 'elementary'. ## +## Configure the database and schema of elementary models. ## +## Check where 'elementary_test_results' is to find it. ## + +elementary: + outputs: + default: + type: trino + host: [hostname] + port: [port] + database: [database name] + schema: [schema name] # elementary schema, usually [schema name]_elementary + threads: [1 or more] + method: [authentication method] # ldap, oauth etc. + user: [username] + # password: [optional, used with ldap authentication ] + # session_properties: [optional, sets Trino session properties used in the connection] +``` + +We support the same format and connection methods as dbt. Please refer to +dbt's documentation of [Trino profile](https://docs.getdbt.com/reference/warehouse-setups/trino-setup) for +further details. diff --git a/docs/snippets/install-cli.mdx b/docs/snippets/install-cli.mdx index 550a69c1b..78259b8ab 100644 --- a/docs/snippets/install-cli.mdx +++ b/docs/snippets/install-cli.mdx @@ -16,6 +16,9 @@ pip install 'elementary-data[databricks]' pip install 'elementary-data[athena]' pip install 'elementary-data[trino]' pip install 'elementary-data[clickhouse]' +pip install 'elementary-data[duckdb]' +pip install 'elementary-data[dremio]' +pip install 'elementary-data[spark]' ## Postgres doesn't require this step ``` diff --git a/docs/snippets/oss/adapters-cards.mdx b/docs/snippets/oss/adapters-cards.mdx index b3031b163..4f12851a9 100644 --- a/docs/snippets/oss/adapters-cards.mdx +++ b/docs/snippets/oss/adapters-cards.mdx @@ -342,6 +342,116 @@ } > + + + + + + } + > + + + + + + + + + + + + + + + + + + + + + } + > + + + + + + } + >