dlt.sources.rest_api

Generic API Source

rest_api

@decorators.source
def rest_api(
    client: ClientConfig = dlt.config.value,
    resources: List[Union[str, EndpointResource,
                          DltResource]] = dlt.config.value,
    resource_defaults: Optional[EndpointResourceBase] = None
) -> List[DltResource]

View source on GitHub

Creates and configures a REST API source with default settings

rest_api_source

def rest_api_source(config: RESTAPIConfig,
                    name: str = None,
                    section: str = None,
                    max_table_nesting: int = None,
                    root_key: bool = None,
                    schema: Schema = None,
                    schema_contract: TSchemaContract = None,
                    parallelized: bool = False) -> DltSource

View source on GitHub

Creates and configures a REST API source for data extraction.

Arguments:

• config RESTAPIConfig - Configuration for the REST API source.
• name str, optional - Name of the source.
• section str, optional - Section of the configuration file.
• max_table_nesting int, optional - Maximum depth of nested table above which the remaining nodes are loaded as structs or JSON.
• root_key bool, optional - Enables merging on all resources by propagating root foreign key to child tables. This option is most useful if you plan to change write disposition of a resource to disable/enable merge. Defaults to False.
• schema Schema, optional - An explicit Schema instance to be associated with the source. If not present, dlt creates a new Schema object with provided name. If such Schema already exists in the same folder as the module containing the decorated function, such schema will be loaded from file.
• schema_contract TSchemaContract, optional - Schema contract settings that will be applied to this resource.
• parallelized bool, optional - If True, resource generators will be extracted in parallel with other resources. Transformers that return items are also parallelized. Non-eligible resources are ignored. Defaults to False which preserves resource settings.

Returns:

• DltSource - A configured dlt source.

Example:

pokemon_source = rest_api_source({

• "client" - {
• "base_url" - "https://pokeapi.co/api/v2/",
• "paginator" - "json_link", },
• "resources" - [ {
• "name" - "pokemon",
• "endpoint" - {
• "path" - "pokemon",
• "params" - {
• "limit" - 100, }, },
• "primary_key" - "id", } ] })

rest_api_resources

def rest_api_resources(config: RESTAPIConfig) -> List[DltResource]

View source on GitHub

Creates a list of resources from a REST API configuration.

Arguments:

• config RESTAPIConfig - Configuration for the REST API source.

Returns:

• List[DltResource] - List of dlt resources.

Example:

github_source = rest_api_resources({

• "client" - {
• "base_url" - "https://api.github.com/repos/dlt-hub/dlt/",
• "auth" - {
• "token" - dlt.secrets["token"], }, },
• "resource_defaults" - {
• "primary_key" - "id",
• "write_disposition" - "merge",
• "endpoint" - {
• "params" - {
• "per_page" - 100, }, }, },
• "resources" - [ {
• "name" - "issues",
• "endpoint" - {
• "path" - "issues",
• "params" - {
• "sort" - "updated",
• "direction" - "desc",
• "state" - "open",
• "since" - "{incremental.start_value}", },
• "incremental" - {
• "cursor_path" - "updated_at",
• "initial_value" - "2024-01-25T11:21:28Z", }, }, }, {
• "name" - "issue_comments",
• "endpoint" - {
• "path" - "issues/{resources.issues.number}/comments", }, }, ], })