Skip to content

DataChain

The DataChain class creates a data chain, which is a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data. See DataChain for examples of how to create a chain.

C module-attribute 

C = Column

Column 

Column(
    text, type_=None, is_literal=False, _selectable=None
)

Bases: ColumnClause

Source code in datachain/query/schema.py 
43
44
45
46
47
48
def __init__(self, text, type_=None, is_literal=False, _selectable=None):
    """Dataset column."""
    self.name = ColumnMeta.to_db_name(text)
    super().__init__(
        self.name, type_=type_, is_literal=is_literal, _selectable=_selectable
    )

glob 

glob(glob_str)

Search for matches using glob pattern matching.

Source code in datachain/query/schema.py 
53
54
55
def glob(self, glob_str):
    """Search for matches using glob pattern matching."""
    return self.op("GLOB")(glob_str)

regexp 

regexp(regexp_str)

Search for matches using regexp pattern matching.

Source code in datachain/query/schema.py 
57
58
59
def regexp(self, regexp_str):
    """Search for matches using regexp pattern matching."""
    return self.op("REGEXP")(regexp_str)

read_csv 

read_csv(
    path,
    delimiter: Optional[str] = None,
    header: bool = True,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows=None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column_types: Optional[
        dict[str, Union[str, DataType]]
    ] = None,
    parse_options: Optional[
        dict[str, Union[str, Union[bool, Callable]]]
    ] = None,
    **kwargs
) -> DataChain

Generate chain from csv files.

Parameters:

  • path –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • delimiter –

    Character for delimiting columns. Takes precedence if also specified in parse_options. Defaults to ",".

  • header –

    Whether the files include a header row.

  • output –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • column –

    Created column name.

  • model_name –

    Generated model name.

  • source –

    Whether to include info about the source file.

  • nrows –

    Optional row limit.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

  • column_types –

    Dictionary of column names and their corresponding types. It is passed to CSV reader and for each column specified type auto inference is disabled.

  • parse_options (Optional[dict[str, Union[str, Union[bool, Callable]]]], default: None ) –

    Tells the parser how to process lines. See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

Example

Reading a csv file: 

import datachain as dc
chain = dc.read_csv("s3://mybucket/file.csv")

Reading csv files from a directory as a combined dataset: 

import datachain as dc
chain = dc.read_csv("s3://mybucket/dir")

Source code in datachain/lib/dc/csv.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def read_csv(
    path,
    delimiter: Optional[str] = None,
    header: bool = True,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows=None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column_types: Optional[dict[str, "Union[str, ArrowDataType]"]] = None,
    parse_options: Optional[dict[str, "Union[str, Union[bool, Callable]]"]] = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from csv files.

    Parameters:
        path : Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        delimiter : Character for delimiting columns. Takes precedence if also
            specified in `parse_options`. Defaults to ",".
        header : Whether the files include a header row.
        output : Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        column : Created column name.
        model_name : Generated model name.
        source : Whether to include info about the source file.
        nrows : Optional row limit.
        session : Session to use for the chain.
        settings : Settings to use for the chain.
        column_types : Dictionary of column names and their corresponding types.
            It is passed to CSV reader and for each column specified type auto
            inference is disabled.
        parse_options: Tells the parser how to process lines.
            See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

    Example:
        Reading a csv file:
        ```py
        import datachain as dc
        chain = dc.read_csv("s3://mybucket/file.csv")
        ```

        Reading csv files from a directory as a combined dataset:
        ```py
        import datachain as dc
        chain = dc.read_csv("s3://mybucket/dir")
        ```
    """
    from pandas.io.parsers.readers import STR_NA_VALUES
    from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
    from pyarrow.dataset import CsvFileFormat
    from pyarrow.lib import type_for_alias

    from .storage import read_storage

    parse_options = parse_options or {}
    if "delimiter" not in parse_options:
        parse_options["delimiter"] = ","
    if delimiter:
        parse_options["delimiter"] = delimiter

    if column_types:
        column_types = {
            name: type_for_alias(typ) if isinstance(typ, str) else typ
            for name, typ in column_types.items()
        }
    else:
        column_types = {}

    chain = read_storage(path, session=session, settings=settings, **kwargs)

    column_names = None
    if not header:
        if not output:
            msg = "error parsing csv - provide output if no header"
            raise DatasetPrepareError(chain.name, msg)
        if isinstance(output, Sequence):
            column_names = output  # type: ignore[assignment]
        elif isinstance(output, dict):
            column_names = list(output.keys())
        elif (fr := ModelStore.to_pydantic(output)) is not None:
            column_names = list(fr.model_fields.keys())
        else:
            msg = f"error parsing csv - incompatible output type {type(output)}"
            raise DatasetPrepareError(chain.name, msg)

    parse_options = ParseOptions(**parse_options)
    read_options = ReadOptions(column_names=column_names)
    convert_options = ConvertOptions(
        strings_can_be_null=True,
        null_values=STR_NA_VALUES,
        column_types=column_types,
    )
    format = CsvFileFormat(
        parse_options=parse_options,
        read_options=read_options,
        convert_options=convert_options,
    )
    return chain.parse_tabular(
        output=output,
        column=column,
        model_name=model_name,
        source=source,
        nrows=nrows,
        format=format,
        parse_options=parse_options,
    )

read_dataset 

read_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[Union[str, int]] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    update: bool = False,
) -> DataChain

Get data from a saved Dataset. It returns the chain itself. If dataset or version is not found locally, it will try to pull it from Studio.

Parameters:

  • name (str) –

    The dataset name, which can be a fully qualified name including the namespace and project. Alternatively, it can be a regular name, in which case the explicitly defined namespace and project will be used if they are set; otherwise, default values will be applied.

  • namespace –

    optional name of namespace in which dataset to read is created

  • project –

    optional name of project in which dataset to read is created

  • version –

    dataset version. Supports: - Exact version strings: "1.2.3" - Legacy integer versions: 1, 2, 3 (finds latest major version) - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

  • delta (Optional[bool], default: False ) –

    If True, only process new or changed files instead of reprocessing everything. This saves time by skipping files that were already processed in previous versions. The optimization is working when a new version of the dataset is created. Default is False.

  • delta_on (Optional[Union[str, Sequence[str]]], default: ('file.path', 'file.etag', 'file.version') ) –

    Field(s) that uniquely identify each record in the source data. Used to detect which records are new or changed. Default is ("file.path", "file.etag", "file.version").

  • delta_result_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) in the result dataset that match delta_on fields. Only needed if you rename the identifying fields during processing. Default is None.

  • delta_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) used to detect if a record has changed. If not specified, all fields except delta_on fields are used. Default is None.

  • delta_retry (Optional[Union[bool, str]], default: None ) –

    Controls retry behavior for failed records: - String (field name): Reprocess records where this field is not empty (error mode) - True: Reprocess records missing from the result dataset (missing mode) - None: No retry processing (default)

  • update (bool, default: False ) –

    If True always checks for newer versions available on Studio, even if some version of the dataset exists locally already. If False (default), it will only fetch the dataset from Studio if it is not found locally.

Example 
import datachain as dc
chain = dc.read_dataset("my_cats")

import datachain as dc
chain = dc.read_dataset("dev.animals.my_cats")

chain = dc.read_dataset("my_cats", version="1.0.0")

# Using version specifiers (PEP 440)
chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")

# Legacy integer version support (finds latest in major version)
chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version

# Always check for newer versions matching a version specifier from Studio
chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)

session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
settings = {
    "cache": True,
    "parallel": 4,
    "workers": 4,
    "min_task_size": 1000,
    "prefetch": 10,
}
chain = dc.read_dataset(
    name="my_cats",
    version="1.0.0",
    session=session,
    settings=settings,
)
Source code in datachain/lib/dc/datasets.py 
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
def read_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[Union[str, int]] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[Union[str, Sequence[str]]] = None,
    delta_compare: Optional[Union[str, Sequence[str]]] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    update: bool = False,
) -> "DataChain":
    """Get data from a saved Dataset. It returns the chain itself.
    If dataset or version is not found locally, it will try to pull it from Studio.

    Parameters:
        name: The dataset name, which can be a fully qualified name including the
            namespace and project. Alternatively, it can be a regular name, in which
            case the explicitly defined namespace and project will be used if they are
            set; otherwise, default values will be applied.
        namespace : optional name of namespace in which dataset to read is created
        project : optional name of project in which dataset to read is created
        version : dataset version. Supports:
            - Exact version strings: "1.2.3"
            - Legacy integer versions: 1, 2, 3 (finds latest major version)
            - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
        session : Session to use for the chain.
        settings : Settings to use for the chain.
        delta: If True, only process new or changed files instead of reprocessing
            everything. This saves time by skipping files that were already processed in
            previous versions. The optimization is working when a new version of the
            dataset is created.
            Default is False.
        delta_on: Field(s) that uniquely identify each record in the source data.
            Used to detect which records are new or changed.
            Default is ("file.path", "file.etag", "file.version").
        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
            Only needed if you rename the identifying fields during processing.
            Default is None.
        delta_compare: Field(s) used to detect if a record has changed.
            If not specified, all fields except `delta_on` fields are used.
            Default is None.
        delta_retry: Controls retry behavior for failed records:
            - String (field name): Reprocess records where this field is not empty
              (error mode)
            - True: Reprocess records missing from the result dataset (missing mode)
            - None: No retry processing (default)
        update: If True always checks for newer versions available on Studio, even if
            some version of the dataset exists locally already. If False (default), it
            will only fetch the dataset from Studio if it is not found locally.


    Example:
        ```py
        import datachain as dc
        chain = dc.read_dataset("my_cats")
        ```

        ```py
        import datachain as dc
        chain = dc.read_dataset("dev.animals.my_cats")
        ```

        ```py
        chain = dc.read_dataset("my_cats", version="1.0.0")
        ```

        ```py
        # Using version specifiers (PEP 440)
        chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")
        ```

        ```py
        # Legacy integer version support (finds latest in major version)
        chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version
        ```

        ```py
        # Always check for newer versions matching a version specifier from Studio
        chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)
        ```

        ```py
        session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
        settings = {
            "cache": True,
            "parallel": 4,
            "workers": 4,
            "min_task_size": 1000,
            "prefetch": 10,
        }
        chain = dc.read_dataset(
            name="my_cats",
            version="1.0.0",
            session=session,
            settings=settings,
        )
        ```
    """
    from datachain.telemetry import telemetry

    from .datachain import DataChain

    telemetry.send_event_once("class", "datachain_init", name=name, version=version)

    session = Session.get(session)
    catalog = session.catalog

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        project_name=project,
        namespace_name=namespace,
    )

    if version is not None:
        dataset = session.catalog.get_dataset_with_remote_fallback(
            name, namespace_name, project_name, update=update
        )

        # Convert legacy integer versions to version specifiers
        # For backward compatibility we still allow users to put version as integer
        # in which case we convert it to a version specifier that finds the latest
        # version where major part is equal to that input version.
        # For example if user sets version=2, we convert it to ">=2.0.0,<3.0.0"
        # which will find something like 2.4.3 (assuming 2.4.3 is the biggest among
        # all 2.* dataset versions)
        if isinstance(version, int):
            version_spec = f">={version}.0.0,<{version + 1}.0.0"
        else:
            version_spec = str(version)

        from packaging.specifiers import InvalidSpecifier, SpecifierSet

        try:
            # Try to parse as version specifier
            SpecifierSet(version_spec)
            # If it's a valid specifier set, find the latest compatible version
            latest_compatible = dataset.latest_compatible_version(version_spec)
            if not latest_compatible:
                raise DatasetVersionNotFoundError(
                    f"No dataset {name} version matching specifier {version_spec}"
                )
            version = latest_compatible
        except InvalidSpecifier:
            # If not a valid specifier, treat as exact version string
            # This handles cases like "1.2.3" which are exact versions, not specifiers
            pass

    if settings:
        _settings = Settings(**settings)
    else:
        _settings = Settings()

    query = DatasetQuery(
        name=name,
        project_name=project_name,
        namespace_name=namespace_name,
        version=version,  #  type: ignore[arg-type]
        session=session,
        update=update,
    )

    signals_schema = SignalSchema({"sys": Sys})
    if query.feature_schema:
        signals_schema |= SignalSchema.deserialize(query.feature_schema)
    else:
        signals_schema |= SignalSchema.from_column_types(query.column_types or {})
    chain = DataChain(query, _settings, signals_schema)

    if delta:
        chain = chain._as_delta(
            on=delta_on,
            right_on=delta_result_on,
            compare=delta_compare,
            delta_retry=delta_retry,
        )

    return chain

datasets 

datasets(
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: Optional[str] = None,
    include_listing: bool = False,
    studio: bool = False,
    attrs: Optional[list[str]] = None,
) -> DataChain

Generate chain with list of registered datasets.

Parameters:

  • session (Optional[Session], default: None ) –

    Optional session instance. If not provided, uses default session.

  • settings (Optional[dict], default: None ) –

    Optional dictionary of settings to configure the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • column (Optional[str], default: None ) –

    Name of the output column in the chain. Defaults to None which means no top level column will be created.

  • include_listing (bool, default: False ) –

    If True, includes listing datasets. Defaults to False.

  • studio (bool, default: False ) –

    If True, returns datasets from Studio only, otherwise returns all local datasets. Defaults to False.

  • attrs (Optional[list[str]], default: None ) –

    Optional list of attributes to filter datasets on. It can be just attribute without value e.g "NLP", or attribute with value e.g "location=US". Attribute with value can also accept "" to target all that have specific name e.g "location="

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance containing dataset information.

Example 
import datachain as dc

chain = dc.datasets(column="dataset")
for ds in chain.to_iter("dataset"):
    print(f"{ds.name}@v{ds.version}")
Source code in datachain/lib/dc/datasets.py 
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
def datasets(
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: Optional[str] = None,
    include_listing: bool = False,
    studio: bool = False,
    attrs: Optional[list[str]] = None,
) -> "DataChain":
    """Generate chain with list of registered datasets.

    Args:
        session: Optional session instance. If not provided, uses default session.
        settings: Optional dictionary of settings to configure the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        column: Name of the output column in the chain. Defaults to None which
            means no top level column will be created.
        include_listing: If True, includes listing datasets. Defaults to False.
        studio: If True, returns datasets from Studio only,
            otherwise returns all local datasets. Defaults to False.
        attrs: Optional list of attributes to filter datasets on. It can be just
            attribute without value e.g "NLP", or attribute with value
            e.g "location=US". Attribute with value can also accept "*" to target
            all that have specific name e.g "location=*"

    Returns:
        DataChain: A new DataChain instance containing dataset information.

    Example:
        ```py
        import datachain as dc

        chain = dc.datasets(column="dataset")
        for ds in chain.to_iter("dataset"):
            print(f"{ds.name}@v{ds.version}")
        ```
    """

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    datasets_values = [
        DatasetInfo.from_models(d, v, j)
        for d, v, j in catalog.list_datasets_versions(
            include_listing=include_listing, studio=studio
        )
    ]
    datasets_values = [d for d in datasets_values if not d.is_temp]

    if attrs:
        for attr in attrs:
            datasets_values = [d for d in datasets_values if d.has_attr(attr)]

    if not column:
        # flattening dataset fields
        schema = {
            k: get_origin(v) if get_origin(v) is dict else v
            for k, v in get_type_hints(DatasetInfo).items()
            if k in DatasetInfo.model_fields
        }
        data = {k: [] for k in DatasetInfo.model_fields}  # type: ignore[var-annotated]
        for d in [d.model_dump() for d in datasets_values]:
            for field, value in d.items():
                data[field].append(value)

        return read_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            output=schema,
            **data,  # type: ignore[arg-type]
        )

    return read_values(
        session=session,
        settings=settings,
        in_memory=in_memory,
        output={column: DatasetInfo},
        **{column: datasets_values},  # type: ignore[arg-type]
    )

delete_dataset 

delete_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[str] = None,
    force: Optional[bool] = False,
    studio: Optional[bool] = False,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None

Removes specific dataset version or all dataset versions, depending on a force flag.

Parameters:

  • name (str) –

    The dataset name, which can be a fully qualified name including the namespace and project. Alternatively, it can be a regular name, in which case the explicitly defined namespace and project will be used if they are set; otherwise, default values will be applied.

  • namespace –

    optional name of namespace in which dataset to delete is created

  • project –

    optional name of project in which dataset to delete is created

  • version –

    Optional dataset version

  • force (Optional[bool], default: False ) –

    If true, all datasets versions will be removed. Defaults to False.

  • studio (Optional[bool], default: False ) –

    If True, removes dataset from Studio only, otherwise removes local dataset. Defaults to False.

  • session (Optional[Session], default: None ) –

    Optional session instance. If not provided, uses default session.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

Returns: None

Example 
import datachain as dc
dc.delete_dataset("cats")

import datachain as dc
dc.delete_dataset("cats", version="1.0.0")
Source code in datachain/lib/dc/datasets.py 
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
def delete_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[str] = None,
    force: Optional[bool] = False,
    studio: Optional[bool] = False,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None:
    """Removes specific dataset version or all dataset versions, depending on
    a force flag.

    Args:
        name: The dataset name, which can be a fully qualified name including the
            namespace and project. Alternatively, it can be a regular name, in which
            case the explicitly defined namespace and project will be used if they are
            set; otherwise, default values will be applied.
        namespace : optional name of namespace in which dataset to delete is created
        project : optional name of project in which dataset to delete is created
        version : Optional dataset version
        force: If true, all datasets versions will be removed. Defaults to False.
        studio: If True, removes dataset from Studio only, otherwise removes local
            dataset. Defaults to False.
        session: Optional session instance. If not provided, uses default session.
        in_memory: If True, creates an in-memory session. Defaults to False.

    Returns: None

    Example:
        ```py
        import datachain as dc
        dc.delete_dataset("cats")
        ```

        ```py
        import datachain as dc
        dc.delete_dataset("cats", version="1.0.0")
        ```
    """
    from datachain.studio import remove_studio_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        project_name=project,
        namespace_name=namespace,
    )

    if not catalog.metastore.is_local_dataset(namespace_name) and studio:
        return remove_studio_dataset(
            None, name, namespace_name, project_name, version=version, force=force
        )

    try:
        ds_project = get_project(project_name, namespace_name, session=session)
    except ProjectNotFoundError:
        raise DatasetNotFoundError(
            f"Dataset {name} not found in namespace {namespace_name} and project",
            f" {project_name}",
        ) from None

    if not force:
        version = version or catalog.get_dataset(name, ds_project).latest_version
    else:
        version = None
    catalog.remove_dataset(name, ds_project, version=version, force=force)

move_dataset 

move_dataset(
    src: str,
    dest: str,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None

Moves an entire dataset between namespaces and projects.

Parameters:

  • src (str) –

    The source dataset name. This can be a fully qualified name that includes the namespace and project, or a regular name. If a regular name is used, default values will be applied. The source dataset will no longer exist after the move.

  • dest (str) –

    The destination dataset name. This can also be a fully qualified name with a namespace and project, or just a regular name (default values will be used in that case). The original dataset will be moved here.

  • session (Optional[Session], default: None ) –

    An optional session instance. If not provided, the default session will be used.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

Returns:

  • None –

    None

Examples:

import datachain as dc
dc.move_dataset("cats", "new_cats")

import datachain as dc
dc.move_dataset("dev.animals.cats", "prod.animals.cats")
Source code in datachain/lib/dc/datasets.py 
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
def move_dataset(
    src: str,
    dest: str,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None:
    """Moves an entire dataset between namespaces and projects.

    Args:
        src: The source dataset name. This can be a fully qualified name that includes
            the namespace and project, or a regular name. If a regular name is used,
            default values will be applied. The source dataset will no longer exist
            after the move.
        dest: The destination dataset name. This can also be a fully qualified
            name with a namespace and project, or just a regular name (default values
            will be used in that case). The original dataset will be moved here.
        session: An optional session instance. If not provided, the default session
            will be used.
        in_memory: If True, creates an in-memory session. Defaults to False.

    Returns:
        None

    Examples:
        ```python
        import datachain as dc
        dc.move_dataset("cats", "new_cats")
        ```

        ```python
        import datachain as dc
        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    namespace, project, name = catalog.get_full_dataset_name(src)
    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)

    dataset = catalog.get_dataset(
        name, catalog.metastore.get_project(project, namespace)
    )

    catalog.update_dataset(
        dataset,
        name=dest_name,
        project_id=catalog.metastore.get_project(
            dest_project,
            dest_namespace,
            create=catalog.metastore.project_allowed_to_create,
        ).id,
    )

read_hf 

read_hf(
    dataset: Union[str, HFDatasetType],
    *args,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column: str = "",
    model_name: str = "",
    **kwargs
) -> DataChain

Generate chain from huggingface hub dataset.

Parameters:

  • dataset –

    Path or name of the dataset to read from Hugging Face Hub, or an instance of datasets.Dataset-like object.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

  • column –

    Generated object column name.

  • model_name –

    Generated model name.

  • kwargs –

    Parameters to pass to datasets.load_dataset.

Example

Load from Hugging Face Hub: 

import datachain as dc
chain = dc.read_hf("beans", split="train")

Generate chain from loaded dataset: 

from datasets import load_dataset
ds = load_dataset("beans", split="train")
import datachain as dc
chain = dc.read_hf(ds)

Source code in datachain/lib/dc/hf.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def read_hf(
    dataset: Union[str, "HFDatasetType"],
    *args,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column: str = "",
    model_name: str = "",
    **kwargs,
) -> "DataChain":
    """Generate chain from huggingface hub dataset.

    Parameters:
        dataset : Path or name of the dataset to read from Hugging Face Hub,
            or an instance of `datasets.Dataset`-like object.
        session : Session to use for the chain.
        settings : Settings to use for the chain.
        column : Generated object column name.
        model_name : Generated model name.
        kwargs : Parameters to pass to datasets.load_dataset.

    Example:
        Load from Hugging Face Hub:
        ```py
        import datachain as dc
        chain = dc.read_hf("beans", split="train")
        ```

        Generate chain from loaded dataset:
        ```py
        from datasets import load_dataset
        ds = load_dataset("beans", split="train")
        import datachain as dc
        chain = dc.read_hf(ds)
        ```
    """
    from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits

    from .values import read_values

    output: dict[str, DataType] = {}
    ds_dict = stream_splits(dataset, *args, **kwargs)
    if len(ds_dict) > 1:
        output = {"split": str}

    model_name = model_name or column or ""
    hf_features = next(iter(ds_dict.values())).features
    output = output | get_output_schema(hf_features)
    model = dict_to_data_model(model_name, output)
    if column:
        output = {column: model}

    chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
    return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)

read_json 

read_json(
    path: Union[str, PathLike[str]],
    type: FileType = "text",
    spec: Optional[DataType] = None,
    schema_from: Optional[str] = "auto",
    jmespath: Optional[str] = None,
    column: Optional[str] = "",
    model_name: Optional[str] = None,
    format: Optional[str] = "json",
    nrows=None,
    **kwargs
) -> DataChain

Get data from JSON. It returns the chain itself.

Parameters:

  • path –

    storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///"

  • type –

    read file as "binary", "text", or "image" data. Default is "text".

  • spec –

    optional Data Model

  • schema_from –

    path to sample to infer spec (if schema not provided)

  • column –

    generated column name

  • model_name –

    optional generated model name

  • format (Optional[str], default: 'json' ) –

    "json", "jsonl"

  • jmespath –

    optional JMESPATH expression to reduce JSON

  • nrows –

    optional row limit for jsonl and JSON arrays

Example

infer JSON schema from data, reduce using JMESPATH 

import datachain as dc
chain = dc.read_json("gs://json", jmespath="key1.key2")

infer JSON schema from a particular path 

import datachain as dc
chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")

Source code in datachain/lib/dc/json.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def read_json(
    path: Union[str, os.PathLike[str]],
    type: FileType = "text",
    spec: Optional[DataType] = None,
    schema_from: Optional[str] = "auto",
    jmespath: Optional[str] = None,
    column: Optional[str] = "",
    model_name: Optional[str] = None,
    format: Optional[str] = "json",
    nrows=None,
    **kwargs,
) -> "DataChain":
    """Get data from JSON. It returns the chain itself.

    Parameters:
        path : storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///"
        type : read file as "binary", "text", or "image" data. Default is "text".
        spec : optional Data Model
        schema_from : path to sample to infer spec (if schema not provided)
        column : generated column name
        model_name : optional generated model name
        format: "json", "jsonl"
        jmespath : optional JMESPATH expression to reduce JSON
        nrows : optional row limit for jsonl and JSON arrays

    Example:
        infer JSON schema from data, reduce using JMESPATH
        ```py
        import datachain as dc
        chain = dc.read_json("gs://json", jmespath="key1.key2")
        ```

        infer JSON schema from a particular path
        ```py
        import datachain as dc
        chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")
        ```
    """
    from .storage import read_storage

    if schema_from == "auto":
        schema_from = os.fspath(path)

    def jmespath_to_name(s: str):
        name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
        return s[:name_end]

    if (not column) and jmespath:
        column = jmespath_to_name(jmespath)
    if not column:
        column = format
    chain = read_storage(uri=path, type=type, **kwargs)
    signal_dict = {
        column: meta_formats.read_meta(
            schema_from=schema_from,
            format=format,
            spec=spec,
            model_name=model_name,
            jmespath=jmespath,
            nrows=nrows,
        ),
        "params": {"file": File},
    }
    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}

    cloudpickle.register_pickle_by_value(meta_formats)

    return chain.settings(**settings).gen(**signal_dict)  # type: ignore[misc, arg-type]

listings 

listings(
    session: Optional[Session] = None,
    in_memory: bool = False,
    column: str = "listing",
    **kwargs
) -> DataChain

Generate chain with list of cached listings. Listing is a special kind of dataset which has directory listing data of some underlying storage (e.g S3 bucket).

Example 
import datachain as dc
dc.listings().show()
Source code in datachain/lib/dc/listings.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def listings(
    session: Optional[Session] = None,
    in_memory: bool = False,
    column: str = "listing",
    **kwargs,
) -> "DataChain":
    """Generate chain with list of cached listings.
    Listing is a special kind of dataset which has directory listing data of
    some underlying storage (e.g S3 bucket).

    Example:
        ```py
        import datachain as dc
        dc.listings().show()
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = kwargs.get("catalog") or session.catalog

    return read_values(
        session=session,
        in_memory=in_memory,
        output={column: ListingInfo},
        **{column: catalog.listings()},  # type: ignore[arg-type]
    )

read_pandas 

read_pandas(
    df: DataFrame,
    name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: str = "",
) -> DataChain

Generate chain from pandas data-frame.

Example 
import pandas as pd
import datachain as dc

df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
dc.read_pandas(df)
Source code in datachain/lib/dc/pandas.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def read_pandas(  # type: ignore[override]
    df: "pd.DataFrame",
    name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: str = "",
) -> "DataChain":
    """Generate chain from pandas data-frame.

    Example:
        ```py
        import pandas as pd
        import datachain as dc

        df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
        dc.read_pandas(df)
        ```
    """
    from .utils import DatasetPrepareError

    def get_col_name(col):
        if isinstance(col, tuple):
            # Join tuple elements with underscore for MultiIndex columns
            return "_".join(map(str, col)).lower()
        # Handle regular string column names
        return str(col).lower()

    fr_map = {get_col_name(col): df[col].tolist() for col in df.columns}

    for c in fr_map:
        if not c.isidentifier():
            raise DatasetPrepareError(
                name,
                f"import from pandas error - '{c}' cannot be a column name",
            )

    return read_values(
        name,
        session,
        settings=settings,
        column=column,
        in_memory=in_memory,
        **fr_map,
    )

read_parquet 

read_parquet(
    path,
    partitioning: Any = "hive",
    output: Optional[dict[str, DataType]] = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    **kwargs
) -> DataChain

Generate chain from parquet files.

Parameters:

  • path –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • partitioning –

    Any pyarrow partitioning schema.

  • output –

    Dictionary defining column names and their corresponding types.

  • column –

    Created column name.

  • model_name –

    Generated model name.

  • source –

    Whether to include info about the source file.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

Example

Reading a single file: 

import datachain as dc
dc.read_parquet("s3://mybucket/file.parquet")

Reading a partitioned dataset from a directory: 

import datachain as dc
dc.read_parquet("s3://mybucket/dir")

Source code in datachain/lib/dc/parquet.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def read_parquet(
    path,
    partitioning: Any = "hive",
    output: Optional[dict[str, DataType]] = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from parquet files.

    Parameters:
        path : Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        partitioning : Any pyarrow partitioning schema.
        output : Dictionary defining column names and their corresponding types.
        column : Created column name.
        model_name : Generated model name.
        source : Whether to include info about the source file.
        session : Session to use for the chain.
        settings : Settings to use for the chain.

    Example:
        Reading a single file:
        ```py
        import datachain as dc
        dc.read_parquet("s3://mybucket/file.parquet")
        ```

        Reading a partitioned dataset from a directory:
        ```py
        import datachain as dc
        dc.read_parquet("s3://mybucket/dir")
        ```
    """
    from .storage import read_storage

    chain = read_storage(path, session=session, settings=settings, **kwargs)
    return chain.parse_tabular(
        output=output,
        column=column,
        model_name=model_name,
        source=source,
        format="parquet",
        partitioning=partitioning,
    )

read_records 

read_records(
    to_insert: Optional[Union[dict, Iterable[dict]]],
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    schema: Optional[dict[str, DataType]] = None,
) -> DataChain

Create a DataChain from the provided records. This method can be used for programmatically generating a chain in contrast of reading data from storages or other sources.

Parameters:

  • to_insert –

    records (or a single record) to insert. Each record is a dictionary of signals and theirs values.

  • schema –

    describes chain signals and their corresponding types

Example 
import datachain as dc
single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
Notes

This call blocks until all records are inserted.

Source code in datachain/lib/dc/records.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def read_records(
    to_insert: Optional[Union[dict, Iterable[dict]]],
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    schema: Optional[dict[str, DataType]] = None,
) -> "DataChain":
    """Create a DataChain from the provided records. This method can be used for
    programmatically generating a chain in contrast of reading data from storages
    or other sources.

    Parameters:
        to_insert : records (or a single record) to insert. Each record is
                    a dictionary of signals and theirs values.
        schema : describes chain signals and their corresponding types

    Example:
        ```py
        import datachain as dc
        single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
        ```

    Notes:
        This call blocks until all records are inserted.
    """
    from datachain.query.dataset import INSERT_BATCH_SIZE, adjust_outputs, get_col_types
    from datachain.sql.types import SQLType
    from datachain.utils import batched

    from .datasets import read_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    name = session.generate_temp_dataset_name()
    signal_schema = None
    columns: list[sqlalchemy.Column] = []

    if schema:
        signal_schema = SignalSchema(schema)
        columns = [
            sqlalchemy.Column(c.name, c.type)  # type: ignore[union-attr]
            for c in signal_schema.db_signals(as_columns=True)
        ]
    else:
        columns = [
            sqlalchemy.Column(name, typ)
            for name, typ in File._datachain_column_types.items()
        ]

    dsr = catalog.create_dataset(
        name,
        catalog.metastore.default_project,
        columns=columns,
        feature_schema=(
            signal_schema.clone_without_sys_signals().serialize()
            if signal_schema
            else None
        ),
    )

    session.add_dataset_version(dsr, dsr.latest_version)

    if isinstance(to_insert, dict):
        to_insert = [to_insert]
    elif not to_insert:
        to_insert = []

    warehouse = catalog.warehouse
    dr = warehouse.dataset_rows(dsr)
    table = dr.get_table()

    # Optimization: Compute row types once, rather than for every row.
    col_types = get_col_types(
        warehouse,
        {c.name: c.type for c in columns if isinstance(c.type, SQLType)},
    )
    records = (adjust_outputs(warehouse, record, col_types) for record in to_insert)
    for chunk in batched(records, INSERT_BATCH_SIZE):
        warehouse.insert_rows(table, chunk)
    warehouse.insert_rows_done(table)
    return read_dataset(name=dsr.full_name, session=session, settings=settings)

read_storage 

read_storage(
    uri: Union[
        str, PathLike[str], list[str], list[PathLike[str]]
    ],
    *,
    type: FileType = "binary",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    recursive: Optional[bool] = True,
    column: str = "file",
    update: bool = False,
    anon: bool = False,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    client_config: Optional[dict] = None
) -> DataChain

Get data from storage(s) as a list of file with all file attributes. It returns the chain itself as usual.

Parameters:

  • uri –

    storage URI with directory or list of URIs. URIs must start with storage prefix such as s3://, gs://, az:// or "file:///"

  • type –

    read file as "binary", "text", or "image" data. Default is "binary".

  • recursive –

    search recursively for the given path.

  • column –

    Created column name.

  • update –

    force storage reindexing. Default is False.

  • anon –

    If True, we will treat cloud bucket as public one

  • client_config –

    Optional client configuration for the storage client.

  • delta (Optional[bool], default: False ) –

    If True, only process new or changed files instead of reprocessing everything. This saves time by skipping files that were already processed in previous versions. The optimization is working when a new version of the dataset is created. Default is False.

  • delta_on (Optional[Union[str, Sequence[str]]], default: ('file.path', 'file.etag', 'file.version') ) –

    Field(s) that uniquely identify each record in the source data. Used to detect which records are new or changed. Default is ("file.path", "file.etag", "file.version").

  • delta_result_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) in the result dataset that match delta_on fields. Only needed if you rename the identifying fields during processing. Default is None.

  • delta_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) used to detect if a record has changed. If not specified, all fields except delta_on fields are used. Default is None.

  • delta_retry (Optional[Union[bool, str]], default: None ) –

    Controls retry behavior for failed records: - String (field name): Reprocess records where this field is not empty (error mode) - True: Reprocess records missing from the result dataset (missing mode) - None: No retry processing (default)

Returns:

  • DataChain ( DataChain ) –

    A DataChain object containing the file information.

Examples:

Simple call from s3: 

import datachain as dc
chain = dc.read_storage("s3://my-bucket/my-dir")

Multiple URIs: 

chain = dc.read_storage([
    "s3://bucket1/dir1",
    "s3://bucket2/dir2"
])

With AWS S3-compatible storage: 

chain = dc.read_storage(
    "s3://my-bucket/my-dir",
    client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
)

Pass existing session 

session = Session.get()
chain = dc.read_storage([
    "path/to/dir1",
    "path/to/dir2"
], session=session, recursive=True)

Note

When using multiple URIs with update=True, the function optimizes by avoiding redundant updates for URIs pointing to the same storage location.

Source code in datachain/lib/dc/storage.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def read_storage(
    uri: Union[str, os.PathLike[str], list[str], list[os.PathLike[str]]],
    *,
    type: FileType = "binary",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    recursive: Optional[bool] = True,
    column: str = "file",
    update: bool = False,
    anon: bool = False,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[Union[str, Sequence[str]]] = None,
    delta_compare: Optional[Union[str, Sequence[str]]] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    client_config: Optional[dict] = None,
) -> "DataChain":
    """Get data from storage(s) as a list of file with all file attributes.
    It returns the chain itself as usual.

    Parameters:
        uri : storage URI with directory or list of URIs.
            URIs must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///"
        type : read file as "binary", "text", or "image" data. Default is "binary".
        recursive : search recursively for the given path.
        column : Created column name.
        update : force storage reindexing. Default is False.
        anon : If True, we will treat cloud bucket as public one
        client_config : Optional client configuration for the storage client.
        delta: If True, only process new or changed files instead of reprocessing
            everything. This saves time by skipping files that were already processed in
            previous versions. The optimization is working when a new version of the
            dataset is created.
            Default is False.
        delta_on: Field(s) that uniquely identify each record in the source data.
            Used to detect which records are new or changed.
            Default is ("file.path", "file.etag", "file.version").
        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
            Only needed if you rename the identifying fields during processing.
            Default is None.
        delta_compare: Field(s) used to detect if a record has changed.
            If not specified, all fields except `delta_on` fields are used.
            Default is None.
        delta_retry: Controls retry behavior for failed records:
            - String (field name): Reprocess records where this field is not empty
              (error mode)
            - True: Reprocess records missing from the result dataset (missing mode)
            - None: No retry processing (default)

    Returns:
        DataChain: A DataChain object containing the file information.

    Examples:
        Simple call from s3:
        ```python
        import datachain as dc
        chain = dc.read_storage("s3://my-bucket/my-dir")
        ```

        Multiple URIs:
        ```python
        chain = dc.read_storage([
            "s3://bucket1/dir1",
            "s3://bucket2/dir2"
        ])
        ```

        With AWS S3-compatible storage:
        ```python
        chain = dc.read_storage(
            "s3://my-bucket/my-dir",
            client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
        )
        ```

        Pass existing session
        ```py
        session = Session.get()
        chain = dc.read_storage([
            "path/to/dir1",
            "path/to/dir2"
        ], session=session, recursive=True)
        ```

    Note:
        When using multiple URIs with `update=True`, the function optimizes by
        avoiding redundant updates for URIs pointing to the same storage location.
    """
    from .datachain import DataChain
    from .datasets import read_dataset
    from .records import read_records
    from .values import read_values

    file_type = get_file_type(type)

    if anon:
        client_config = (client_config or {}) | {"anon": True}
    session = Session.get(session, client_config=client_config, in_memory=in_memory)
    catalog = session.catalog
    cache = catalog.cache
    client_config = session.catalog.client_config
    listing_namespace_name = catalog.metastore.system_namespace_name
    listing_project_name = catalog.metastore.listing_project_name

    uris = uri if isinstance(uri, (list, tuple)) else [uri]

    if not uris:
        raise ValueError("No URIs provided")

    chains = []
    listed_ds_name = set()
    file_values = []

    for single_uri in uris:
        list_ds_name, list_uri, list_path, list_ds_exists = get_listing(
            single_uri, session, update=update
        )

        # list_ds_name is None if object is a file, we don't want to use cache
        # or do listing in that case - just read that single object
        if not list_ds_name:
            file_values.append(
                get_file_info(list_uri, cache, client_config=client_config)
            )
            continue

        dc = read_dataset(
            list_ds_name,
            namespace=listing_namespace_name,
            project=listing_project_name,
            session=session,
            settings=settings,
        )
        dc._query.update = update
        dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})

        if update or not list_ds_exists:

            def lst_fn(ds_name, lst_uri):
                # disable prefetch for listing, as it pre-downloads all files
                (
                    read_records(
                        DataChain.DEFAULT_FILE_RECORD,
                        session=session,
                        settings=settings,
                        in_memory=in_memory,
                    )
                    .settings(
                        prefetch=0,
                        namespace=listing_namespace_name,
                        project=listing_project_name,
                    )
                    .gen(
                        list_bucket(lst_uri, cache, client_config=client_config),
                        output={f"{column}": file_type},
                    )
                    # for internal listing datasets, we always bump major version
                    .save(ds_name, listing=True, update_version="major")
                )

            dc._query.set_listing_fn(
                lambda ds_name=list_ds_name, lst_uri=list_uri: lst_fn(ds_name, lst_uri)
            )

        chains.append(ls(dc, list_path, recursive=recursive, column=column))
        listed_ds_name.add(list_ds_name)

    storage_chain = None if not chains else reduce(lambda x, y: x.union(y), chains)

    if file_values:
        file_chain = read_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            file=file_values,
        )
        file_chain.signals_schema = file_chain.signals_schema.mutate(
            {f"{column}": file_type}
        )
        storage_chain = storage_chain.union(file_chain) if storage_chain else file_chain

    assert storage_chain is not None

    if delta:
        storage_chain = storage_chain._as_delta(
            on=delta_on,
            right_on=delta_result_on,
            compare=delta_compare,
            delta_retry=delta_retry,
        )

    return storage_chain

read_values 

read_values(
    ds_name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    output: OutputType = None,
    column: str = "",
    **fr_map
) -> DataChain

Generate chain from list of values.

Example 
import datachain as dc
dc.read_values(fib=[1, 2, 3, 5, 8])
Source code in datachain/lib/dc/values.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def read_values(
    ds_name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    output: OutputType = None,
    column: str = "",
    **fr_map,
) -> "DataChain":
    """Generate chain from list of values.

    Example:
        ```py
        import datachain as dc
        dc.read_values(fib=[1, 2, 3, 5, 8])
        ```
    """
    from .datachain import DataChain

    tuple_type, output, tuples = values_to_tuples(ds_name, output, **fr_map)

    def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
        yield from tuples

    _func_fr.__name__ = "read_values"

    chain = read_records(
        DataChain.DEFAULT_FILE_RECORD,
        session=session,
        settings=settings,
        in_memory=in_memory,
    )
    if column:
        output = {column: dict_to_data_model(column, output)}  # type: ignore[arg-type]
    return chain.gen(_func_fr, output=output)

read_database 

read_database(
    query: Union[str, Executable],
    connection: ConnectionType,
    params: Union[
        Sequence[Mapping[str, Any]], Mapping[str, Any], None
    ] = None,
    *,
    output: Optional[dict[str, DataType]] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    infer_schema_length: Optional[int] = 100
) -> DataChain

Read the results of a SQL query into a DataChain, using a given database connection.

Parameters:

  • query (Union[str, Executable]) –

    The SQL query to execute. Can be a raw SQL string or a SQLAlchemy Executable object.

  • connection (ConnectionType) –

    SQLAlchemy connectable, str, or a sqlite3 connection Using SQLAlchemy makes it possible to use any DB supported by that library. If a DBAPI2 object, only sqlite3 is supported. The user is responsible for engine disposal and connection closure for the SQLAlchemy connectable; str connections are closed automatically.

  • params (Union[Sequence[Mapping[str, Any]], Mapping[str, Any], None], default: None ) –

    Parameters to pass to execute method.

  • output (Optional[dict[str, DataType]], default: None ) –

    A dictionary mapping column names to types, used to override the schema inferred from the query results.

  • session (Optional[Session], default: None ) –

    Session to use for the chain.

  • settings (Optional[dict], default: None ) –

    Settings to use for the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • infer_schema_length (Optional[int], default: 100 ) –

    The maximum number of rows to scan for inferring schema. If set to None, the full data may be scanned. The rows used for schema inference are stored in memory, so large values can lead to high memory usage. Only applies if the output parameter is not set for the given column.

Examples:

Reading from a SQL query against a user-supplied connection: 

query = "SELECT key, value FROM tbl"
chain = dc.read_database(query, connection, output={"value": float})

Load data from a SQLAlchemy driver/engine: 

from sqlalchemy import create_engine
engine = create_engine("postgresql+psycopg://myuser:mypassword@localhost:5432/mydb")
chain = dc.read_database("select * from tbl", engine)

Load data from a parameterized SQLAlchemy query: 

query = "SELECT key, value FROM tbl WHERE value > :value"
dc.read_database(query, engine, params={"value": 50})

Notes
  • This function works with a variety of databases — including, but not limited to, SQLite, DuckDB, PostgreSQL, and Snowflake, provided the appropriate driver is installed.
  • This call is blocking, and will execute the query and return once the results are saved.
Source code in datachain/lib/dc/database.py 
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
def read_database(
    query: Union[str, "sqlalchemy.sql.expression.Executable"],
    connection: "ConnectionType",
    params: Union["Sequence[Mapping[str, Any]]", "Mapping[str, Any]", None] = None,
    *,
    output: Optional["dict[str, DataType]"] = None,
    session: Optional["Session"] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    infer_schema_length: Optional[int] = 100,
) -> "DataChain":
    """
    Read the results of a SQL query into a DataChain, using a given database connection.

    Args:
        query:
            The SQL query to execute. Can be a raw SQL string or a SQLAlchemy
            `Executable` object.
        connection: SQLAlchemy connectable, str, or a sqlite3 connection
            Using SQLAlchemy makes it possible to use any DB supported by that
            library. If a DBAPI2 object, only sqlite3 is supported. The user is
            responsible for engine disposal and connection closure for the
            SQLAlchemy connectable; str connections are closed automatically.
        params: Parameters to pass to execute method.
        output: A dictionary mapping column names to types, used to override the
            schema inferred from the query results.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        infer_schema_length:
            The maximum number of rows to scan for inferring schema.
            If set to `None`, the full data may be scanned.
            The rows used for schema inference are stored in memory,
            so large values can lead to high memory usage.
            Only applies if the `output` parameter is not set for the given column.

    Examples:
        Reading from a SQL query against a user-supplied connection:
        ```python
        query = "SELECT key, value FROM tbl"
        chain = dc.read_database(query, connection, output={"value": float})
        ```

        Load data from a SQLAlchemy driver/engine:
        ```python
        from sqlalchemy import create_engine
        engine = create_engine("postgresql+psycopg://myuser:mypassword@localhost:5432/mydb")
        chain = dc.read_database("select * from tbl", engine)
        ```

        Load data from a parameterized SQLAlchemy query:
        ```python
        query = "SELECT key, value FROM tbl WHERE value > :value"
        dc.read_database(query, engine, params={"value": 50})
        ```

    Notes:
        - This function works with a variety of databases — including,
        but not limited to, SQLite, DuckDB, PostgreSQL, and Snowflake,
        provided the appropriate driver is installed.
        - This call is blocking, and will execute the query and return once the
          results are saved.
    """
    from datachain.lib.dc.records import read_records

    output = output or {}
    if isinstance(query, str):
        query = sqlalchemy.text(query)
    kw = {"execution_options": {"stream_results": True}}  # use server-side cursors
    with _connect(connection) as conn, conn.execute(query, params, **kw) as result:
        cols = result.keys()
        to_infer = [k for k in cols if k not in output]  # preserve the order
        rows, inferred_schema = _infer_schema(result, to_infer, infer_schema_length)
        records = (row._asdict() for row in itertools.chain(rows, result))
        return read_records(
            records,
            session=session,
            settings=settings,
            in_memory=in_memory,
            schema=inferred_schema | output,
        )

ConnectionType module-attribute 

ConnectionType = Union[
    str,
    URL,
    Connectable,
    Engine,
    Connection,
    Session,
    Connection,
]

DataChain 

DataChain(
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: Optional[dict] = None,
    _sys: bool = False,
)

DataChain - a data structure for batch data processing and evaluation.

It represents a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data.

Data in DataChain is presented as Python classes with arbitrary set of fields, including nested classes. The data classes have to inherit from DataModel class. The supported set of field types include: majority of the type supported by the underlyind library Pydantic.

See Also

read_storage("s3://my-bucket/my-dir/") - reading unstructured data files from storages such as S3, gs or Azure ADLS.

DataChain.save("name") - saving to a dataset.

read_dataset("name") - reading from a dataset.

read_values(fib=[1, 2, 3, 5, 8]) - generating from values.

read_pandas(pd.DataFrame(...)) - generating from pandas.

read_json("file.json") - generating from json.

read_csv("file.csv") - generating from csv.

read_parquet("file.parquet") - generating from parquet.

Example 
import os

from mistralai.client import MistralClient
from mistralai.models.chat_completion import ChatMessage
import datachain as dc

PROMPT = (
    "Was this bot dialog successful? "
    "Describe the 'result' as 'Yes' or 'No' in a short JSON"
)

model = "mistral-large-latest"
api_key = os.environ["MISTRAL_API_KEY"]

chain = (
    dc.read_storage("gs://datachain-demo/chatbot-KiT/")
    .limit(5)
    .settings(cache=True, parallel=5)
    .map(
        mistral_response=lambda file: MistralClient(api_key=api_key)
        .chat(
            model=model,
            response_format={"type": "json_object"},
            messages=[
                ChatMessage(role="user", content=f"{PROMPT}: {file.read()}")
            ],
        )
        .choices[0]
        .message.content,
    )
    .persist()
)

try:
    print(chain.select("mistral_response").results())
except Exception as e:
    print(f"do you have the right Mistral API key? {e}")
Source code in datachain/lib/dc/datachain.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
def __init__(
    self,
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: Optional[dict] = None,
    _sys: bool = False,
) -> None:
    """Don't instantiate this directly, use one of the from_XXX constructors."""
    self._query = query
    self._settings = settings
    self.signals_schema = signal_schema
    self._setup: dict = setup or {}
    self._sys = _sys
    self._delta = False
    self._delta_on: Optional[Union[str, Sequence[str]]] = None
    self._delta_result_on: Optional[Union[str, Sequence[str]]] = None
    self._delta_compare: Optional[Union[str, Sequence[str]]] = None
    self._delta_retry: Optional[Union[bool, str]] = None

dataset property 

dataset: Optional[DatasetRecord]

Underlying dataset, if there is one.

delta property 

delta: bool

Returns True if this chain is ran in "delta" update mode

empty property 

empty: bool

Returns True if chain has zero number of rows

name property 

name: Optional[str]

Name of the underlying dataset, if there is one.

namespace_name property 

namespace_name: str

Current namespace name in which the chain is running

project_name property 

project_name: str

Current project name in which the chain is running

schema property 

schema: dict[str, DataType]

Get schema of the chain.

session property 

session: Session

Session of the chain.

version property 

version: Optional[str]

Version of the underlying dataset, if there is one.

__iter__ 

__iter__() -> Iterator[tuple[DataValue, ...]]

Make DataChain objects iterable.

Yields:

  • tuple[DataValue, ...] –

    Yields tuples of all column values for each row.

Example 
for row in chain:
    print(row)
Source code in datachain/lib/dc/datachain.py 
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
def __iter__(self) -> Iterator[tuple[DataValue, ...]]:
    """Make DataChain objects iterable.

    Yields:
        (tuple[DataValue, ...]): Yields tuples of all column values for each row.

    Example:
        ```py
        for row in chain:
            print(row)
        ```
    """
    return self.to_iter()

__or__ 

__or__(other: Self) -> Self

Return self.union(other).

Source code in datachain/lib/dc/datachain.py 
276
277
278
def __or__(self, other: "Self") -> "Self":
    """Return `self.union(other)`."""
    return self.union(other)

__repr__ 

__repr__() -> str

Return a string representation of the chain.

Source code in datachain/lib/dc/datachain.py 
187
188
189
190
191
192
193
194
195
196
197
def __repr__(self) -> str:
    """Return a string representation of the chain."""
    classname = self.__class__.__name__
    if not self._effective_signals_schema.values:
        return f"Empty {classname}"

    import io

    file = io.StringIO()
    self.print_schema(file=file)
    return file.getvalue()

agg 

agg(
    func: Optional[Callable] = None,
    partition_by: Optional[PartitionByType] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map: Callable
) -> Self

Aggregate rows using partition_by statement and apply a function to the groups of aggregated rows. The function needs to return new objects for each group of the new rows. It returns a chain itself with new signals.

Input-output relationship: N:M

This method bears similarity to gen() and map(), employing a comparable set of parameters, yet differs in two crucial aspects:

  1. The partition_by parameter: This specifies the column name or a list of column names that determine the grouping criteria for aggregation.
  2. Group-based UDF function input: Instead of individual rows, the function receives a list of all rows within each group defined by partition_by.

If partition_by is not set or is an empty list, all rows will be placed into a single group.

Parameters:

  • func (Optional[Callable], default: None ) –

    Function applied to each group of rows.

  • partition_by (Optional[PartitionByType], default: None ) –

    Column name(s) to group by. If None, all rows go into one group.

  • params (Union[None, str, Sequence[str]], default: None ) –

    List of column names used as input for the function. Default is taken from function signature.

  • output (OutputType, default: None ) –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature.

  • **signal_map (Callable, default: {} ) –

    kwargs can be used to define func together with its return signal name in format of agg(result_column=my_func).

Examples:

Basic aggregation with lambda function: 

chain = chain.agg(
    total=lambda category, amount: [sum(amount)],
    output=float,
    partition_by="category",
)
chain.save("new_dataset")

An alternative syntax, when you need to specify a more complex function: 

# It automatically resolves which columns to pass to the function
# by looking at the function signature.
def agg_sum(
    file: list[File], amount: list[float]
) -> Iterator[tuple[File, float]]:
    yield file[0], sum(amount)

chain = chain.agg(
    agg_sum,
    output={"file": File, "total": float},
    # Alternative syntax is to use `C` (short for Column) to specify
    # a column name or a nested column, e.g. C("file.path").
    partition_by=C("category"),
)
chain.save("new_dataset")

Using complex signals for partitioning (File or any Pydantic BaseModel): 

def my_agg(files: list[File]) -> Iterator[tuple[File, int]]:
    yield files[0], sum(f.size for f in files)

chain = chain.agg(
    my_agg,
    params=("file",),
    output={"file": File, "total": int},
    partition_by="file",  # Column referring to all sub-columns of File
)
chain.save("new_dataset")

Aggregating all rows into a single group (when partition_by is not set): 

chain = chain.agg(
    total_size=lambda file, size: [sum(size)],
    output=int,
    # No partition_by specified - all rows go into one group
)
chain.save("new_dataset")

Multiple partition columns: 

chain = chain.agg(
    total=lambda category, subcategory, amount: [sum(amount)],
    output=float,
    partition_by=["category", "subcategory"],
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
@delta_disabled
def agg(
    self,
    /,
    func: Optional[Callable] = None,
    partition_by: Optional[PartitionByType] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map: Callable,
) -> "Self":
    """Aggregate rows using `partition_by` statement and apply a function to the
    groups of aggregated rows. The function needs to return new objects for each
    group of the new rows. It returns a chain itself with new signals.

    Input-output relationship: N:M

    This method bears similarity to `gen()` and `map()`, employing a comparable set
    of parameters, yet differs in two crucial aspects:

    1. The `partition_by` parameter: This specifies the column name or a list of
       column names that determine the grouping criteria for aggregation.
    2. Group-based UDF function input: Instead of individual rows, the function
       receives a list of all rows within each group defined by `partition_by`.

    If `partition_by` is not set or is an empty list, all rows will be placed
    into a single group.

    Parameters:
        func: Function applied to each group of rows.
        partition_by: Column name(s) to group by. If None, all rows go
            into one group.
        params: List of column names used as input for the function. Default is
            taken from function signature.
        output: Dictionary defining new signals and their corresponding types.
            Default type is taken from function signature.
        **signal_map: kwargs can be used to define `func` together with its return
            signal name in format of `agg(result_column=my_func)`.

    Examples:
        Basic aggregation with lambda function:
        ```py
        chain = chain.agg(
            total=lambda category, amount: [sum(amount)],
            output=float,
            partition_by="category",
        )
        chain.save("new_dataset")
        ```

        An alternative syntax, when you need to specify a more complex function:
        ```py
        # It automatically resolves which columns to pass to the function
        # by looking at the function signature.
        def agg_sum(
            file: list[File], amount: list[float]
        ) -> Iterator[tuple[File, float]]:
            yield file[0], sum(amount)

        chain = chain.agg(
            agg_sum,
            output={"file": File, "total": float},
            # Alternative syntax is to use `C` (short for Column) to specify
            # a column name or a nested column, e.g. C("file.path").
            partition_by=C("category"),
        )
        chain.save("new_dataset")
        ```

        Using complex signals for partitioning (`File` or any Pydantic `BaseModel`):
        ```py
        def my_agg(files: list[File]) -> Iterator[tuple[File, int]]:
            yield files[0], sum(f.size for f in files)

        chain = chain.agg(
            my_agg,
            params=("file",),
            output={"file": File, "total": int},
            partition_by="file",  # Column referring to all sub-columns of File
        )
        chain.save("new_dataset")
        ```

        Aggregating all rows into a single group (when `partition_by` is not set):
        ```py
        chain = chain.agg(
            total_size=lambda file, size: [sum(size)],
            output=int,
            # No partition_by specified - all rows go into one group
        )
        chain.save("new_dataset")
        ```

        Multiple partition columns:
        ```py
        chain = chain.agg(
            total=lambda category, subcategory, amount: [sum(amount)],
            output=float,
            partition_by=["category", "subcategory"],
        )
        chain.save("new_dataset")
        ```
    """
    if partition_by is not None:
        # Convert string partition_by parameters to Column objects
        if isinstance(partition_by, (str, Function, ColumnElement)):
            list_partition_by = [partition_by]
        else:
            list_partition_by = list(partition_by)

        processed_partition_columns: list[ColumnElement] = []
        for col in list_partition_by:
            if isinstance(col, str):
                columns = self.signals_schema.db_signals(name=col, as_columns=True)
                if not columns:
                    raise SignalResolvingError([col], "is not found")
                processed_partition_columns.extend(cast("list[Column]", columns))
            elif isinstance(col, Function):
                column = col.get_column(self.signals_schema)
                processed_partition_columns.append(column)
            else:
                # Assume it's already a ColumnElement
                processed_partition_columns.append(col)

        processed_partition_by = processed_partition_columns
    else:
        processed_partition_by = []

    udf_obj = self._udf_to_obj(Aggregator, func, params, output, signal_map)
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(),
            partition_by=processed_partition_by,
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

apply 

apply(func, *args, **kwargs)

Apply any function to the chain.

Useful for reusing in a chain of operations.

Example 
import datachain as dc
def parse_stem(chain):
    return chain.map(
        lambda file: file.get_file_stem()
        output={"stem": str}
    )

chain = (
    dc.read_storage("s3://my-bucket")
    .apply(parse_stem)
    .filter(C("stem").glob("*cat*"))
)
Source code in datachain/lib/dc/datachain.py 
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
def apply(self, func, *args, **kwargs):
    """Apply any function to the chain.

    Useful for reusing in a chain of operations.

    Example:
        ```py
        import datachain as dc
        def parse_stem(chain):
            return chain.map(
                lambda file: file.get_file_stem()
                output={"stem": str}
            )

        chain = (
            dc.read_storage("s3://my-bucket")
            .apply(parse_stem)
            .filter(C("stem").glob("*cat*"))
        )
        ```
    """
    return func(self, *args, **kwargs)

avg 

avg(col: str) -> StandardType

Compute the average of a column.

Parameters:

  • col (str) –

    The column to compute the average for.

Returns:

  • StandardType –

    The average of the column values.

Example 
average_size = chain.avg("file.size")
print(f"Average size: {average_size}")
Source code in datachain/lib/dc/datachain.py 
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
def avg(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the average of a column.

    Parameters:
        col: The column to compute the average for.

    Returns:
        The average of the column values.

    Example:
        ```py
        average_size = chain.avg("file.size")
        print(f"Average size: {average_size}")
        ```
    """
    return self._extend_to_data_model("avg", col)

batch_map 

batch_map(
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map
) -> Self

This is a batch version of map().

Input-output relationship: N:N

It accepts the same parameters plus an additional parameter:

batch : Size of each batch passed to `func`. Defaults to 1000.
Example 
chain = chain.batch_map(
    sqrt=lambda size: np.sqrt(size),
    output=float
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
def batch_map(
    self,
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map,
) -> "Self":
    """This is a batch version of `map()`.

    Input-output relationship: N:N

    It accepts the same parameters plus an
    additional parameter:

        batch : Size of each batch passed to `func`. Defaults to 1000.

    Example:
        ```py
        chain = chain.batch_map(
            sqrt=lambda size: np.sqrt(size),
            output=float
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(BatchMapper, func, params, output, signal_map)
    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(batch),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

c 

c(column: Union[str, Column]) -> Column

Returns Column instance attached to the current chain.

Source code in datachain/lib/dc/datachain.py 
248
249
250
251
252
def c(self, column: Union[str, Column]) -> Column:
    """Returns Column instance attached to the current chain."""
    c = self.column(column) if isinstance(column, str) else self.column(column.name)
    c.table = self._query.table
    return c

chunk 

chunk(index: int, total: int) -> Self

Split a chain into smaller chunks for e.g. parallelization.

Parameters:

  • index (int) –

    The index of the chunk (0-indexed).

  • total (int) –

    The total number of chunks.

Example 
import datachain as dc

chain = dc.read_storage(...)
chunk_1 = query._chunk(0, 2)
chunk_2 = query._chunk(1, 2)
Note

Bear in mind that index is 0-indexed but total isn't. Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.

Source code in datachain/lib/dc/datachain.py 
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
def chunk(self, index: int, total: int) -> "Self":
    """Split a chain into smaller chunks for e.g. parallelization.

    Parameters:
        index: The index of the chunk (0-indexed).
        total: The total number of chunks.

    Example:
        ```py
        import datachain as dc

        chain = dc.read_storage(...)
        chunk_1 = query._chunk(0, 2)
        chunk_2 = query._chunk(1, 2)
        ```

    Note:
        Bear in mind that `index` is 0-indexed but `total` isn't.
        Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.
    """
    return self._evolve(query=self._query.chunk(index, total))

clone 

clone() -> Self

Make a copy of the chain in a new table.

Source code in datachain/lib/dc/datachain.py 
284
285
286
def clone(self) -> "Self":
    """Make a copy of the chain in a new table."""
    return self._evolve(query=self._query.clone(new_table=True))

collect 

collect() -> Iterator[tuple[DataValue, ...]]

collect(col: str) -> Iterator[DataValue]

collect(*cols: str) -> Iterator[tuple[DataValue, ...]]

collect(
    *cols: str,
) -> Iterator[Union[DataValue, tuple[DataValue, ...]]]

Deprecated. Use to_iter method instead.

Source code in datachain/lib/dc/datachain.py 
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
def collect(self, *cols: str) -> Iterator[Union[DataValue, tuple[DataValue, ...]]]:  # type: ignore[overload-overlap,misc]
    """
    Deprecated. Use `to_iter` method instead.
    """
    warnings.warn(
        "Method `collect` is deprecated. Use `to_iter` method instead.",
        DeprecationWarning,
        stacklevel=2,
    )

    if len(cols) == 1:
        yield from [item[0] for item in self.to_iter(*cols)]
    else:
        yield from self.to_iter(*cols)

column 

column(name: str) -> Column

Returns Column instance with a type if name is found in current schema, otherwise raises an exception.

Source code in datachain/lib/dc/datachain.py 
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
def column(self, name: str) -> Column:
    """Returns Column instance with a type if name is found in current schema,
    otherwise raises an exception.
    """
    if "." in name:
        name_path = name.split(".")
    elif DEFAULT_DELIMITER in name:
        name_path = name.split(DEFAULT_DELIMITER)
    else:
        name_path = [name]
    for path, type_, _, _ in self.signals_schema.get_flat_tree():
        if path == name_path:
            return Column(name, python_to_sql(type_))

    raise ValueError(f"Column with name {name} not found in the schema")

count 

count() -> int

Return the number of rows in the chain.

Source code in datachain/lib/dc/datachain.py 
2553
2554
2555
def count(self) -> int:
    """Return the number of rows in the chain."""
    return self._query.count()

diff 

diff(
    other: DataChain,
    on: Union[str, Sequence[str]],
    right_on: Optional[Union[str, Sequence[str]]] = None,
    compare: Optional[Union[str, Sequence[str]]] = None,
    right_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: Optional[str] = None,
) -> DataChain

Calculate differences between two chains.

This method identifies records that are added, deleted, modified, or unchanged between two chains. It adds a status column with values: A=added, D=deleted, M=modified, S=same.

Parameters:

  • other (DataChain) –

    Chain to compare against.

  • on (Union[str, Sequence[str]]) –

    Column(s) to match records between chains.

  • right_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Column(s) in the other chain to match against. Defaults to on.

  • compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Column(s) to check for changes. If not specified,all columns are used.

  • right_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Column(s) in the other chain to compare against. Defaults to values of compare.

  • added (bool, default: True ) –

    Include records that exist in this chain but not in the other.

  • deleted (bool, default: True ) –

    Include records that exist only in the other chain.

  • modified (bool, default: True ) –

    Include records that exist in both but have different values.

  • same (bool, default: False ) –

    Include records that are identical in both chains.

  • status_col (str, default: None ) –

    Name for the status column showing differences.

Default behavior: By default, shows added, deleted, and modified records, but excludes unchanged records (same=False). Status column is not created.

Example 
res = persons.diff(
    new_persons,
    on=["id"],
    right_on=["other_id"],
    compare=["name"],
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
def diff(
    self,
    other: "DataChain",
    on: Union[str, Sequence[str]],
    right_on: Optional[Union[str, Sequence[str]]] = None,
    compare: Optional[Union[str, Sequence[str]]] = None,
    right_compare: Optional[Union[str, Sequence[str]]] = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: Optional[str] = None,
) -> "DataChain":
    """Calculate differences between two chains.

    This method identifies records that are added, deleted, modified, or unchanged
    between two chains. It adds a status column with values: A=added, D=deleted,
    M=modified, S=same.

    Parameters:
        other: Chain to compare against.
        on: Column(s) to match records between chains.
        right_on: Column(s) in the other chain to match against. Defaults to `on`.
        compare: Column(s) to check for changes.
                 If not specified,all columns are used.
        right_compare: Column(s) in the other chain to compare against.
                 Defaults to values of `compare`.
        added (bool): Include records that exist in this chain but not in the other.
        deleted (bool): Include records that exist only in the other chain.
        modified (bool): Include records that exist in both
                 but have different values.
        same (bool): Include records that are identical in both chains.
        status_col (str): Name for the status column showing differences.

    Default behavior: By default, shows added, deleted, and modified records,
    but excludes unchanged records (same=False). Status column is not created.

    Example:
        ```py
        res = persons.diff(
            new_persons,
            on=["id"],
            right_on=["other_id"],
            compare=["name"],
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    from datachain.diff import _compare

    return _compare(
        self,
        other,
        on,
        right_on=right_on,
        compare=compare,
        right_compare=right_compare,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

distinct 

distinct(arg: str, *args: str) -> Self

Removes duplicate rows based on uniqueness of some input column(s) i.e if rows are found with the same value of input column(s), only one row is left in the result set.

Example 
dc.distinct("file.path")
Source code in datachain/lib/dc/datachain.py 
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
@delta_disabled
def distinct(self, arg: str, *args: str) -> "Self":  # type: ignore[override]
    """Removes duplicate rows based on uniqueness of some input column(s)
    i.e if rows are found with the same value of input column(s), only one
    row is left in the result set.

    Example:
        ```py
        dc.distinct("file.path")
        ```
    """
    return self._evolve(
        query=self._query.distinct(
            *self.signals_schema.resolve(arg, *args).db_signals()
        )
    )

exec 

exec() -> Self

Execute the chain.

Source code in datachain/lib/dc/datachain.py 
2557
2558
2559
def exec(self) -> "Self":
    """Execute the chain."""
    return self._evolve(query=self._query.exec())

explode 

explode(
    col: str,
    model_name: Optional[str] = None,
    column: Optional[str] = None,
    schema_sample_size: int = 1,
) -> DataChain

Explodes a column containing JSON objects (dict or str DataChain type) into individual columns based on the schema of the JSON. Schema is inferred from the first row of the column.

Parameters:

  • col (str) –

    the name of the column containing JSON to be exploded.

  • model_name (Optional[str], default: None ) –

    optional generated model name. By default generates the name automatically.

  • column (Optional[str], default: None ) –

    optional generated column name. By default generates the name automatically.

  • schema_sample_size (int, default: 1 ) –

    the number of rows to use for inferring the schema of the JSON (in case some fields are optional and it's not enough to analyze a single row).

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance with the new set of columns.

Source code in datachain/lib/dc/datachain.py 
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
def explode(
    self,
    col: str,
    model_name: Optional[str] = None,
    column: Optional[str] = None,
    schema_sample_size: int = 1,
) -> "DataChain":
    """Explodes a column containing JSON objects (dict or str DataChain type) into
       individual columns based on the schema of the JSON. Schema is inferred from
       the first row of the column.

    Args:
        col: the name of the column containing JSON to be exploded.
        model_name: optional generated model name.  By default generates the name
            automatically.
        column: optional generated column name. By default generates the
            name automatically.
        schema_sample_size: the number of rows to use for inferring the schema of
            the JSON (in case some fields are optional and it's not enough to
            analyze a single row).

    Returns:
        DataChain: A new DataChain instance with the new set of columns.
    """
    import json

    import pyarrow as pa

    from datachain.lib.arrow import schema_to_output

    json_values = self.limit(schema_sample_size).to_list(col)
    json_dicts = [
        json.loads(json_value) if isinstance(json_value, str) else json_value
        for (json_value,) in json_values
    ]

    if any(not isinstance(json_dict, dict) for json_dict in json_dicts):
        raise TypeError(f"Column {col} should be a string or dict type with JSON")

    schema = pa.Table.from_pylist(json_dicts).schema
    output, original_names = schema_to_output(schema, None)

    if not model_name:
        model_name = f"{col.title()}ExplodedModel"

    model = dict_to_data_model(model_name, output, original_names)

    def json_to_model(json_value: Union[str, dict]):
        json_dict = (
            json.loads(json_value) if isinstance(json_value, str) else json_value
        )
        return model.model_validate(json_dict)

    if not column:
        column = f"{col}_expl"

    return self.map(json_to_model, params=col, output={column: model})

file_diff 

file_diff(
    other: DataChain,
    on: str = "file",
    right_on: Optional[str] = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: Optional[str] = None,
) -> DataChain

Calculate differences between two chains containing files.

This method is specifically designed for file chains. It uses file source and path to match files, and file version and etag to detect changes.

Parameters:

  • other (DataChain) –

    Chain to compare against.

  • on (str, default: 'file' ) –

    File column name in this chain. Default is "file".

  • right_on (Optional[str], default: None ) –

    File column name in the other chain. Defaults to on.

  • added (bool, default: True ) –

    Include files that exist in this chain but not in the other.

  • deleted (bool, default: False ) –

    Include files that exist only in the other chain.

  • modified (bool, default: True ) –

    Include files that exist in both but have different versions/etags.

  • same (bool, default: False ) –

    Include files that are identical in both chains.

  • status_col (str, default: None ) –

    Name for the status column showing differences (A=added, D=deleted, M=modified, S=same).

Default behavior: By default, includes only new files (added=True and modified=True). This is useful for incremental processing.

Example 
diff = images.file_diff(
    new_images,
    on="file",
    right_on="other_file",
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
def file_diff(
    self,
    other: "DataChain",
    on: str = "file",
    right_on: Optional[str] = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: Optional[str] = None,
) -> "DataChain":
    """Calculate differences between two chains containing files.

    This method is specifically designed for file chains. It uses file `source`
    and `path` to match files, and file `version` and `etag` to detect changes.

    Parameters:
        other: Chain to compare against.
        on: File column name in this chain. Default is "file".
        right_on: File column name in the other chain. Defaults to `on`.
        added (bool): Include files that exist in this chain but not in the other.
        deleted (bool): Include files that exist only in the other chain.
        modified (bool): Include files that exist in both but have different
                         versions/etags.
        same (bool): Include files that are identical in both chains.
        status_col (str): Name for the status column showing differences
                          (A=added, D=deleted, M=modified, S=same).

    Default behavior: By default, includes only new files (added=True and
    modified=True). This is useful for incremental processing.

    Example:
        ```py
        diff = images.file_diff(
            new_images,
            on="file",
            right_on="other_file",
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    on_file_signals = ["source", "path"]
    compare_file_signals = ["version", "etag"]

    def get_file_signals(file: str, signals):
        return [f"{file}.{c}" for c in signals]

    right_on = right_on or on

    on_cols = get_file_signals(on, on_file_signals)
    right_on_cols = get_file_signals(right_on, on_file_signals)
    compare_cols = get_file_signals(on, compare_file_signals)
    right_compare_cols = get_file_signals(right_on, compare_file_signals)

    return self.diff(
        other,
        on_cols,
        right_on=right_on_cols,
        compare=compare_cols,
        right_compare=right_compare_cols,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

filter 

filter(*args: Any) -> Self

Filter the chain according to conditions.

Example

Basic usage with built-in operators 

dc.filter(C("width") < 200)

Using glob to match patterns 

dc.filter(C("file.path").glob("*.jpg"))

Using in to match lists 

ids = [1,2,3]
dc.filter(C("experiment_id").in_(ids))

Using datachain.func 

from datachain.func import string
dc.filter(string.length(C("file.path")) > 5)

Combining filters with "or" 

dc.filter(
    C("file.path").glob("cat*") |
    C("file.path").glob("dog*")
)


dc.filter(dc.func.or_(
    C("file.path").glob("cat*"),
    C("file.path").glob("dog*")
))

Combining filters with "and" 

dc.filter(
    C("file.path").glob("*.jpg"),
    string.length(C("file.path")) > 5
)


dc.filter(
    C("file.path").glob("*.jpg") &
    (string.length(C("file.path")) > 5)
)

dc.filter(dc.func.and_(
    C("file.path").glob("*.jpg"),
    string.length(C("file.path")) > 5
))

Combining filters with "not" 

dc.filter(~(C("file.path").glob("*.jpg")))

Source code in datachain/lib/dc/datachain.py 
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
def filter(self, *args: Any) -> "Self":
    """Filter the chain according to conditions.

    Example:
        Basic usage with built-in operators
        ```py
        dc.filter(C("width") < 200)
        ```

        Using glob to match patterns
        ```py
        dc.filter(C("file.path").glob("*.jpg"))
        ```

        Using in to match lists
        ```py
        ids = [1,2,3]
        dc.filter(C("experiment_id").in_(ids))
        ```

        Using `datachain.func`
        ```py
        from datachain.func import string
        dc.filter(string.length(C("file.path")) > 5)
        ```

        Combining filters with "or"
        ```py
        dc.filter(
            C("file.path").glob("cat*") |
            C("file.path").glob("dog*")
        )
        ```

        ```py
        dc.filter(dc.func.or_(
            C("file.path").glob("cat*"),
            C("file.path").glob("dog*")
        ))
        ```

        Combining filters with "and"
        ```py
        dc.filter(
            C("file.path").glob("*.jpg"),
            string.length(C("file.path")) > 5
        )
        ```

        ```py
        dc.filter(
            C("file.path").glob("*.jpg") &
            (string.length(C("file.path")) > 5)
        )
        ```

        ```py
        dc.filter(dc.func.and_(
            C("file.path").glob("*.jpg"),
            string.length(C("file.path")) > 5
        ))
        ```

        Combining filters with "not"
        ```py
        dc.filter(~(C("file.path").glob("*.jpg")))
        ```
    """
    return self._evolve(query=self._query.filter(*args))

gen 

gen(
    func: Optional[Union[Callable, Generator]] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map
) -> Self

Apply a function to each row to create new rows (with potentially new signals). The function needs to return a new objects for each of the new rows. It returns a chain itself with new signals.

Input-output relationship: 1:N

This method is similar to map(), uses the same list of parameters, but with one key differences: It produces a sequence of rows for each input row (like extracting multiple file records from a single tar file or bounding boxes from a single image file).

Example 
chain = chain.gen(
    line=lambda file: [l for l in file.read().split("\n")],
    output=str,
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
def gen(
    self,
    func: Optional[Union[Callable, Generator]] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    r"""Apply a function to each row to create new rows (with potentially new
    signals). The function needs to return a new objects for each of the new rows.
    It returns a chain itself with new signals.

    Input-output relationship: 1:N

    This method is similar to `map()`, uses the same list of parameters, but with
    one key differences: It produces a sequence of rows for each input row (like
    extracting multiple file records from a single tar file or bounding boxes from a
    single image file).

    Example:
        ```py
        chain = chain.gen(
            line=lambda file: [l for l in file.read().split("\n")],
            output=str,
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Generator, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(),
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

group_by 

group_by(
    *,
    partition_by: Optional[
        Union[str, Func, Sequence[Union[str, Func]]]
    ] = None,
    **kwargs: Func
) -> Self

Group rows by specified set of signals and return new signals with aggregated values.

The supported functions

count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

Example 
chain = chain.group_by(
    cnt=func.count(),
    partition_by=("file_source", "file_ext"),
)

Using complex signals: 

chain = chain.group_by(
    total_size=func.sum("file.size"),
    count=func.count(),
    partition_by="file",  # Uses column name, expands to File's unique keys
)

Source code in datachain/lib/dc/datachain.py 
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
@delta_disabled  # type: ignore[arg-type]
def group_by(  # noqa: C901, PLR0912
    self,
    *,
    partition_by: Optional[Union[str, Func, Sequence[Union[str, Func]]]] = None,
    **kwargs: Func,
) -> "Self":
    """Group rows by specified set of signals and return new signals
    with aggregated values.

    The supported functions:
       count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

    Example:
        ```py
        chain = chain.group_by(
            cnt=func.count(),
            partition_by=("file_source", "file_ext"),
        )
        ```

        Using complex signals:
        ```py
        chain = chain.group_by(
            total_size=func.sum("file.size"),
            count=func.count(),
            partition_by="file",  # Uses column name, expands to File's unique keys
        )
        ```
    """
    if partition_by is None:
        partition_by = []
    elif isinstance(partition_by, (str, Func)):
        partition_by = [partition_by]

    partition_by_columns: list[Column] = []
    signal_columns: list[Column] = []
    schema_fields: dict[str, DataType] = {}
    keep_columns: list[str] = []
    partial_fields: list[str] = []  # Track specific fields for partial creation
    schema_partition_by: list[str] = []

    for col in partition_by:
        if isinstance(col, str):
            columns = self.signals_schema.db_signals(name=col, as_columns=True)
            if not columns:
                raise SignalResolvingError([col], "is not found")
            partition_by_columns.extend(cast("list[Column]", columns))

            # For nested field references (e.g., "nested.level1.name"),
            # we need to distinguish between:
            # 1. References to fields within a complex signal (create partials)
            # 2. Deep nested references that should be flattened
            if "." in col:
                # Split the column reference to analyze it
                parts = col.split(".")
                parent_signal = parts[0]
                parent_type = self.signals_schema.values.get(parent_signal)

                if ModelStore.is_partial(parent_type):
                    if parent_signal not in keep_columns:
                        keep_columns.append(parent_signal)
                    partial_fields.append(col)
                    schema_partition_by.append(col)
                else:
                    # BaseModel or other - add flattened columns directly
                    for column in cast("list[Column]", columns):
                        col_type = self.signals_schema.get_column_type(column.name)
                        schema_fields[column.name] = col_type
                    schema_partition_by.append(col)
            else:
                # simple signal - but we need to check if it's a complex signal
                # complex signal - only include the columns used for partitioning
                col_type = self.signals_schema.get_column_type(
                    col, with_subtree=True
                )
                if isinstance(col_type, type) and issubclass(col_type, BaseModel):
                    # Complex signal - add only the partitioning columns
                    for column in cast("list[Column]", columns):
                        col_type = self.signals_schema.get_column_type(column.name)
                        schema_fields[column.name] = col_type
                    schema_partition_by.append(col)
                # Simple signal - keep the entire signal
                else:
                    if col not in keep_columns:
                        keep_columns.append(col)
                    schema_partition_by.append(col)
        elif isinstance(col, Function):
            column = col.get_column(self.signals_schema)
            col_db_name = column.name
            col_type = column.type.python_type
            schema_fields[col_db_name] = col_type
            partition_by_columns.append(column)
            signal_columns.append(column)
        else:
            raise DataChainColumnError(
                col,
                (
                    f"partition_by column {col} has type {type(col)}"
                    " but expected str or Function"
                ),
            )

    if not kwargs:
        raise ValueError("At least one column should be provided for group_by")
    for col_name, func in kwargs.items():
        if not isinstance(func, Func):
            raise DataChainColumnError(
                col_name,
                f"Column {col_name} has type {type(func)} but expected Func object",
            )
        column = func.get_column(self.signals_schema, label=col_name)
        signal_columns.append(column)
        schema_fields[col_name] = func.get_result_type(self.signals_schema)

    signal_schema = self.signals_schema.group_by(
        schema_partition_by, signal_columns
    )

    return self._evolve(
        query=self._query.group_by(signal_columns, partition_by_columns),
        signal_schema=signal_schema,
    )

limit 

limit(n: int) -> Self

Return the first n rows of the chain.

If the chain is unordered, which rows are returned is undefined. If the chain has less than n rows, the whole chain is returned.

Parameters:

  • n (int) –

    Number of rows to return.

Source code in datachain/lib/dc/datachain.py 
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
def limit(self, n: int) -> "Self":
    """Return the first `n` rows of the chain.

    If the chain is unordered, which rows are returned is undefined.
    If the chain has less than `n` rows, the whole chain is returned.

    Parameters:
        n (int): Number of rows to return.
    """
    return self._evolve(query=self._query.limit(n))

map 

map(
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map
) -> Self

Apply a function to each row to create new signals. The function should return a new object for each row. It returns a chain itself with new signals.

Input-output relationship: 1:1

Parameters:

  • func –

    Function applied to each row.

  • params –

    List of column names used as input for the function. Default is taken from function signature.

  • output –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature. Default can be also taken from kwargs - **signal_map (see below). If signal name is defined using signal_map (see below) only a single type value can be used.

  • **signal_map –

    kwargs can be used to define func together with it's return signal name in format of map(my_sign=my_func). This helps define signal names and function in a nicer way.

Example

Using signal_map and single type in output: 

chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
chain.save("new_dataset")

Using func and output as a map: 

chain = chain.map(
    lambda name: name.split("."), output={"stem": str, "ext": str}
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
def map(
    self,
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    """Apply a function to each row to create new signals. The function should
    return a new object for each row. It returns a chain itself with new signals.

    Input-output relationship: 1:1

    Parameters:
        func : Function applied to each row.
        params : List of column names used as input for the function. Default
                is taken from function signature.
        output : Dictionary defining new signals and their corresponding types.
                Default type is taken from function signature. Default can be also
                taken from kwargs - **signal_map (see below).
                If signal name is defined using signal_map (see below) only a single
                type value can be used.
        **signal_map : kwargs can be used to define `func` together with it's return
                signal name in format of `map(my_sign=my_func)`. This helps define
                signal names and function in a nicer way.

    Example:
        Using signal_map and single type in output:
        ```py
        chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
        chain.save("new_dataset")
        ```

        Using func and output as a map:
        ```py
        chain = chain.map(
            lambda name: name.split("."), output={"stem": str, "ext": str}
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Mapper, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch

    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

max 

max(col: str) -> StandardType

Compute the maximum of a column.

Parameters:

  • col (str) –

    The column to compute the maximum for.

Returns:

  • StandardType –

    The maximum value in the column.

Example 
max_size = chain.max("file.size")
print(f"Maximum size: {max_size}")
Source code in datachain/lib/dc/datachain.py 
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
def max(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the maximum of a column.

    Parameters:
        col: The column to compute the maximum for.

    Returns:
        The maximum value in the column.

    Example:
        ```py
        max_size = chain.max("file.size")
        print(f"Maximum size: {max_size}")
        ```
    """
    return self._extend_to_data_model("max", col)

merge 

merge(
    right_ds: DataChain,
    on: Union[MergeColType, Sequence[MergeColType]],
    right_on: Optional[
        Union[MergeColType, Sequence[MergeColType]]
    ] = None,
    inner=False,
    full=False,
    rname="right_",
) -> Self

Merge two chains based on the specified criteria.

Parameters:

  • right_ds (DataChain) –

    Chain to join with.

  • on (Union[MergeColType, Sequence[MergeColType]]) –

    Predicate ("column.name", C("column.name"), or Func) or list of Predicates to join on. If both chains have the same predicates then this predicate is enough for the join. Otherwise, right_on parameter has to specify the predicates for the other chain.

  • right_on (Optional[Union[MergeColType, Sequence[MergeColType]]], default: None ) –

    Optional predicate or list of Predicates for the right_ds to join.

  • inner (bool, default: False ) –

    Whether to run inner join or outer join.

  • full (bool, default: False ) –

    Whether to run full outer join.

  • rname (str, default: 'right_' ) –

    Name prefix for conflicting signal names.

Examples:

meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                      right_on=(C.name, C.pq__index))

imgs.merge(captions,
           on=func.path.file_stem(imgs.c("file.path")),
           right_on=func.path.file_stem(captions.c("file.path"))

)

Source code in datachain/lib/dc/datachain.py 
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
@delta_disabled
def merge(
    self,
    right_ds: "DataChain",
    on: Union[MergeColType, Sequence[MergeColType]],
    right_on: Optional[Union[MergeColType, Sequence[MergeColType]]] = None,
    inner=False,
    full=False,
    rname="right_",
) -> "Self":
    """Merge two chains based on the specified criteria.

    Parameters:
        right_ds: Chain to join with.
        on: Predicate ("column.name", C("column.name"), or Func) or list of
            Predicates to join on. If both chains have the same predicates then
            this predicate is enough for the join. Otherwise, `right_on` parameter
            has to specify the predicates for the other chain.
        right_on: Optional predicate or list of Predicates for the `right_ds`
            to join.
        inner (bool): Whether to run inner join or outer join.
        full (bool): Whether to run full outer join.
        rname (str): Name prefix for conflicting signal names.

    Examples:
        ```py
        meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                              right_on=(C.name, C.pq__index))
        ```

        ```py
        imgs.merge(captions,
                   on=func.path.file_stem(imgs.c("file.path")),
                   right_on=func.path.file_stem(captions.c("file.path"))
        ```
    )
    """
    if on is None:
        raise DatasetMergeError(["None"], None, "'on' must be specified")

    on = _validate_merge_on(on, self)
    if not on:
        raise DatasetMergeError(
            on,
            right_on,
            (
                "'on' must be 'str', 'Func' or 'Sequence' object "
                f"but got type '{type(on)}'"
            ),
        )

    if right_on is not None:
        right_on = _validate_merge_on(right_on, right_ds)
        if not right_on:
            raise DatasetMergeError(
                on,
                right_on,
                "'right_on' must be 'str', 'Func' or 'Sequence' object"
                f" but got type '{type(right_on)}'",
            )

        if len(right_on) != len(on):
            raise DatasetMergeError(
                on, right_on, "'on' and 'right_on' must have the same length'"
            )

    if self == right_ds:
        right_ds = right_ds.clone()

    errors = []

    def _resolve(
        ds: DataChain,
        col: Union[str, Function, sqlalchemy.ColumnElement],
        side: Union[str, None],
    ):
        try:
            if isinstance(col, Function):
                return ds.c(col.get_column())
            return ds.c(col) if isinstance(col, (str, C)) else col
        except ValueError:
            if side:
                errors.append(f"{_get_merge_error_str(col)} in {side}")

    ops = [
        _resolve(self, left, "left")
        == _resolve(right_ds, right, "right" if right_on else None)
        for left, right in zip(on, right_on or on)
    ]

    if errors:
        raise DatasetMergeError(
            on, right_on, f"Could not resolve {', '.join(errors)}"
        )

    query = self._query.join(
        right_ds._query, sqlalchemy.and_(*ops), inner, full, rname + "{name}"
    )
    query.feature_schema = None
    ds = self._evolve(query=query)

    signals_schema = self.signals_schema.clone_without_sys_signals()
    right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
    ds.signals_schema = SignalSchema({"sys": Sys}) | signals_schema.merge(
        right_signals_schema, rname
    )

    return ds

min 

min(col: str) -> StandardType

Compute the minimum of a column.

Parameters:

  • col (str) –

    The column to compute the minimum for.

Returns:

  • StandardType –

    The minimum value in the column.

Example 
min_size = chain.min("file.size")
print(f"Minimum size: {min_size}")
Source code in datachain/lib/dc/datachain.py 
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
def min(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the minimum of a column.

    Parameters:
        col: The column to compute the minimum for.

    Returns:
        The minimum value in the column.

    Example:
        ```py
        min_size = chain.min("file.size")
        print(f"Minimum size: {min_size}")
        ```
    """
    return self._extend_to_data_model("min", col)

mutate 

mutate(**kwargs) -> Self

Create new signals based on existing signals.

This method cannot modify existing columns. If you need to modify an existing column, use a different name for the new column and then use select() to choose which columns to keep.

This method is vectorized and more efficient compared to map(), and it does not extract or download any data from the internal database. However, it can only utilize predefined built-in functions and their combinations.

The supported functions

Numerical: +, -, *, /, rand(), avg(), count(), func(), greatest(), least(), max(), min(), sum() String: length(), split(), replace(), regexp_replace() Filename: name(), parent(), file_stem(), file_ext() Array: length(), sip_hash_64(), euclidean_distance(), cosine_distance() Window: row_number(), rank(), dense_rank(), first()

Example: 

 dc.mutate(
    area=Column("image.height") * Column("image.width"),
    extension=file_ext(Column("file.path")),
    dist=cosine_distance(embedding_text, embedding_image)
)

Window function example: 

window = func.window(partition_by="file.parent", order_by="file.size")
dc.mutate(
    row_number=func.row_number().over(window),
)

This method can be also used to rename signals. If the Column("name") provided as value for the new signal - the old column will be dropped. Otherwise a new column is created.

Example: 

 dc.mutate(
    newkey=Column("oldkey")
)

Source code in datachain/lib/dc/datachain.py 
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
def mutate(self, **kwargs) -> "Self":
    """Create new signals based on existing signals.

    This method cannot modify existing columns. If you need to modify an
    existing column, use a different name for the new column and then use
    `select()` to choose which columns to keep.

    This method is vectorized and more efficient compared to map(), and it does not
    extract or download any data from the internal database. However, it can only
    utilize predefined built-in functions and their combinations.

    The supported functions:
       Numerical:   +, -, *, /, rand(), avg(), count(), func(),
                    greatest(), least(), max(), min(), sum()
       String:      length(), split(), replace(), regexp_replace()
       Filename:    name(), parent(), file_stem(), file_ext()
       Array:       length(), sip_hash_64(), euclidean_distance(),
                    cosine_distance()
       Window:      row_number(), rank(), dense_rank(), first()

    Example:
    ```py
     dc.mutate(
        area=Column("image.height") * Column("image.width"),
        extension=file_ext(Column("file.path")),
        dist=cosine_distance(embedding_text, embedding_image)
    )
    ```

    Window function example:
    ```py
    window = func.window(partition_by="file.parent", order_by="file.size")
    dc.mutate(
        row_number=func.row_number().over(window),
    )
    ```

    This method can be also used to rename signals. If the Column("name") provided
    as value for the new signal - the old column will be dropped. Otherwise a new
    column is created.

    Example:
    ```py
     dc.mutate(
        newkey=Column("oldkey")
    )
    ```
    """
    from sqlalchemy.sql.sqltypes import NullType

    primitives = (bool, str, int, float)

    for col_name, expr in kwargs.items():
        if not isinstance(expr, (*primitives, Column, Func)) and isinstance(
            expr.type, NullType
        ):
            raise DataChainColumnError(
                col_name, f"Cannot infer type with expression {expr}"
            )

    mutated = {}
    schema = self.signals_schema
    for name, value in kwargs.items():
        if isinstance(value, Column):
            # renaming existing column
            for signal in schema.db_signals(name=value.name, as_columns=True):
                mutated[signal.name.replace(value.name, name, 1)] = signal  # type: ignore[union-attr]
        elif isinstance(value, Func):
            # adding new signal
            mutated[name] = value.get_column(schema)
        elif isinstance(value, primitives):
            # adding simple python constant primitives like str, int, float, bool
            val = literal(value)
            val.type = python_to_sql(type(value))()
            mutated[name] = val  # type: ignore[assignment]
        else:
            # adding new signal
            mutated[name] = value

    return self._evolve(
        query=self._query.mutate(**mutated), signal_schema=schema.mutate(kwargs)
    )

offset 

offset(offset: int) -> Self

Return the results starting with the offset row.

If the chain is unordered, which rows are skipped in undefined. If the chain has less than offset rows, the result is an empty chain.

Parameters:

  • offset (int) –

    Number of rows to skip.

Source code in datachain/lib/dc/datachain.py 
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
def offset(self, offset: int) -> "Self":
    """Return the results starting with the offset row.

    If the chain is unordered, which rows are skipped in undefined.
    If the chain has less than `offset` rows, the result is an empty chain.

    Parameters:
        offset (int): Number of rows to skip.
    """
    return self._evolve(query=self._query.offset(offset))

order_by 

order_by(*args, descending: bool = False) -> Self

Orders by specified set of columns.

Parameters:

  • descending (bool, default: False ) –

    Whether to sort in descending order or not.

Example 
dc.order_by("similarity_score", descending=True).limit(10)
Note

Order is not guaranteed when steps are added after an order_by statement. I.e. when using read_dataset an order_by statement should be used if the order of the records in the chain is important. Using order_by directly before limit, to_list and similar methods will give expected results. See https://github.com/iterative/datachain/issues/477 for further details.

Source code in datachain/lib/dc/datachain.py 
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
@resolve_columns
def order_by(self, *args, descending: bool = False) -> "Self":
    """Orders by specified set of columns.

    Parameters:
        descending (bool): Whether to sort in descending order or not.

    Example:
        ```py
        dc.order_by("similarity_score", descending=True).limit(10)
        ```

    Note:
        Order is not guaranteed when steps are added after an `order_by` statement.
        I.e. when using `read_dataset` an `order_by` statement should be used if
        the order of the records in the chain is important.
        Using `order_by` directly before `limit`, `to_list` and similar methods
        will give expected results.
        See https://github.com/iterative/datachain/issues/477 for further details.
    """
    if descending:
        args = tuple(sqlalchemy.desc(a) for a in args)

    return self._evolve(query=self._query.order_by(*args))

parse_tabular 

parse_tabular(
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    **kwargs
) -> Self

Generate chain from list of tabular files.

Parameters:

  • output –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • column –

    Generated column name.

  • model_name –

    Generated model name.

  • source –

    Whether to include info about the source file.

  • nrows –

    Optional row limit.

  • kwargs –

    Parameters to pass to pyarrow.dataset.dataset.

Example

Reading a json lines file: 

import datachain as dc
chain = dc.read_storage("s3://mybucket/file.jsonl")
chain = chain.parse_tabular(format="json")

Reading a filtered list of files as a dataset: 

import datachain as dc

chain = dc.read_storage("s3://mybucket")
chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
chain = chain.parse_tabular(format="json")

Source code in datachain/lib/dc/datachain.py 
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
def parse_tabular(
    self,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    **kwargs,
) -> "Self":
    """Generate chain from list of tabular files.

    Parameters:
        output : Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        column : Generated column name.
        model_name : Generated model name.
        source : Whether to include info about the source file.
        nrows : Optional row limit.
        kwargs : Parameters to pass to pyarrow.dataset.dataset.

    Example:
        Reading a json lines file:
        ```py
        import datachain as dc
        chain = dc.read_storage("s3://mybucket/file.jsonl")
        chain = chain.parse_tabular(format="json")
        ```

        Reading a filtered list of files as a dataset:
        ```py
        import datachain as dc

        chain = dc.read_storage("s3://mybucket")
        chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
        chain = chain.parse_tabular(format="json")
        ```
    """
    from pyarrow.dataset import CsvFileFormat, JsonFileFormat

    from datachain.lib.arrow import (
        ArrowGenerator,
        fix_pyarrow_format,
        infer_schema,
        schema_to_output,
    )

    parse_options = kwargs.pop("parse_options", None)
    if format := kwargs.get("format"):
        kwargs["format"] = fix_pyarrow_format(format, parse_options)

    if (
        nrows
        and format not in ["csv", "json"]
        and not isinstance(format, (CsvFileFormat, JsonFileFormat))
    ):
        raise DatasetPrepareError(
            self.name,
            "error in `parse_tabular` - "
            "`nrows` only supported for csv and json formats.",
        )

    if "file" not in self.schema or not self.count():
        raise DatasetPrepareError(self.name, "no files to parse.")

    schema = None
    col_names = output if isinstance(output, Sequence) else None
    if col_names or not output:
        try:
            schema = infer_schema(self, **kwargs, parse_options=parse_options)
            output, _ = schema_to_output(schema, col_names)
        except ValueError as e:
            raise DatasetPrepareError(self.name, e) from e

    if isinstance(output, dict):
        model_name = model_name or column or ""
        model = dict_to_data_model(model_name, output)
        output = model
    else:
        model = output  # type: ignore[assignment]

    if column:
        output = {column: model}  # type: ignore[dict-item]
    elif isinstance(output, type(BaseModel)):
        output = {
            name: info.annotation  # type: ignore[misc]
            for name, info in output.model_fields.items()
        }

    if source:
        output = {"source": ArrowRow} | output  # type: ignore[assignment,operator]

    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}
    return self.settings(**settings).gen(  # type: ignore[arg-type]
        ArrowGenerator(
            schema,
            model,
            source,
            nrows,
            parse_options=parse_options,
            **kwargs,
        ),
        output=output,
    )

persist 

persist() -> Self

Saves temporary chain that will be removed after the process ends. Temporary datasets are useful for optimization, for example when we have multiple chains starting with identical sub-chain. We can then persist that common chain and use it to calculate other chains, to avoid re-calculation every time. It returns the chain itself.

Source code in datachain/lib/dc/datachain.py 
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
def persist(self) -> "Self":
    """Saves temporary chain that will be removed after the process ends.
    Temporary datasets are useful for optimization, for example when we have
    multiple chains starting with identical sub-chain. We can then persist that
    common chain and use it to calculate other chains, to avoid re-calculation
    every time.
    It returns the chain itself.
    """
    schema = self.signals_schema.clone_without_sys_signals().serialize()
    project = self.session.catalog.metastore.get_project(
        self.project_name,
        self.namespace_name,
        create=True,
    )
    return self._evolve(
        query=self._query.save(project=project, feature_schema=schema)
    )

print_schema 

print_schema(file: Optional[IO] = None) -> None

Print schema of the chain.

Source code in datachain/lib/dc/datachain.py 
280
281
282
def print_schema(self, file: Optional[IO] = None) -> None:
    """Print schema of the chain."""
    self._effective_signals_schema.print_tree(file=file)

reset_settings 

reset_settings(settings: Optional[Settings] = None) -> Self

Reset all settings to default values.

Source code in datachain/lib/dc/datachain.py 
364
365
366
367
def reset_settings(self, settings: Optional[Settings] = None) -> "Self":
    """Reset all settings to default values."""
    self._settings = settings if settings else Settings()
    return self

sample 

sample(n: int) -> Self

Return a random sample from the chain.

Parameters:

  • n (int) –

    Number of samples to draw.

Note

Samples are not deterministic, and streamed/paginated queries or multiple workers will draw samples with replacement.

Source code in datachain/lib/dc/datachain.py 
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
def sample(self, n: int) -> "Self":
    """Return a random sample from the chain.

    Parameters:
        n: Number of samples to draw.

    Note:
        Samples are not deterministic, and streamed/paginated queries or
        multiple workers will draw samples with replacement.
    """
    return self._evolve(query=self._query.sample(n))

save 

save(
    name: str,
    version: Optional[str] = None,
    description: Optional[str] = None,
    attrs: Optional[list[str]] = None,
    update_version: Optional[str] = "patch",
    **kwargs
) -> DataChain

Save to a Dataset. It returns the chain itself.

Parameters:

  • name –

    dataset name. It can be full name consisting of namespace and project, but it can also be just a regular dataset name in which case we are taking namespace and project from settings, if they are defined there, or default ones instead.

  • version –

    version of a dataset. If version is not specified and dataset already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.

  • description –

    description of a dataset.

  • attrs –

    attributes of a dataset. They can be without value, e.g "NLP", or with a value, e.g "location=US".

  • update_version (Optional[str], default: 'patch' ) –

    which part of the dataset version to automatically increase. Available values: major, minor or patch. Default is patch.

Source code in datachain/lib/dc/datachain.py 
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
def save(  # type: ignore[override]
    self,
    name: str,
    version: Optional[str] = None,
    description: Optional[str] = None,
    attrs: Optional[list[str]] = None,
    update_version: Optional[str] = "patch",
    **kwargs,
) -> "DataChain":
    """Save to a Dataset. It returns the chain itself.

    Parameters:
        name : dataset name. It can be full name consisting of namespace and
            project, but it can also be just a regular dataset name in which
            case we are taking namespace and project from settings, if they
            are defined there, or default ones instead.
        version : version of a dataset. If version is not specified and dataset
            already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.
        description : description of a dataset.
        attrs : attributes of a dataset. They can be without value, e.g "NLP",
            or with a value, e.g "location=US".
        update_version: which part of the dataset version to automatically increase.
            Available values: `major`, `minor` or `patch`. Default is `patch`.
    """
    catalog = self.session.catalog
    if version is not None:
        semver.validate(version)

    if update_version is not None and update_version not in [
        "patch",
        "major",
        "minor",
    ]:
        raise ValueError(
            "update_version can have one of the following values: major, minor or"
            " patch"
        )

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        namespace_name=self._settings.namespace,
        project_name=self._settings.project,
    )

    try:
        project = self.session.catalog.metastore.get_project(
            project_name,
            namespace_name,
            create=self.session.catalog.metastore.project_allowed_to_create,
        )
    except ProjectNotFoundError as e:
        # not being able to create it as creation is not allowed
        raise ProjectCreateNotAllowedError("Creating project is not allowed") from e

    schema = self.signals_schema.clone_without_sys_signals().serialize()

    # Handle retry and delta functionality
    if self.delta and name:
        from datachain.delta import delta_retry_update

        # Delta chains must have delta_on defined (ensured by _as_delta method)
        assert self._delta_on is not None, "Delta chain must have delta_on defined"

        result_ds, dependencies, has_changes = delta_retry_update(
            self,
            namespace_name,
            project_name,
            name,
            on=self._delta_on,
            right_on=self._delta_result_on,
            compare=self._delta_compare,
            delta_retry=self._delta_retry,
        )

        if result_ds:
            return self._evolve(
                query=result_ds._query.save(
                    name=name,
                    version=version,
                    project=project,
                    feature_schema=schema,
                    dependencies=dependencies,
                    **kwargs,
                )
            )

        if not has_changes:
            # sources have not been changed so new version of resulting dataset
            # would be the same as previous one. To avoid duplicating exact
            # datasets, we won't create new version of it and we will return
            # current latest version instead.
            from .datasets import read_dataset

            return read_dataset(name, **kwargs)

    return self._evolve(
        query=self._query.save(
            name=name,
            version=version,
            project=project,
            description=description,
            attrs=attrs,
            feature_schema=schema,
            update_version=update_version,
            **kwargs,
        )
    )

select 

select(*args: str, _sys: bool = True) -> Self

Select only a specified set of signals.

Source code in datachain/lib/dc/datachain.py 
1004
1005
1006
1007
1008
1009
1010
1011
1012
def select(self, *args: str, _sys: bool = True) -> "Self":
    """Select only a specified set of signals."""
    new_schema = self.signals_schema.resolve(*args)
    if self._sys and _sys:
        new_schema = SignalSchema({"sys": Sys}) | new_schema
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

select_except 

select_except(*args: str) -> Self

Select all the signals expect the specified signals.

Source code in datachain/lib/dc/datachain.py 
1014
1015
1016
1017
1018
1019
1020
def select_except(self, *args: str) -> "Self":
    """Select all the signals expect the specified signals."""
    new_schema = self.signals_schema.select_except_signals(*args)
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

settings 

settings(
    cache=None,
    parallel=None,
    workers=None,
    min_task_size=None,
    prefetch: Optional[int] = None,
    sys: Optional[bool] = None,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
) -> Self

Change settings for chain.

This function changes specified settings without changing not specified ones. It returns chain, so, it can be chained later with next operation.

Parameters:

  • cache –

    data caching (default=False)

  • parallel –

    number of thread for processors. True is a special value to enable all available CPUs (default=1)

  • workers –

    number of distributed workers. Only for Studio mode. (default=1)

  • min_task_size –

    minimum number of tasks (default=1)

  • prefetch (Optional[int], default: None ) –

    number of workers to use for downloading files in advance. This is enabled by default and uses 2 workers. To disable prefetching, set it to 0.

  • namespace (Optional[str], default: None ) –

    namespace name.

  • project (Optional[str], default: None ) –

    project name.

Example 
chain = (
    chain
    .settings(cache=True, parallel=8)
    .map(laion=process_webdataset(spec=WDSLaion), params="file")
)
Source code in datachain/lib/dc/datachain.py 
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
def settings(
    self,
    cache=None,
    parallel=None,
    workers=None,
    min_task_size=None,
    prefetch: Optional[int] = None,
    sys: Optional[bool] = None,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
) -> "Self":
    """Change settings for chain.

    This function changes specified settings without changing not specified ones.
    It returns chain, so, it can be chained later with next operation.

    Parameters:
        cache : data caching (default=False)
        parallel : number of thread for processors. True is a special value to
            enable all available CPUs (default=1)
        workers : number of distributed workers. Only for Studio mode. (default=1)
        min_task_size : minimum number of tasks (default=1)
        prefetch: number of workers to use for downloading files in advance.
                  This is enabled by default and uses 2 workers.
                  To disable prefetching, set it to 0.
        namespace: namespace name.
        project: project name.

    Example:
        ```py
        chain = (
            chain
            .settings(cache=True, parallel=8)
            .map(laion=process_webdataset(spec=WDSLaion), params="file")
        )
        ```
    """
    if sys is None:
        sys = self._sys
    settings = copy.copy(self._settings)
    settings.add(
        Settings(
            cache, parallel, workers, min_task_size, prefetch, namespace, project
        )
    )
    return self._evolve(settings=settings, _sys=sys)

setup 

setup(**kwargs) -> Self

Setup variables to pass to UDF functions.

Use before running map/gen/agg/batch_map to save an object and pass it as an argument to the UDF.

The value must be a callable (a lambda: <value> syntax can be used to quickly create one) that returns the object to be passed to the UDF. It is evaluated lazily when UDF is running, in case of multiple machines the callable is run on a worker machine.

Example 
import anthropic
from anthropic.types import Message
import datachain as dc

(
    dc.read_storage(DATA, type="text")
    .settings(parallel=4, cache=True)

    # Setup Anthropic client and pass it to the UDF below automatically
    # The value is callable (see the note above)
    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))

    .map(
        claude=lambda client, file: client.messages.create(
            model=MODEL,
            system=PROMPT,
            messages=[{"role": "user", "content": file.get_value()}],
        ),
        output=Message,
    )
)
Source code in datachain/lib/dc/datachain.py 
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
def setup(self, **kwargs) -> "Self":
    """Setup variables to pass to UDF functions.

    Use before running map/gen/agg/batch_map to save an object and pass it as an
    argument to the UDF.

    The value must be a callable (a `lambda: <value>` syntax can be used to quickly
    create one) that returns the object to be passed to the UDF. It is evaluated
    lazily when UDF is running, in case of multiple machines the callable is run on
    a worker machine.

    Example:
        ```py
        import anthropic
        from anthropic.types import Message
        import datachain as dc

        (
            dc.read_storage(DATA, type="text")
            .settings(parallel=4, cache=True)

            # Setup Anthropic client and pass it to the UDF below automatically
            # The value is callable (see the note above)
            .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))

            .map(
                claude=lambda client, file: client.messages.create(
                    model=MODEL,
                    system=PROMPT,
                    messages=[{"role": "user", "content": file.get_value()}],
                ),
                output=Message,
            )
        )
        ```
    """
    intersection = set(self._setup.keys()) & set(kwargs.keys())
    if intersection:
        keys = ", ".join(intersection)
        raise DatasetPrepareError(self.name, f"this value(s) already setup: {keys}")

    self._setup = self._setup | kwargs
    return self

show 

show(
    limit: int = 20,
    flatten: bool = False,
    transpose: bool = False,
    truncate: bool = True,
    include_hidden: bool = False,
) -> None

Show a preview of the chain results.

Parameters:

  • limit (int, default: 20 ) –

    How many rows to show.

  • flatten (bool, default: False ) –

    Whether to use a multiindex or flatten column names.

  • transpose (bool, default: False ) –

    Whether to transpose rows and columns.

  • truncate (bool, default: True ) –

    Whether or not to truncate the contents of columns.

  • include_hidden (bool, default: False ) –

    Whether to include hidden columns.

Source code in datachain/lib/dc/datachain.py 
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
def show(
    self,
    limit: int = 20,
    flatten: bool = False,
    transpose: bool = False,
    truncate: bool = True,
    include_hidden: bool = False,
) -> None:
    """Show a preview of the chain results.

    Parameters:
        limit: How many rows to show.
        flatten: Whether to use a multiindex or flatten column names.
        transpose: Whether to transpose rows and columns.
        truncate: Whether or not to truncate the contents of columns.
        include_hidden: Whether to include hidden columns.
    """
    import pandas as pd

    dc = self.limit(limit) if limit > 0 else self  # type: ignore[misc]
    df = dc.to_pandas(flatten, include_hidden=include_hidden)

    if df.empty:
        print("Empty result")
        print(f"Columns: {list(df.columns)}")
        return

    if transpose:
        df = df.T

    options: list = [
        "display.max_columns",
        None,
        "display.multi_sparse",
        False,
    ]

    try:
        if columns := os.get_terminal_size().columns:
            options.extend(["display.width", columns])
    except OSError:
        pass

    if not truncate:
        options.extend(["display.max_colwidth", None])

    with pd.option_context(*options):
        if inside_notebook():
            from IPython.display import display

            display(df)
        else:
            print(df)

    if len(df) == limit:
        print(f"\n[Limited by {len(df)} rows]")

shuffle 

shuffle() -> Self

Shuffle the rows of the chain deterministically.

Source code in datachain/lib/dc/datachain.py 
2445
2446
2447
def shuffle(self) -> "Self":
    """Shuffle the rows of the chain deterministically."""
    return self.order_by("sys.rand")

subtract 

subtract(
    other: DataChain,
    on: Optional[Union[str, Sequence[str]]] = None,
    right_on: Optional[Union[str, Sequence[str]]] = None,
) -> Self

Remove rows that appear in another chain.

Parameters:

  • other (DataChain) –

    chain whose rows will be removed from self

  • on (Optional[Union[str, Sequence[str]]], default: None ) –

    columns to consider for determining row equality in self. If unspecified, defaults to all common columns between self and other.

  • right_on (Optional[Union[str, Sequence[str]]], default: None ) –

    columns to consider for determining row equality in other. If unspecified, defaults to the same values as on.

Source code in datachain/lib/dc/datachain.py 
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
def subtract(  # type: ignore[override]
    self,
    other: "DataChain",
    on: Optional[Union[str, Sequence[str]]] = None,
    right_on: Optional[Union[str, Sequence[str]]] = None,
) -> "Self":
    """Remove rows that appear in another chain.

    Parameters:
        other: chain whose rows will be removed from `self`
        on: columns to consider for determining row equality in `self`.
            If unspecified, defaults to all common columns
            between `self` and `other`.
        right_on: columns to consider for determining row equality in `other`.
            If unspecified, defaults to the same values as `on`.
    """
    if isinstance(on, str):
        if not on:
            raise DataChainParamsError("'on' cannot be an empty string")
        on = [on]
    elif isinstance(on, Sequence):
        if not on or any(not col for col in on):
            raise DataChainParamsError("'on' cannot contain empty strings")

    if isinstance(right_on, str):
        if not right_on:
            raise DataChainParamsError("'right_on' cannot be an empty string")
        right_on = [right_on]
    elif isinstance(right_on, Sequence):
        if not right_on or any(not col for col in right_on):
            raise DataChainParamsError("'right_on' cannot contain empty strings")

    if on is None and right_on is None:
        other_columns = set(other._effective_signals_schema.db_signals())
        signals = [
            c
            for c in self._effective_signals_schema.db_signals()
            if c in other_columns
        ]
        if not signals:
            raise DataChainParamsError("subtract(): no common columns")
    elif on is not None and right_on is None:
        right_on = on
        signals = list(self.signals_schema.resolve(*on).db_signals())
    elif on is None and right_on is not None:
        raise DataChainParamsError(
            "'on' must be specified when 'right_on' is provided"
        )
    else:
        if not isinstance(on, Sequence) or not isinstance(right_on, Sequence):
            raise TypeError(
                "'on' and 'right_on' must be 'str' or 'Sequence' object"
            )
        if len(on) != len(right_on):
            raise DataChainParamsError(
                "'on' and 'right_on' must have the same length"
            )
        signals = list(
            zip(
                self.signals_schema.resolve(*on).db_signals(),
                other.signals_schema.resolve(*right_on).db_signals(),
            )  # type: ignore[arg-type]
        )
    return self._evolve(query=self._query.subtract(other._query, signals))  # type: ignore[arg-type]

sum 

sum(col: str) -> StandardType

Compute the sum of a column.

Parameters:

  • col (str) –

    The column to compute the sum for.

Returns:

  • StandardType –

    The sum of the column values.

Example 
total_size = chain.sum("file.size")
print(f"Total size: {total_size}")
Source code in datachain/lib/dc/datachain.py 
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
def sum(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the sum of a column.

    Parameters:
        col: The column to compute the sum for.

    Returns:
        The sum of the column values.

    Example:
        ```py
        total_size = chain.sum("file.size")
        print(f"Total size: {total_size}")
        ```
    """
    return self._extend_to_data_model("sum", col)

to_columnar_data_with_names 

to_columnar_data_with_names(
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
) -> tuple[list[str], Iterator[list[list[Any]]]]

Returns column names and the results as an iterator that provides chunks, with each chunk containing a list of columns, where each column contains a list of the row values for that column in that chunk. Useful for columnar data formats, such as parquet or other OLAP databases.

Source code in datachain/lib/dc/datachain.py 
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
def to_columnar_data_with_names(
    self, chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE
) -> tuple[list[str], Iterator[list[list[Any]]]]:
    """Returns column names and the results as an iterator that provides chunks,
    with each chunk containing a list of columns, where each column contains a
    list of the row values for that column in that chunk. Useful for columnar data
    formats, such as parquet or other OLAP databases.
    """
    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self._leaf_values()

    def column_chunks() -> Iterator[list[list[Any]]]:
        for chunk_iter in batched_it(results_iter, chunk_size):
            columns: list[list[Any]] = [[] for _ in column_names]
            for row in chunk_iter:
                for i, col in enumerate(columns):
                    col.append(row[i])
            yield columns

    return column_names, column_chunks()

to_csv 

to_csv(
    path: Union[str, PathLike[str]],
    delimiter: str = ",",
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs
) -> None

Save chain to a csv (comma-separated values) file.

Parameters:

  • path –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • delimiter –

    Delimiter to use for the resulting file.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
def to_csv(
    self,
    path: Union[str, os.PathLike[str]],
    delimiter: str = ",",
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs,
) -> None:
    """Save chain to a csv (comma-separated values) file.

    Parameters:
        path : Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        delimiter : Delimiter to use for the resulting file.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import csv

    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self._leaf_values()

    with opener(path, "w", newline="") as f:
        writer = csv.writer(f, delimiter=delimiter, **kwargs)
        writer.writerow(column_names)

        for row in results_iter:
            writer.writerow(row)

to_iter 

to_iter(*cols: str) -> Iterator[tuple[DataValue, ...]]

Yields rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Yields:

Example

Iterating over all rows: 

for row in ds.to_iter():
    print(row)

DataChain is iterable and can be used in a for loop directly which is equivalent to ds.to_iter(): 

for row in ds:
    print(row)

Iterating over all rows with selected columns: 

for name, size in ds.to_iter("file.path", "file.size"):
    print(name, size)

Iterating over a single column: 

for (file,) in ds.to_iter("file.path"):
    print(file)

Source code in datachain/lib/dc/datachain.py 
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
def to_iter(self, *cols: str) -> Iterator[tuple[DataValue, ...]]:
    """Yields rows of values, optionally limited to the specified columns.

    Args:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Yields:
        (tuple[DataType, ...]): Yields a tuple of items for each row.

    Example:
        Iterating over all rows:
        ```py
        for row in ds.to_iter():
            print(row)
        ```

        DataChain is iterable and can be used in a for loop directly which is
        equivalent to `ds.to_iter()`:
        ```py
        for row in ds:
            print(row)
        ```

        Iterating over all rows with selected columns:
        ```py
        for name, size in ds.to_iter("file.path", "file.size"):
            print(name, size)
        ```

        Iterating over a single column:
        ```py
        for (file,) in ds.to_iter("file.path"):
            print(file)
        ```
    """
    chain = self.select(*cols) if cols else self
    signals_schema = chain._effective_signals_schema
    db_signals = signals_schema.db_signals()
    with self._query.ordered_select(*db_signals).as_iterable() as rows:
        for row in rows:
            ret = signals_schema.row_to_features(
                row, catalog=chain.session.catalog, cache=chain._settings.cache
            )
            yield tuple(ret)

to_json 

to_json(
    path: Union[str, PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
    include_outer_list: bool = True,
) -> None

Save chain to a JSON file.

Parameters:

  • path –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

  • include_outer_list –

    Sets whether to include an outer list for all rows. Setting this to True makes the file valid JSON, while False instead writes in the JSON lines format.

Source code in datachain/lib/dc/datachain.py 
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
def to_json(
    self,
    path: Union[str, os.PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
    include_outer_list: bool = True,
) -> None:
    """Save chain to a JSON file.

    Parameters:
        path : Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
        include_outer_list : Sets whether to include an outer list for all rows.
            Setting this to True makes the file valid JSON, while False instead
            writes in the JSON lines format.
    """
    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    headers = [list(filter(None, header)) for header in headers]

    is_first = True

    with opener(path, "wb") as f:
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"[\n")
        for row in self._leaf_values():
            if not is_first:
                if include_outer_list:
                    # This makes the file JSON instead of JSON lines.
                    f.write(b",\n")
                else:
                    f.write(b"\n")
            else:
                is_first = False
            f.write(orjson.dumps(row_to_nested_dict(headers, row)))
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"\n]\n")

to_jsonl 

to_jsonl(
    path: Union[str, PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
) -> None

Save chain to a JSON lines file.

Parameters:

  • path –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
def to_jsonl(
    self,
    path: Union[str, os.PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
) -> None:
    """Save chain to a JSON lines file.

    Parameters:
        path : Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    self.to_json(path, fs_kwargs, include_outer_list=False)

to_list 

to_list(*cols: str) -> list[tuple[DataValue, ...]]

Returns a list of rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Returns:

  • list[tuple[DataValue, ...]] –

    list[tuple[DataType, ...]]: Returns a list of tuples of items for each row.

Example

Getting all rows as a list: 

rows = dc.to_list()
print(rows)

Getting all rows with selected columns as a list: 

name_size_pairs = dc.to_list("file.path", "file.size")
print(name_size_pairs)

Getting a single column as a list: 

files = dc.to_list("file.path")
print(files)  # Returns list of 1-tuples

Source code in datachain/lib/dc/datachain.py 
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
def to_list(self, *cols: str) -> list[tuple[DataValue, ...]]:
    """Returns a list of rows of values, optionally limited to the specified
    columns.

    Parameters:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Returns:
        list[tuple[DataType, ...]]: Returns a list of tuples of items for each row.

    Example:
        Getting all rows as a list:
        ```py
        rows = dc.to_list()
        print(rows)
        ```

        Getting all rows with selected columns as a list:
        ```py
        name_size_pairs = dc.to_list("file.path", "file.size")
        print(name_size_pairs)
        ```

        Getting a single column as a list:
        ```py
        files = dc.to_list("file.path")
        print(files)  # Returns list of 1-tuples
        ```
    """
    return list(self.to_iter(*cols))

to_pandas 

to_pandas(
    flatten: bool = False, include_hidden: bool = True
) -> DataFrame

Return a pandas DataFrame from the chain.

Parameters:

  • flatten (bool, default: False ) –

    Whether to use a multiindex or flatten column names.

  • include_hidden (bool, default: True ) –

    Whether to include hidden columns.

Returns:

  • DataFrame –

    pd.DataFrame: A pandas DataFrame representation of the chain.

Source code in datachain/lib/dc/datachain.py 
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
def to_pandas(
    self,
    flatten: bool = False,
    include_hidden: bool = True,
) -> "pd.DataFrame":
    """Return a pandas DataFrame from the chain.

    Parameters:
        flatten: Whether to use a multiindex or flatten column names.
        include_hidden: Whether to include hidden columns.

    Returns:
        pd.DataFrame: A pandas DataFrame representation of the chain.
    """
    import pandas as pd

    headers, max_length = self._effective_signals_schema.get_headers_with_length(
        include_hidden=include_hidden
    )
    if flatten or max_length < 2:
        columns = [".".join(filter(None, header)) for header in headers]
    else:
        columns = pd.MultiIndex.from_tuples(map(tuple, headers))

    results = self.results(include_hidden=include_hidden)
    return pd.DataFrame.from_records(results, columns=columns)

to_parquet 

to_parquet(
    path: Union[str, PathLike[str], BinaryIO],
    partition_cols: Optional[Sequence[str]] = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs
) -> None

Save chain to parquet file with SignalSchema metadata.

Parameters:

  • path –

    Path or a file-like binary object to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • partition_cols –

    Column names by which to partition the dataset.

  • chunk_size –

    The chunk size of results to read and convert to columnar data, to avoid running out of memory.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
def to_parquet(
    self,
    path: Union[str, os.PathLike[str], BinaryIO],
    partition_cols: Optional[Sequence[str]] = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs,
) -> None:
    """Save chain to parquet file with SignalSchema metadata.

    Parameters:
        path : Path or a file-like binary object to save the file. This supports
            local paths as well as remote paths, such as s3:// or hf:// with fsspec.
        partition_cols : Column names by which to partition the dataset.
        chunk_size : The chunk size of results to read and convert to columnar
            data, to avoid running out of memory.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import pyarrow as pa
    import pyarrow.parquet as pq

    from datachain.lib.arrow import DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY

    fsspec_fs = None

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        if path.startswith("file://"):
            # pyarrow does not handle file:// uris, and needs a direct path instead.
            from urllib.parse import urlparse

            path = urlparse(path).path
            if sys.platform == "win32":
                path = os.path.normpath(path.lstrip("/"))

        fsspec_fs = client.create_fs(**fs_kwargs)

    _partition_cols = list(partition_cols) if partition_cols else None
    signal_schema_metadata = orjson.dumps(
        self._effective_signals_schema.serialize()
    )

    column_names, column_chunks = self.to_columnar_data_with_names(chunk_size)

    parquet_schema = None
    parquet_writer = None
    first_chunk = True

    for chunk in column_chunks:
        # pyarrow infers the best parquet schema from the python types of
        # the input data.
        table = pa.Table.from_pydict(
            dict(zip(column_names, chunk)),
            schema=parquet_schema,
        )

        # Preserve any existing metadata, and add the DataChain SignalSchema.
        existing_metadata = table.schema.metadata or {}
        merged_metadata = {
            **existing_metadata,
            DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY: signal_schema_metadata,
        }
        table = table.replace_schema_metadata(merged_metadata)
        parquet_schema = table.schema

        if _partition_cols:
            # Write to a partitioned parquet dataset.
            pq.write_to_dataset(
                table,
                root_path=path,
                partition_cols=_partition_cols,
                filesystem=fsspec_fs,
                **kwargs,
            )
        else:
            if first_chunk:
                # Write to a single parquet file.
                parquet_writer = pq.ParquetWriter(
                    path, parquet_schema, filesystem=fsspec_fs, **kwargs
                )
                first_chunk = False

            assert parquet_writer
            parquet_writer.write_table(table)

    if parquet_writer:
        parquet_writer.close()

to_pytorch 

to_pytorch(
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
)

Convert to pytorch dataset format.

Parameters:

  • transform (Transform, default: None ) –

    Torchvision transforms to apply to the dataset.

  • tokenizer (Callable, default: None ) –

    Tokenizer to use to tokenize text values.

  • tokenizer_kwargs (dict, default: None ) –

    Additional kwargs to pass when calling tokenizer.

  • num_samples (int, default: 0 ) –

    Number of random samples to draw for each epoch. This argument is ignored if num_samples=0 (the default).

  • remove_prefetched (bool, default: False ) –

    Whether to remove prefetched files after reading.

Example 
from torch.utils.data import DataLoader
loader = DataLoader(
    chain.select("file", "label").to_pytorch(),
    batch_size=16
)
Source code in datachain/lib/dc/datachain.py 
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
def to_pytorch(
    self,
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
):
    """Convert to pytorch dataset format.

    Args:
        transform (Transform): Torchvision transforms to apply to the dataset.
        tokenizer (Callable): Tokenizer to use to tokenize text values.
        tokenizer_kwargs (dict): Additional kwargs to pass when calling tokenizer.
        num_samples (int): Number of random samples to draw for each epoch.
            This argument is ignored if `num_samples=0` (the default).
        remove_prefetched (bool): Whether to remove prefetched files after reading.

    Example:
        ```py
        from torch.utils.data import DataLoader
        loader = DataLoader(
            chain.select("file", "label").to_pytorch(),
            batch_size=16
        )
        ```
    """
    from datachain.torch import PytorchDataset

    if self._query.attached:
        chain = self
    else:
        chain = self.persist()
    assert chain.name is not None  # for mypy
    return PytorchDataset(
        chain.name,
        chain.version,
        catalog=self.session.catalog,
        transform=transform,
        tokenizer=tokenizer,
        tokenizer_kwargs=tokenizer_kwargs,
        num_samples=num_samples,
        dc_settings=chain._settings,
        remove_prefetched=remove_prefetched,
    )

to_records 

to_records() -> list[dict[str, Any]]

Convert every row to a dictionary.

Source code in datachain/lib/dc/datachain.py 
1323
1324
1325
1326
1327
1328
1329
def to_records(self) -> list[dict[str, Any]]:
    """Convert every row to a dictionary."""

    def to_dict(cols: list[str], row: tuple[Any, ...]) -> dict[str, Any]:
        return dict(zip(cols, row))

    return self.results(row_factory=to_dict)

to_storage 

to_storage(
    output: Union[str, PathLike[str]],
    signal: str = "file",
    placement: ExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
    anon: bool = False,
    client_config: Optional[dict] = None,
) -> None

Export files from a specified signal to a directory. Files can be exported to a local or cloud directory.

Parameters:

  • output (Union[str, PathLike[str]]) –

    Path to the target directory for exporting files.

  • signal (str, default: 'file' ) –

    Name of the signal to export files from.

  • placement (ExportPlacement, default: 'fullpath' ) –

    The method to use for naming exported files. The possible values are: "filename", "etag", "fullpath", and "checksum".

  • link_type (Literal['copy', 'symlink'], default: 'copy' ) –

    Method to use for exporting files. Falls back to 'copy' if symlinking fails.

  • num_threads –

    number of threads to use for exporting files. By default it uses 5 threads.

  • anon (bool, default: False ) –

    If true, we will treat cloud bucket as public one

  • client_config (Optional[dict], default: None ) –

    Optional configuration for the destination storage client

Example

Cross cloud transfer 

import datachain as dc

ds = dc.read_storage("s3://mybucket")
ds.to_storage("gs://mybucket", placement="filename")

Source code in datachain/lib/dc/datachain.py 
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
def to_storage(
    self,
    output: Union[str, os.PathLike[str]],
    signal: str = "file",
    placement: FileExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
    anon: bool = False,
    client_config: Optional[dict] = None,
) -> None:
    """Export files from a specified signal to a directory. Files can be
    exported to a local or cloud directory.

    Args:
        output: Path to the target directory for exporting files.
        signal: Name of the signal to export files from.
        placement: The method to use for naming exported files.
            The possible values are: "filename", "etag", "fullpath", and "checksum".
        link_type: Method to use for exporting files.
            Falls back to `'copy'` if symlinking fails.
        num_threads : number of threads to use for exporting files.
            By default it uses 5 threads.
        anon: If true, we will treat cloud bucket as public one
        client_config: Optional configuration for the destination storage client

    Example:
        Cross cloud transfer
        ```py
        import datachain as dc

        ds = dc.read_storage("s3://mybucket")
        ds.to_storage("gs://mybucket", placement="filename")
        ```
    """
    if placement == "filename" and (
        self._query.distinct(pathfunc.name(C(f"{signal}__path"))).count()
        != self._query.count()
    ):
        raise ValueError("Files with the same name found")

    if anon:
        client_config = (client_config or {}) | {"anon": True}

    progress_bar = tqdm(
        desc=f"Exporting files to {output}: ",
        unit=" files",
        unit_scale=True,
        unit_divisor=10,
        total=self.count(),
        leave=False,
    )
    file_exporter = FileExporter(
        output,
        placement,
        self._settings.cache if self._settings else False,
        link_type,
        max_threads=num_threads or 1,
        client_config=client_config,
    )
    file_exporter.run(self.to_values(signal), progress_bar)

to_values 

to_values(col: str) -> list[DataValue]

Returns a flat list of values from a single column.

Parameters:

  • col (str) –

    The name of the column to extract values from.

Returns:

  • list[DataValue] –

    list[DataValue]: Returns a flat list of values from the specified column.

Example

Getting all values from a single column: 

file_paths = dc.to_values("file.path")
print(file_paths)  # Returns list of strings

Getting all file sizes: 

sizes = dc.to_values("file.size")
print(sizes)  # Returns list of integers

Source code in datachain/lib/dc/datachain.py 
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
def to_values(self, col: str) -> list[DataValue]:
    """Returns a flat list of values from a single column.

    Parameters:
        col: The name of the column to extract values from.

    Returns:
        list[DataValue]: Returns a flat list of values from the specified column.

    Example:
        Getting all values from a single column:
        ```py
        file_paths = dc.to_values("file.path")
        print(file_paths)  # Returns list of strings
        ```

        Getting all file sizes:
        ```py
        sizes = dc.to_values("file.size")
        print(sizes)  # Returns list of integers
        ```
    """
    return [row[0] for row in self.to_list(col)]

union 

union(other: Self) -> Self

Return the set union of the two datasets.

Parameters:

  • other (Self) –

    chain whose rows will be added to self.

Source code in datachain/lib/dc/datachain.py 
1555
1556
1557
1558
1559
1560
1561
1562
@delta_disabled
def union(self, other: "Self") -> "Self":
    """Return the set union of the two datasets.

    Parameters:
        other: chain whose rows will be added to `self`.
    """
    return self._evolve(query=self._query.union(other._query))

DataChainError

Bases: Exception

Session 

Session(
    name="",
    catalog: Optional[Catalog] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
)

Session is a context that keeps track of temporary DataChain datasets for a proper cleanup. By default, a global session is created.

Temporary or ephemeral datasets are the ones created without specified name. They are useful for optimization purposes and should be automatically removed.

Temp dataset has specific name format

"session_"

The suffix is optional. Both s are auto-generated.

Temp dataset examples

session_myname_624b41_48e8b4 session_4b962d_2a5dff

Parameters:

name (str): The name of the session. Only latters and numbers are supported. It can be empty. catalog (Catalog): Catalog object.

Source code in datachain/query/session.py 
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def __init__(
    self,
    name="",
    catalog: Optional["Catalog"] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
):
    if re.match(r"^[0-9a-zA-Z]*$", name) is None:
        raise ValueError(
            f"Session name can contain only letters or numbers - '{name}' given."
        )

    if not name:
        name = self.GLOBAL_SESSION_NAME

    session_uuid = uuid4().hex[: self.SESSION_UUID_LEN]
    self.name = f"{name}_{session_uuid}"
    self.is_new_catalog = not catalog
    self.catalog = catalog or get_catalog(
        client_config=client_config, in_memory=in_memory
    )
    self.dataset_versions: list[tuple[DatasetRecord, str, bool]] = []

get classmethod 

get(
    session: Optional[Session] = None,
    catalog: Optional[Catalog] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
) -> Session

Creates a Session() object from a catalog.

Parameters:

  • session (Session, default: None ) –

    Optional Session(). If not provided a new session will be created. It's needed mostly for simple API purposes.

  • catalog (Catalog, default: None ) –

    Optional catalog. By default, a new catalog is created.

Source code in datachain/query/session.py 
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
@classmethod
def get(
    cls,
    session: Optional["Session"] = None,
    catalog: Optional["Catalog"] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
) -> "Session":
    """Creates a Session() object from a catalog.

    Parameters:
        session (Session): Optional Session(). If not provided a new session will
                be created. It's needed mostly for simple API purposes.
        catalog (Catalog): Optional catalog. By default, a new catalog is created.
    """
    if session:
        return session

    # Access the active (most recent) context from the stack
    if cls.SESSION_CONTEXTS:
        session = cls.SESSION_CONTEXTS[-1]

    elif cls.GLOBAL_SESSION_CTX is None:
        cls.GLOBAL_SESSION_CTX = Session(
            cls.GLOBAL_SESSION_NAME,
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session = cls.GLOBAL_SESSION_CTX

        atexit.register(cls._global_cleanup)
        cls.ORIGINAL_EXCEPT_HOOK = sys.excepthook
        sys.excepthook = cls.except_hook
    else:
        session = cls.GLOBAL_SESSION_CTX

    if client_config and session.catalog.client_config != client_config:
        session = Session(
            "session" + uuid4().hex[:4],
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session.__enter__()

    return session

Sys

Bases: DataModel

Model for internal DataChain signals id and rand.