Skip to content

DataChain

The DataChain class creates a data chain, which is a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data. See DataChain for examples of how to create a chain.

C module-attribute 

C = Column

Column 

Column(
    text, type_=None, is_literal=False, _selectable=None
)

Bases: ColumnClause

Source code in datachain/query/schema.py 
47
48
49
50
51
52
def __init__(self, text, type_=None, is_literal=False, _selectable=None):
    """Dataset column."""
    self.name = ColumnMeta.to_db_name(text)
    super().__init__(
        self.name, type_=type_, is_literal=is_literal, _selectable=_selectable
    )

glob 

glob(glob_str)

Search for matches using glob pattern matching.

Source code in datachain/query/schema.py 
57
58
59
def glob(self, glob_str):
    """Search for matches using glob pattern matching."""
    return self.op("GLOB")(glob_str)

regexp 

regexp(regexp_str)

Search for matches using regexp pattern matching.

Source code in datachain/query/schema.py 
61
62
63
def regexp(self, regexp_str):
    """Search for matches using regexp pattern matching."""
    return self.op("REGEXP")(regexp_str)

read_csv 

read_csv(
    path: Union[
        str, PathLike[str], list[str], list[PathLike[str]]
    ],
    delimiter: Optional[str] = None,
    header: bool = True,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column_types: Optional[
        dict[str, Union[str, DataType]]
    ] = None,
    parse_options: Optional[
        dict[str, Union[str, Union[bool, Callable]]]
    ] = None,
    **kwargs
) -> DataChain

Generate chain from csv files.

Parameters:

  • path (Union[str, PathLike[str], list[str], list[PathLike[str]]]) –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • delimiter (Optional[str], default: None ) –

    Character for delimiting columns. Takes precedence if also specified in parse_options. Defaults to ",".

  • header (bool, default: True ) –

    Whether the files include a header row.

  • output (OutputType, default: None ) –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • column (str, default: '' ) –

    Created column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • source (bool, default: True ) –

    Whether to include info about the source file.

  • nrows (Optional[int], default: None ) –

    Optional row limit.

  • session (Optional[Session], default: None ) –

    Session to use for the chain.

  • settings (Optional[dict], default: None ) –

    Settings to use for the chain.

  • column_types (Optional[dict[str, Union[str, DataType]]], default: None ) –

    Dictionary of column names and their corresponding types. It is passed to CSV reader and for each column specified type auto inference is disabled.

  • parse_options (Optional[dict[str, Union[str, Union[bool, Callable]]]], default: None ) –

    Tells the parser how to process lines. See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

Example

Reading a csv file: 

import datachain as dc
chain = dc.read_csv("s3://mybucket/file.csv")

Reading csv files from a directory as a combined dataset: 

import datachain as dc
chain = dc.read_csv("s3://mybucket/dir")

Source code in datachain/lib/dc/csv.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def read_csv(
    path: Union[str, os.PathLike[str], list[str], list[os.PathLike[str]]],
    delimiter: Optional[str] = None,
    header: bool = True,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column_types: Optional[dict[str, "Union[str, ArrowDataType]"]] = None,
    parse_options: Optional[dict[str, "Union[str, Union[bool, Callable]]"]] = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from csv files.

    Parameters:
        path: Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        delimiter: Character for delimiting columns. Takes precedence if also
            specified in `parse_options`. Defaults to ",".
        header: Whether the files include a header row.
        output: Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        column: Created column name.
        model_name: Generated model name.
        source: Whether to include info about the source file.
        nrows: Optional row limit.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        column_types: Dictionary of column names and their corresponding types.
            It is passed to CSV reader and for each column specified type auto
            inference is disabled.
        parse_options: Tells the parser how to process lines.
            See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

    Example:
        Reading a csv file:
        ```py
        import datachain as dc
        chain = dc.read_csv("s3://mybucket/file.csv")
        ```

        Reading csv files from a directory as a combined dataset:
        ```py
        import datachain as dc
        chain = dc.read_csv("s3://mybucket/dir")
        ```
    """
    from pandas.io.parsers.readers import STR_NA_VALUES
    from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
    from pyarrow.dataset import CsvFileFormat
    from pyarrow.lib import type_for_alias

    from .storage import read_storage

    parse_options = parse_options or {}
    if "delimiter" not in parse_options:
        parse_options["delimiter"] = ","
    if delimiter:
        parse_options["delimiter"] = delimiter

    if column_types:
        column_types = {
            name: type_for_alias(typ) if isinstance(typ, str) else typ
            for name, typ in column_types.items()
        }
    else:
        column_types = {}

    chain = read_storage(path, session=session, settings=settings, **kwargs)

    column_names = None
    if not header:
        if not output:
            msg = "error parsing csv - provide output if no header"
            raise DatasetPrepareError(chain.name, msg)
        if isinstance(output, Sequence):
            column_names = output  # type: ignore[assignment]
        elif isinstance(output, dict):
            column_names = list(output.keys())
        elif (fr := ModelStore.to_pydantic(output)) is not None:
            column_names = list(fr.model_fields.keys())
        else:
            msg = f"error parsing csv - incompatible output type {type(output)}"
            raise DatasetPrepareError(chain.name, msg)

    parse_options = ParseOptions(**parse_options)
    read_options = ReadOptions(column_names=column_names)
    convert_options = ConvertOptions(
        strings_can_be_null=True,
        null_values=STR_NA_VALUES,
        column_types=column_types,
    )
    format = CsvFileFormat(
        parse_options=parse_options,
        read_options=read_options,
        convert_options=convert_options,
    )
    return chain.parse_tabular(
        output=output,
        column=column,
        model_name=model_name,
        source=source,
        nrows=nrows,
        format=format,
        parse_options=parse_options,
    )

read_dataset 

read_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[Union[str, int]] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    delta_unsafe: bool = False,
    update: bool = False,
) -> DataChain

Get data from a saved Dataset. It returns the chain itself. If dataset or version is not found locally, it will try to pull it from Studio.

Parameters:

  • name (str) –

    The dataset name, which can be a fully qualified name including the namespace and project. Alternatively, it can be a regular name, in which case the explicitly defined namespace and project will be used if they are set; otherwise, default values will be applied.

  • namespace (Optional[str], default: None ) –

    optional name of namespace in which dataset to read is created

  • project (Optional[str], default: None ) –

    optional name of project in which dataset to read is created

  • version (Optional[Union[str, int]], default: None ) –

    dataset version. Supports: - Exact version strings: "1.2.3" - Legacy integer versions: 1, 2, 3 (finds latest major version) - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.

  • session (Optional[Session], default: None ) –

    Session to use for the chain.

  • settings (Optional[dict], default: None ) –

    Settings to use for the chain.

  • delta (Optional[bool], default: False ) –

    If True, only process new or changed files instead of reprocessing everything. This saves time by skipping files that were already processed in previous versions. The optimization is working when a new version of the dataset is created. Default is False.

  • delta_on (Optional[Union[str, Sequence[str]]], default: ('file.path', 'file.etag', 'file.version') ) –

    Field(s) that uniquely identify each record in the source data. Used to detect which records are new or changed. Default is ("file.path", "file.etag", "file.version").

  • delta_result_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) in the result dataset that match delta_on fields. Only needed if you rename the identifying fields during processing. Default is None.

  • delta_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) used to detect if a record has changed. If not specified, all fields except delta_on fields are used. Default is None.

  • delta_retry (Optional[Union[bool, str]], default: None ) –

    Controls retry behavior for failed records: - String (field name): Reprocess records where this field is not empty (error mode) - True: Reprocess records missing from the result dataset (missing mode) - None: No retry processing (default)

  • update (bool, default: False ) –

    If True always checks for newer versions available on Studio, even if some version of the dataset exists locally already. If False (default), it will only fetch the dataset from Studio if it is not found locally.

  • delta_unsafe (bool, default: False ) –

    Allow restricted ops in delta: merge, agg, union, group_by, distinct.

Example 
import datachain as dc
chain = dc.read_dataset("my_cats")

import datachain as dc
chain = dc.read_dataset("dev.animals.my_cats")

chain = dc.read_dataset("my_cats", version="1.0.0")

# Using version specifiers (PEP 440)
chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")

# Legacy integer version support (finds latest in major version)
chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version

# Always check for newer versions matching a version specifier from Studio
chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)

session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
settings = {
    "cache": True,
    "parallel": 4,
    "workers": 4,
    "min_task_size": 1000,
    "prefetch": 10,
}
chain = dc.read_dataset(
    name="my_cats",
    version="1.0.0",
    session=session,
    settings=settings,
)
Source code in datachain/lib/dc/datasets.py 
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
def read_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[Union[str, int]] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[Union[str, Sequence[str]]] = None,
    delta_compare: Optional[Union[str, Sequence[str]]] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    delta_unsafe: bool = False,
    update: bool = False,
) -> "DataChain":
    """Get data from a saved Dataset. It returns the chain itself.
    If dataset or version is not found locally, it will try to pull it from Studio.

    Parameters:
        name: The dataset name, which can be a fully qualified name including the
            namespace and project. Alternatively, it can be a regular name, in which
            case the explicitly defined namespace and project will be used if they are
            set; otherwise, default values will be applied.
        namespace: optional name of namespace in which dataset to read is created
        project: optional name of project in which dataset to read is created
        version: dataset version. Supports:
            - Exact version strings: "1.2.3"
            - Legacy integer versions: 1, 2, 3 (finds latest major version)
            - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        delta: If True, only process new or changed files instead of reprocessing
            everything. This saves time by skipping files that were already processed in
            previous versions. The optimization is working when a new version of the
            dataset is created.
            Default is False.
        delta_on: Field(s) that uniquely identify each record in the source data.
            Used to detect which records are new or changed.
            Default is ("file.path", "file.etag", "file.version").
        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
            Only needed if you rename the identifying fields during processing.
            Default is None.
        delta_compare: Field(s) used to detect if a record has changed.
            If not specified, all fields except `delta_on` fields are used.
            Default is None.
        delta_retry: Controls retry behavior for failed records:
            - String (field name): Reprocess records where this field is not empty
              (error mode)
            - True: Reprocess records missing from the result dataset (missing mode)
            - None: No retry processing (default)
        update: If True always checks for newer versions available on Studio, even if
            some version of the dataset exists locally already. If False (default), it
            will only fetch the dataset from Studio if it is not found locally.
        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
            distinct.


    Example:
        ```py
        import datachain as dc
        chain = dc.read_dataset("my_cats")
        ```

        ```py
        import datachain as dc
        chain = dc.read_dataset("dev.animals.my_cats")
        ```

        ```py
        chain = dc.read_dataset("my_cats", version="1.0.0")
        ```

        ```py
        # Using version specifiers (PEP 440)
        chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")
        ```

        ```py
        # Legacy integer version support (finds latest in major version)
        chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version
        ```

        ```py
        # Always check for newer versions matching a version specifier from Studio
        chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)
        ```

        ```py
        session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
        settings = {
            "cache": True,
            "parallel": 4,
            "workers": 4,
            "min_task_size": 1000,
            "prefetch": 10,
        }
        chain = dc.read_dataset(
            name="my_cats",
            version="1.0.0",
            session=session,
            settings=settings,
        )
        ```
    """
    from datachain.telemetry import telemetry

    from .datachain import DataChain

    telemetry.send_event_once("class", "datachain_init", name=name, version=version)

    session = Session.get(session)
    catalog = session.catalog

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        project_name=project,
        namespace_name=namespace,
    )

    if version is not None:
        dataset = session.catalog.get_dataset_with_remote_fallback(
            name, namespace_name, project_name, update=update
        )

        # Convert legacy integer versions to version specifiers
        # For backward compatibility we still allow users to put version as integer
        # in which case we convert it to a version specifier that finds the latest
        # version where major part is equal to that input version.
        # For example if user sets version=2, we convert it to ">=2.0.0,<3.0.0"
        # which will find something like 2.4.3 (assuming 2.4.3 is the biggest among
        # all 2.* dataset versions)
        if isinstance(version, int):
            version_spec = f">={version}.0.0,<{version + 1}.0.0"
        else:
            version_spec = str(version)

        from packaging.specifiers import InvalidSpecifier, SpecifierSet

        try:
            # Try to parse as version specifier
            SpecifierSet(version_spec)
            # If it's a valid specifier set, find the latest compatible version
            latest_compatible = dataset.latest_compatible_version(version_spec)
            if not latest_compatible:
                raise DatasetVersionNotFoundError(
                    f"No dataset {name} version matching specifier {version_spec}"
                )
            version = latest_compatible
        except InvalidSpecifier:
            # If not a valid specifier, treat as exact version string
            # This handles cases like "1.2.3" which are exact versions, not specifiers
            pass

    if settings:
        _settings = Settings(**settings)
    else:
        _settings = Settings()

    query = DatasetQuery(
        name=name,
        project_name=project_name,
        namespace_name=namespace_name,
        version=version,  #  type: ignore[arg-type]
        session=session,
        update=update,
    )

    signals_schema = SignalSchema({"sys": Sys})
    if query.feature_schema:
        signals_schema |= SignalSchema.deserialize(query.feature_schema)
    else:
        signals_schema |= SignalSchema.from_column_types(query.column_types or {})
    chain = DataChain(query, _settings, signals_schema)

    if delta:
        chain = chain._as_delta(
            on=delta_on,
            right_on=delta_result_on,
            compare=delta_compare,
            delta_retry=delta_retry,
            delta_unsafe=delta_unsafe,
        )

    return chain

datasets 

datasets(
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: Optional[str] = None,
    include_listing: bool = False,
    studio: bool = False,
    attrs: Optional[list[str]] = None,
) -> DataChain

Generate chain with list of registered datasets.

Parameters:

  • session (Optional[Session], default: None ) –

    Optional session instance. If not provided, uses default session.

  • settings (Optional[dict], default: None ) –

    Optional dictionary of settings to configure the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • column (Optional[str], default: None ) –

    Name of the output column in the chain. Defaults to None which means no top level column will be created.

  • include_listing (bool, default: False ) –

    If True, includes listing datasets. Defaults to False.

  • studio (bool, default: False ) –

    If True, returns datasets from Studio only, otherwise returns all local datasets. Defaults to False.

  • attrs (Optional[list[str]], default: None ) –

    Optional list of attributes to filter datasets on. It can be just attribute without value e.g "NLP", or attribute with value e.g "location=US". Attribute with value can also accept "" to target all that have specific name e.g "location="

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance containing dataset information.

Example 
import datachain as dc

chain = dc.datasets(column="dataset")
for ds in chain.to_iter("dataset"):
    print(f"{ds.name}@v{ds.version}")
Source code in datachain/lib/dc/datasets.py 
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
def datasets(
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: Optional[str] = None,
    include_listing: bool = False,
    studio: bool = False,
    attrs: Optional[list[str]] = None,
) -> "DataChain":
    """Generate chain with list of registered datasets.

    Args:
        session: Optional session instance. If not provided, uses default session.
        settings: Optional dictionary of settings to configure the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        column: Name of the output column in the chain. Defaults to None which
            means no top level column will be created.
        include_listing: If True, includes listing datasets. Defaults to False.
        studio: If True, returns datasets from Studio only,
            otherwise returns all local datasets. Defaults to False.
        attrs: Optional list of attributes to filter datasets on. It can be just
            attribute without value e.g "NLP", or attribute with value
            e.g "location=US". Attribute with value can also accept "*" to target
            all that have specific name e.g "location=*"

    Returns:
        DataChain: A new DataChain instance containing dataset information.

    Example:
        ```py
        import datachain as dc

        chain = dc.datasets(column="dataset")
        for ds in chain.to_iter("dataset"):
            print(f"{ds.name}@v{ds.version}")
        ```
    """

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    datasets_values = [
        DatasetInfo.from_models(d, v, j)
        for d, v, j in catalog.list_datasets_versions(
            include_listing=include_listing, studio=studio
        )
    ]
    datasets_values = [d for d in datasets_values if not d.is_temp]

    if attrs:
        for attr in attrs:
            datasets_values = [d for d in datasets_values if d.has_attr(attr)]

    if not column:
        # flattening dataset fields
        schema = {
            k: get_origin(v) if get_origin(v) is dict else v
            for k, v in get_type_hints(DatasetInfo).items()
            if k in DatasetInfo.model_fields
        }
        data = {k: [] for k in DatasetInfo.model_fields}  # type: ignore[var-annotated]
        for d in [d.model_dump() for d in datasets_values]:
            for field, value in d.items():
                data[field].append(value)

        return read_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            output=schema,
            **data,  # type: ignore[arg-type]
        )

    return read_values(
        session=session,
        settings=settings,
        in_memory=in_memory,
        output={column: DatasetInfo},
        **{column: datasets_values},  # type: ignore[arg-type]
    )

delete_dataset 

delete_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[str] = None,
    force: Optional[bool] = False,
    studio: Optional[bool] = False,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None

Removes specific dataset version or all dataset versions, depending on a force flag.

Parameters:

  • name (str) –

    The dataset name, which can be a fully qualified name including the namespace and project. Alternatively, it can be a regular name, in which case the explicitly defined namespace and project will be used if they are set; otherwise, default values will be applied.

  • namespace (Optional[str], default: None ) –

    optional name of namespace in which dataset to delete is created

  • project (Optional[str], default: None ) –

    optional name of project in which dataset to delete is created

  • version (Optional[str], default: None ) –

    Optional dataset version

  • force (Optional[bool], default: False ) –

    If true, all datasets versions will be removed. Defaults to False.

  • studio (Optional[bool], default: False ) –

    If True, removes dataset from Studio only, otherwise removes local dataset. Defaults to False.

  • session (Optional[Session], default: None ) –

    Optional session instance. If not provided, uses default session.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

Returns: None

Example 
import datachain as dc
dc.delete_dataset("cats")

import datachain as dc
dc.delete_dataset("cats", version="1.0.0")
Source code in datachain/lib/dc/datasets.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
def delete_dataset(
    name: str,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    version: Optional[str] = None,
    force: Optional[bool] = False,
    studio: Optional[bool] = False,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None:
    """Removes specific dataset version or all dataset versions, depending on
    a force flag.

    Args:
        name: The dataset name, which can be a fully qualified name including the
            namespace and project. Alternatively, it can be a regular name, in which
            case the explicitly defined namespace and project will be used if they are
            set; otherwise, default values will be applied.
        namespace: optional name of namespace in which dataset to delete is created
        project: optional name of project in which dataset to delete is created
        version: Optional dataset version
        force: If true, all datasets versions will be removed. Defaults to False.
        studio: If True, removes dataset from Studio only, otherwise removes local
            dataset. Defaults to False.
        session: Optional session instance. If not provided, uses default session.
        in_memory: If True, creates an in-memory session. Defaults to False.

    Returns: None

    Example:
        ```py
        import datachain as dc
        dc.delete_dataset("cats")
        ```

        ```py
        import datachain as dc
        dc.delete_dataset("cats", version="1.0.0")
        ```
    """
    from datachain.studio import remove_studio_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        project_name=project,
        namespace_name=namespace,
    )

    if not is_studio() and studio:
        return remove_studio_dataset(
            None, name, namespace_name, project_name, version=version, force=force
        )

    try:
        ds_project = get_project(project_name, namespace_name, session=session)
    except ProjectNotFoundError:
        raise DatasetNotFoundError(
            f"Dataset {name} not found in namespace {namespace_name} and project",
            f" {project_name}",
        ) from None

    if not force:
        version = (
            version
            or catalog.get_dataset(
                name,
                namespace_name=ds_project.namespace.name,
                project_name=ds_project.name,
            ).latest_version
        )
    else:
        version = None
    catalog.remove_dataset(name, ds_project, version=version, force=force)

move_dataset 

move_dataset(
    src: str,
    dest: str,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None

Moves an entire dataset between namespaces and projects.

Parameters:

  • src (str) –

    The source dataset name. This can be a fully qualified name that includes the namespace and project, or a regular name. If a regular name is used, default values will be applied. The source dataset will no longer exist after the move.

  • dest (str) –

    The destination dataset name. This can also be a fully qualified name with a namespace and project, or just a regular name (default values will be used in that case). The original dataset will be moved here.

  • session (Optional[Session], default: None ) –

    An optional session instance. If not provided, the default session will be used.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

Returns:

  • None –

    None

Examples:

import datachain as dc
dc.move_dataset("cats", "new_cats")

import datachain as dc
dc.move_dataset("dev.animals.cats", "prod.animals.cats")
Source code in datachain/lib/dc/datasets.py 
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def move_dataset(
    src: str,
    dest: str,
    session: Optional[Session] = None,
    in_memory: bool = False,
) -> None:
    """Moves an entire dataset between namespaces and projects.

    Args:
        src: The source dataset name. This can be a fully qualified name that includes
            the namespace and project, or a regular name. If a regular name is used,
            default values will be applied. The source dataset will no longer exist
            after the move.
        dest: The destination dataset name. This can also be a fully qualified
            name with a namespace and project, or just a regular name (default values
            will be used in that case). The original dataset will be moved here.
        session: An optional session instance. If not provided, the default session
            will be used.
        in_memory: If True, creates an in-memory session. Defaults to False.

    Returns:
        None

    Examples:
        ```python
        import datachain as dc
        dc.move_dataset("cats", "new_cats")
        ```

        ```python
        import datachain as dc
        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    namespace, project, name = catalog.get_full_dataset_name(src)
    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)

    dataset = catalog.get_dataset(name, namespace_name=namespace, project_name=project)

    catalog.update_dataset(
        dataset,
        name=dest_name,
        project_id=catalog.metastore.get_project(
            dest_project,
            dest_namespace,
            create=is_studio(),
        ).id,
    )

read_hf 

read_hf(
    dataset: Union[str, HFDatasetType],
    *args: Any,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column: str = "",
    model_name: str = "",
    limit: int = 0,
    **kwargs: Any
) -> DataChain

Generate chain from Hugging Face Hub dataset.

Parameters:

  • dataset (Union[str, HFDatasetType]) –

    Path or name of the dataset to read from Hugging Face Hub, or an instance of datasets.Dataset-like object.

  • args (Any, default: () ) –

    Additional positional arguments to pass to datasets.load_dataset.

  • session (Optional[Session], default: None ) –

    Session to use for the chain.

  • settings (Optional[dict], default: None ) –

    Settings to use for the chain.

  • column (str, default: '' ) –

    Generated object column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • limit (int, default: 0 ) –

    The maximum number of items to read from the HF dataset. Applies take(limit) to datasets.load_dataset. Defaults to 0 (no limit).

  • kwargs (Any, default: {} ) –

    Parameters to pass to datasets.load_dataset.

Example

Load from Hugging Face Hub: 

import datachain as dc
chain = dc.read_hf("beans", split="train")

Generate chain from loaded dataset: 

from datasets import load_dataset
ds = load_dataset("beans", split="train")
import datachain as dc
chain = dc.read_hf(ds)

Streaming with limit, for large datasets: 

import datachain as dc
ds = dc.read_hf("beans", split="train", streaming=True, limit=10)

or use HF split syntax (not supported if streaming is enabled): 

import datachain as dc
ds = dc.read_hf("beans", split="train[%10]")

Source code in datachain/lib/dc/hf.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def read_hf(
    dataset: Union[str, "HFDatasetType"],
    *args: Any,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column: str = "",
    model_name: str = "",
    limit: int = 0,
    **kwargs: Any,
) -> "DataChain":
    """Generate chain from Hugging Face Hub dataset.

    Parameters:
        dataset: Path or name of the dataset to read from Hugging Face Hub,
            or an instance of `datasets.Dataset`-like object.
        args: Additional positional arguments to pass to `datasets.load_dataset`.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        column: Generated object column name.
        model_name: Generated model name.
        limit: The maximum number of items to read from the HF dataset.
            Applies `take(limit)` to `datasets.load_dataset`.
            Defaults to 0 (no limit).
        kwargs: Parameters to pass to `datasets.load_dataset`.

    Example:
        Load from Hugging Face Hub:
        ```py
        import datachain as dc
        chain = dc.read_hf("beans", split="train")
        ```

        Generate chain from loaded dataset:
        ```py
        from datasets import load_dataset
        ds = load_dataset("beans", split="train")
        import datachain as dc
        chain = dc.read_hf(ds)
        ```

        Streaming with limit, for large datasets:
        ```py
        import datachain as dc
        ds = dc.read_hf("beans", split="train", streaming=True, limit=10)
        ```

        or use HF split syntax (not supported if streaming is enabled):
        ```py
        import datachain as dc
        ds = dc.read_hf("beans", split="train[%10]")
        ```
    """
    from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits

    from .values import read_values

    output: dict[str, DataType] = {}
    ds_dict = stream_splits(dataset, *args, **kwargs)
    if len(ds_dict) > 1:
        output = {"split": str}

    model_name = model_name or column or ""
    hf_features = next(iter(ds_dict.values())).features
    hf_output, normalized_names = get_output_schema(hf_features, list(output.keys()))
    output = output | hf_output
    model = dict_to_data_model(model_name, output, list(normalized_names.values()))
    if column:
        output = {column: model}

    chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
    return chain.gen(HFGenerator(dataset, model, limit, *args, **kwargs), output=output)

read_json 

read_json(
    path: Union[str, PathLike[str]],
    type: FileType = "text",
    spec: Optional[DataType] = None,
    schema_from: Optional[str] = "auto",
    jmespath: Optional[str] = None,
    column: Optional[str] = "",
    model_name: Optional[str] = None,
    format: Optional[str] = "json",
    nrows: Optional[int] = None,
    **kwargs
) -> DataChain

Get data from JSON. It returns the chain itself.

Parameters:

  • path (Union[str, PathLike[str]]) –

    storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///"

  • type (FileType, default: 'text' ) –

    read file as "binary", "text", or "image" data. Default is "text".

  • spec (Optional[DataType], default: None ) –

    optional Data Model

  • schema_from (Optional[str], default: 'auto' ) –

    path to sample to infer spec (if schema not provided)

  • column (Optional[str], default: '' ) –

    generated column name

  • model_name (Optional[str], default: None ) –

    optional generated model name

  • format (Optional[str], default: 'json' ) –

    "json", "jsonl"

  • jmespath (Optional[str], default: None ) –

    optional JMESPATH expression to reduce JSON

  • nrows (Optional[int], default: None ) –

    optional row limit for jsonl and JSON arrays

Example

infer JSON schema from data, reduce using JMESPATH 

import datachain as dc
chain = dc.read_json("gs://json", jmespath="key1.key2")

infer JSON schema from a particular path 

import datachain as dc
chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")

Source code in datachain/lib/dc/json.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def read_json(
    path: Union[str, os.PathLike[str]],
    type: FileType = "text",
    spec: Optional[DataType] = None,
    schema_from: Optional[str] = "auto",
    jmespath: Optional[str] = None,
    column: Optional[str] = "",
    model_name: Optional[str] = None,
    format: Optional[str] = "json",
    nrows: Optional[int] = None,
    **kwargs,
) -> "DataChain":
    """Get data from JSON. It returns the chain itself.

    Parameters:
        path: storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///"
        type: read file as "binary", "text", or "image" data. Default is "text".
        spec: optional Data Model
        schema_from: path to sample to infer spec (if schema not provided)
        column: generated column name
        model_name: optional generated model name
        format: "json", "jsonl"
        jmespath: optional JMESPATH expression to reduce JSON
        nrows: optional row limit for jsonl and JSON arrays

    Example:
        infer JSON schema from data, reduce using JMESPATH
        ```py
        import datachain as dc
        chain = dc.read_json("gs://json", jmespath="key1.key2")
        ```

        infer JSON schema from a particular path
        ```py
        import datachain as dc
        chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")
        ```
    """
    from .storage import read_storage

    if schema_from == "auto":
        schema_from = os.fspath(path)

    def jmespath_to_name(s: str):
        name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
        return s[:name_end]

    if (not column) and jmespath:
        column = jmespath_to_name(jmespath)
    if not column:
        column = format
    chain = read_storage(uri=path, type=type, **kwargs)
    signal_dict = {
        column: meta_formats.read_meta(
            schema_from=schema_from,
            format=format,
            spec=spec,
            model_name=model_name,
            jmespath=jmespath,
            nrows=nrows,
        ),
        "params": {"file": File},
    }
    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}

    cloudpickle.register_pickle_by_value(meta_formats)

    return chain.settings(**settings).gen(**signal_dict)  # type: ignore[misc, arg-type]

listings 

listings(
    session: Optional[Session] = None,
    in_memory: bool = False,
    column: str = "listing",
    **kwargs
) -> DataChain

Generate chain with list of cached listings. Listing is a special kind of dataset which has directory listing data of some underlying storage (e.g S3 bucket).

Example 
import datachain as dc
dc.listings().show()
Source code in datachain/lib/dc/listings.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def listings(
    session: Optional[Session] = None,
    in_memory: bool = False,
    column: str = "listing",
    **kwargs,
) -> "DataChain":
    """Generate chain with list of cached listings.
    Listing is a special kind of dataset which has directory listing data of
    some underlying storage (e.g S3 bucket).

    Example:
        ```py
        import datachain as dc
        dc.listings().show()
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = kwargs.get("catalog") or session.catalog

    return read_values(
        session=session,
        in_memory=in_memory,
        output={column: ListingInfo},
        **{column: catalog.listings()},  # type: ignore[arg-type]
    )

read_pandas 

read_pandas(
    df: DataFrame,
    name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: str = "",
) -> DataChain

Generate chain from pandas data-frame.

Example 
import pandas as pd
import datachain as dc

df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
dc.read_pandas(df)
Source code in datachain/lib/dc/pandas.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def read_pandas(  # type: ignore[override]
    df: "pd.DataFrame",
    name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    column: str = "",
) -> "DataChain":
    """Generate chain from pandas data-frame.

    Example:
        ```py
        import pandas as pd
        import datachain as dc

        df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
        dc.read_pandas(df)
        ```
    """
    from .utils import DatasetPrepareError

    def get_col_name(col):
        if isinstance(col, tuple):
            # Join tuple elements with underscore for MultiIndex columns
            return "_".join(map(str, col)).lower()
        # Handle regular string column names
        return str(col).lower()

    fr_map = {get_col_name(col): df[col].tolist() for col in df.columns}

    for c in fr_map:
        if not c.isidentifier():
            raise DatasetPrepareError(
                name,
                f"import from pandas error - '{c}' cannot be a column name",
            )

    return read_values(
        name,
        session,
        settings=settings,
        column=column,
        in_memory=in_memory,
        **fr_map,
    )

read_parquet 

read_parquet(
    path: Union[
        str, PathLike[str], list[str], list[PathLike[str]]
    ],
    partitioning: Any = "hive",
    output: Optional[dict[str, DataType]] = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    **kwargs
) -> DataChain

Generate chain from parquet files.

Parameters:

  • path (Union[str, PathLike[str], list[str], list[PathLike[str]]]) –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • partitioning (Any, default: 'hive' ) –

    Any pyarrow partitioning schema.

  • output (Optional[dict[str, DataType]], default: None ) –

    Dictionary defining column names and their corresponding types.

  • column (str, default: '' ) –

    Created column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • source (bool, default: True ) –

    Whether to include info about the source file.

  • session (Optional[Session], default: None ) –

    Session to use for the chain.

  • settings (Optional[dict], default: None ) –

    Settings to use for the chain.

Example

Reading a single file: 

import datachain as dc
dc.read_parquet("s3://mybucket/file.parquet")

Reading a partitioned dataset from a directory: 

import datachain as dc
dc.read_parquet("s3://mybucket/dir")

Source code in datachain/lib/dc/parquet.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def read_parquet(
    path: Union[str, os.PathLike[str], list[str], list[os.PathLike[str]]],
    partitioning: Any = "hive",
    output: Optional[dict[str, DataType]] = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from parquet files.

    Parameters:
        path: Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        partitioning: Any pyarrow partitioning schema.
        output: Dictionary defining column names and their corresponding types.
        column: Created column name.
        model_name: Generated model name.
        source: Whether to include info about the source file.
        session: Session to use for the chain.
        settings: Settings to use for the chain.

    Example:
        Reading a single file:
        ```py
        import datachain as dc
        dc.read_parquet("s3://mybucket/file.parquet")
        ```

        Reading a partitioned dataset from a directory:
        ```py
        import datachain as dc
        dc.read_parquet("s3://mybucket/dir")
        ```
    """
    from .storage import read_storage

    chain = read_storage(path, session=session, settings=settings, **kwargs)
    return chain.parse_tabular(
        output=output,
        column=column,
        model_name=model_name,
        source=source,
        format="parquet",
        partitioning=partitioning,
    )

read_records 

read_records(
    to_insert: Optional[Union[dict, Iterable[dict]]],
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    schema: Optional[dict[str, DataType]] = None,
) -> DataChain

Create a DataChain from the provided records. This method can be used for programmatically generating a chain in contrast of reading data from storages or other sources.

Parameters:

  • to_insert (Optional[Union[dict, Iterable[dict]]]) –

    records (or a single record) to insert. Each record is a dictionary of signals and their values.

  • schema (Optional[dict[str, DataType]], default: None ) –

    describes chain signals and their corresponding types

Example 
import datachain as dc
single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
Notes

This call blocks until all records are inserted.

Source code in datachain/lib/dc/records.py 
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def read_records(
    to_insert: Optional[Union[dict, Iterable[dict]]],
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    schema: Optional[dict[str, DataType]] = None,
) -> "DataChain":
    """Create a DataChain from the provided records. This method can be used for
    programmatically generating a chain in contrast of reading data from storages
    or other sources.

    Parameters:
        to_insert: records (or a single record) to insert. Each record is
                    a dictionary of signals and their values.
        schema: describes chain signals and their corresponding types

    Example:
        ```py
        import datachain as dc
        single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
        ```

    Notes:
        This call blocks until all records are inserted.
    """
    from datachain.query.dataset import adjust_outputs, get_col_types
    from datachain.sql.types import SQLType

    from .datasets import read_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    name = session.generate_temp_dataset_name()
    signal_schema = None
    columns: list[sqlalchemy.Column] = []

    if schema:
        signal_schema = SignalSchema(schema)
        columns = [
            sqlalchemy.Column(c.name, c.type)  # type: ignore[union-attr]
            for c in signal_schema.db_signals(as_columns=True)
        ]
    else:
        columns = [
            sqlalchemy.Column(name, typ)
            for name, typ in File._datachain_column_types.items()
        ]

    dsr = catalog.create_dataset(
        name,
        catalog.metastore.default_project,
        columns=columns,
        feature_schema=(
            signal_schema.clone_without_sys_signals().serialize()
            if signal_schema
            else None
        ),
    )

    session.add_dataset_version(dsr, dsr.latest_version)

    if isinstance(to_insert, dict):
        to_insert = [to_insert]
    elif not to_insert:
        to_insert = []

    warehouse = catalog.warehouse
    dr = warehouse.dataset_rows(dsr)
    table = dr.get_table()

    # Optimization: Compute row types once, rather than for every row.
    col_types = get_col_types(
        warehouse,
        {c.name: c.type for c in columns if isinstance(c.type, SQLType)},
    )
    records = (adjust_outputs(warehouse, record, col_types) for record in to_insert)
    warehouse.insert_rows(table, records, batch_size=READ_RECORDS_BATCH_SIZE)
    warehouse.insert_rows_done(table)
    return read_dataset(name=dsr.full_name, session=session, settings=settings)

read_storage 

read_storage(
    uri: Union[
        str, PathLike[str], list[str], list[PathLike[str]]
    ],
    *,
    type: FileType = "binary",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    recursive: Optional[bool] = True,
    column: str = "file",
    update: bool = False,
    anon: Optional[bool] = None,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    delta_unsafe: bool = False,
    client_config: Optional[dict] = None
) -> DataChain

Get data from storage(s) as a list of file with all file attributes. It returns the chain itself as usual.

Parameters:

  • uri (Union[str, PathLike[str], list[str], list[PathLike[str]]]) –

    Storage path(s) or URI(s). Can be a local path or start with a storage prefix like s3://, gs://, az://, hf:// or "file:///". Supports glob patterns: - * : wildcard - ** : recursive wildcard - ? : single character - {a,b} : brace expansion

  • type (FileType, default: 'binary' ) –

    read file as "binary", "text", or "image" data. Default is "binary".

  • recursive (Optional[bool], default: True ) –

    search recursively for the given path.

  • column (str, default: 'file' ) –

    Column name that will contain File objects. Default is "file".

  • update (bool, default: False ) –

    force storage reindexing. Default is False.

  • anon (Optional[bool], default: None ) –

    If True, we will treat cloud bucket as public one.

  • client_config (Optional[dict], default: None ) –

    Optional client configuration for the storage client.

  • delta (Optional[bool], default: False ) –

    If True, only process new or changed files instead of reprocessing everything. This saves time by skipping files that were already processed in previous versions. The optimization is working when a new version of the dataset is created. Default is False.

  • delta_on (Optional[Union[str, Sequence[str]]], default: ('file.path', 'file.etag', 'file.version') ) –

    Field(s) that uniquely identify each record in the source data. Used to detect which records are new or changed. Default is ("file.path", "file.etag", "file.version").

  • delta_result_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) in the result dataset that match delta_on fields. Only needed if you rename the identifying fields during processing. Default is None.

  • delta_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Field(s) used to detect if a record has changed. If not specified, all fields except delta_on fields are used. Default is None.

  • delta_retry (Optional[Union[bool, str]], default: None ) –

    Controls retry behavior for failed records: - String (field name): Reprocess records where this field is not empty (error mode) - True: Reprocess records missing from the result dataset (missing mode) - None: No retry processing (default)

  • delta_unsafe (bool, default: False ) –

    Allow restricted ops in delta: merge, agg, union, group_by, distinct. Caller must ensure datasets are consistent and not partially updated.

Returns:

  • DataChain ( DataChain ) –

    A DataChain object containing the file information.

Examples:

Simple call from s3: 

import datachain as dc
chain = dc.read_storage("s3://my-bucket/my-dir")

Match all .json files recursively using glob pattern 

chain = dc.read_storage("gs://bucket/meta/**/*.json")

Match image file extensions for directories with pattern 

chain = dc.read_storage("s3://bucket/202?/**/*.{jpg,jpeg,png}")

Multiple URIs: 

chain = dc.read_storage(["s3://my-bkt/dir1", "s3://bucket2/dir2/dir3"])

With AWS S3-compatible storage: 

chain = dc.read_storage(
    "s3://my-bucket/my-dir",
    client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
)

Source code in datachain/lib/dc/storage.py 
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
def read_storage(
    uri: Union[str, os.PathLike[str], list[str], list[os.PathLike[str]]],
    *,
    type: FileType = "binary",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    recursive: Optional[bool] = True,
    column: str = "file",
    update: bool = False,
    anon: Optional[bool] = None,
    delta: Optional[bool] = False,
    delta_on: Optional[Union[str, Sequence[str]]] = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: Optional[Union[str, Sequence[str]]] = None,
    delta_compare: Optional[Union[str, Sequence[str]]] = None,
    delta_retry: Optional[Union[bool, str]] = None,
    delta_unsafe: bool = False,
    client_config: Optional[dict] = None,
) -> "DataChain":
    """Get data from storage(s) as a list of file with all file attributes.
    It returns the chain itself as usual.

    Parameters:
        uri: Storage path(s) or URI(s). Can be a local path or start with a
            storage prefix like `s3://`, `gs://`, `az://`, `hf://` or "file:///".
            Supports glob patterns:
              - `*` : wildcard
              - `**` : recursive wildcard
              - `?` : single character
              - `{a,b}` : brace expansion
        type: read file as "binary", "text", or "image" data. Default is "binary".
        recursive: search recursively for the given path.
        column: Column name that will contain File objects. Default is "file".
        update: force storage reindexing. Default is False.
        anon: If True, we will treat cloud bucket as public one.
        client_config: Optional client configuration for the storage client.
        delta: If True, only process new or changed files instead of reprocessing
            everything. This saves time by skipping files that were already processed in
            previous versions. The optimization is working when a new version of the
            dataset is created.
            Default is False.
        delta_on: Field(s) that uniquely identify each record in the source data.
            Used to detect which records are new or changed.
            Default is ("file.path", "file.etag", "file.version").
        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
            Only needed if you rename the identifying fields during processing.
            Default is None.
        delta_compare: Field(s) used to detect if a record has changed.
            If not specified, all fields except `delta_on` fields are used.
            Default is None.
        delta_retry: Controls retry behavior for failed records:
            - String (field name): Reprocess records where this field is not empty
              (error mode)
            - True: Reprocess records missing from the result dataset (missing mode)
            - None: No retry processing (default)
        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
            distinct. Caller must ensure datasets are consistent and not partially
            updated.

    Returns:
        DataChain: A DataChain object containing the file information.

    Examples:
        Simple call from s3:
        ```python
        import datachain as dc
        chain = dc.read_storage("s3://my-bucket/my-dir")
        ```

        Match all .json files recursively using glob pattern
        ```py
        chain = dc.read_storage("gs://bucket/meta/**/*.json")
        ```

        Match image file extensions for directories with pattern
        ```py
        chain = dc.read_storage("s3://bucket/202?/**/*.{jpg,jpeg,png}")
        ```

        Multiple URIs:
        ```python
        chain = dc.read_storage(["s3://my-bkt/dir1", "s3://bucket2/dir2/dir3"])
        ```

        With AWS S3-compatible storage:
        ```python
        chain = dc.read_storage(
            "s3://my-bucket/my-dir",
            client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
        )
        ```
    """
    from .datachain import DataChain
    from .datasets import read_dataset
    from .records import read_records
    from .values import read_values

    file_type = get_file_type(type)

    if anon is not None:
        client_config = (client_config or {}) | {"anon": anon}
    session = Session.get(session, client_config=client_config, in_memory=in_memory)
    catalog = session.catalog
    cache = catalog.cache
    client_config = session.catalog.client_config
    listing_namespace_name = catalog.metastore.system_namespace_name
    listing_project_name = catalog.metastore.listing_project_name

    uris = uri if isinstance(uri, (list, tuple)) else [uri]

    if not uris:
        raise ValueError("No URIs provided")

    # Then expand all URIs that contain brace patterns
    expanded_uris = []
    for single_uri in uris:
        uri_str = str(single_uri)
        validate_cloud_bucket_name(uri_str)
        expanded_uris.extend(expand_brace_pattern(uri_str))

    # Now process each expanded URI
    chains = []
    listed_ds_name = set()
    file_values = []

    updated_uris = set()

    for single_uri in expanded_uris:
        # Check if URI contains glob patterns and split them
        base_uri, glob_pattern = split_uri_pattern(single_uri)

        # If a pattern is found, use the base_uri for listing
        # The pattern will be used for filtering later
        list_uri_to_use = base_uri if glob_pattern else single_uri

        # Avoid double updates for the same URI
        update_single_uri = False
        if update and (list_uri_to_use not in updated_uris):
            updated_uris.add(list_uri_to_use)
            update_single_uri = True

        list_ds_name, list_uri, list_path, list_ds_exists = get_listing(
            list_uri_to_use, session, update=update_single_uri
        )

        # list_ds_name is None if object is a file, we don't want to use cache
        # or do listing in that case - just read that single object
        if not list_ds_name:
            file_values.append(
                get_file_info(list_uri, cache, client_config=client_config)
            )
            continue

        dc = read_dataset(
            list_ds_name,
            namespace=listing_namespace_name,
            project=listing_project_name,
            session=session,
            settings=settings,
        )
        dc._query.update = update
        dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})

        if update or not list_ds_exists:

            def lst_fn(ds_name, lst_uri):
                # disable prefetch for listing, as it pre-downloads all files
                (
                    read_records(
                        DataChain.DEFAULT_FILE_RECORD,
                        session=session,
                        settings=settings,
                        in_memory=in_memory,
                    )
                    .settings(
                        prefetch=0,
                        namespace=listing_namespace_name,
                        project=listing_project_name,
                    )
                    .gen(
                        list_bucket(lst_uri, cache, client_config=client_config),
                        output={f"{column}": file_type},
                    )
                    # for internal listing datasets, we always bump major version
                    .save(ds_name, listing=True, update_version="major")
                )

            dc._query.set_listing_fn(
                lambda ds_name=list_ds_name, lst_uri=list_uri: lst_fn(ds_name, lst_uri)
            )

        # If a glob pattern was detected, use it for filtering
        # Otherwise, use the original list_path from get_listing
        if glob_pattern:
            # Determine if we should use recursive listing based on the pattern
            use_recursive = should_use_recursion(glob_pattern, recursive or False)

            # Apply glob filter - no need for brace expansion here as it's done above
            chain = apply_glob_filter(
                dc, glob_pattern, list_path, use_recursive, column
            )
            chains.append(chain)
        else:
            # No glob pattern detected, use normal ls behavior
            chains.append(ls(dc, list_path, recursive=recursive, column=column))

        listed_ds_name.add(list_ds_name)

    storage_chain = None if not chains else reduce(lambda x, y: x.union(y), chains)

    if file_values:
        file_chain = read_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            file=file_values,
        )
        file_chain.signals_schema = file_chain.signals_schema.mutate(
            {f"{column}": file_type}
        )
        storage_chain = storage_chain.union(file_chain) if storage_chain else file_chain

    assert storage_chain is not None

    if delta:
        storage_chain = storage_chain._as_delta(
            on=delta_on,
            right_on=delta_result_on,
            compare=delta_compare,
            delta_retry=delta_retry,
            delta_unsafe=delta_unsafe,
        )

    return storage_chain

read_values 

read_values(
    ds_name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    output: OutputType = None,
    column: str = "",
    **fr_map
) -> DataChain

Generate chain from list of values.

Example 
import datachain as dc
dc.read_values(fib=[1, 2, 3, 5, 8])
Source code in datachain/lib/dc/values.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def read_values(
    ds_name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    output: OutputType = None,
    column: str = "",
    **fr_map,
) -> "DataChain":
    """Generate chain from list of values.

    Example:
        ```py
        import datachain as dc
        dc.read_values(fib=[1, 2, 3, 5, 8])
        ```
    """
    from .datachain import DataChain

    tuple_type, output, tuples = values_to_tuples(ds_name, output, **fr_map)

    def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
        yield from tuples

    _func_fr.__name__ = "read_values"

    chain = read_records(
        DataChain.DEFAULT_FILE_RECORD,
        session=session,
        settings=settings,
        in_memory=in_memory,
    )
    if column:
        output = {column: dict_to_data_model(column, output)}  # type: ignore[arg-type]
    return chain.gen(_func_fr, output=output)

read_database 

read_database(
    query: Union[str, Executable],
    connection: ConnectionType,
    params: Union[
        Sequence[Mapping[str, Any]], Mapping[str, Any], None
    ] = None,
    *,
    output: Optional[dict[str, DataType]] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    infer_schema_length: Optional[int] = 100
) -> DataChain

Read the results of a SQL query into a DataChain, using a given database connection.

Parameters:

  • query (Union[str, Executable]) –

    The SQL query to execute. Can be a raw SQL string or a SQLAlchemy Executable object.

  • connection (ConnectionType) –

    SQLAlchemy connectable, str, or a sqlite3 connection Using SQLAlchemy makes it possible to use any DB supported by that library. If a DBAPI2 object, only sqlite3 is supported. The user is responsible for engine disposal and connection closure for the SQLAlchemy connectable; str connections are closed automatically.

  • params (Union[Sequence[Mapping[str, Any]], Mapping[str, Any], None], default: None ) –

    Parameters to pass to execute method.

  • output (Optional[dict[str, DataType]], default: None ) –

    A dictionary mapping column names to types, used to override the schema inferred from the query results.

  • session (Optional[Session], default: None ) –

    Session to use for the chain.

  • settings (Optional[dict], default: None ) –

    Settings to use for the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • infer_schema_length (Optional[int], default: 100 ) –

    The maximum number of rows to scan for inferring schema. If set to None, the full data may be scanned. The rows used for schema inference are stored in memory, so large values can lead to high memory usage. Only applies if the output parameter is not set for the given column.

Examples:

Reading from a SQL query against a user-supplied connection: 

query = "SELECT key, value FROM tbl"
chain = dc.read_database(query, connection, output={"value": float})

Load data from a SQLAlchemy driver/engine: 

from sqlalchemy import create_engine
engine = create_engine("postgresql+psycopg://myuser:mypassword@localhost:5432/mydb")
chain = dc.read_database("select * from tbl", engine)

Load data from a parameterized SQLAlchemy query: 

query = "SELECT key, value FROM tbl WHERE value > :value"
dc.read_database(query, engine, params={"value": 50})

Notes
  • This function works with a variety of databases — including, but not limited to, SQLite, DuckDB, PostgreSQL, and Snowflake, provided the appropriate driver is installed.
  • This call is blocking, and will execute the query and return once the results are saved.
Source code in datachain/lib/dc/database.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
def read_database(
    query: Union[str, "sqlalchemy.sql.expression.Executable"],
    connection: "ConnectionType",
    params: Union["Sequence[Mapping[str, Any]]", "Mapping[str, Any]", None] = None,
    *,
    output: Optional["dict[str, DataType]"] = None,
    session: Optional["Session"] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    infer_schema_length: Optional[int] = 100,
) -> "DataChain":
    """
    Read the results of a SQL query into a DataChain, using a given database connection.

    Args:
        query:
            The SQL query to execute. Can be a raw SQL string or a SQLAlchemy
            `Executable` object.
        connection: SQLAlchemy connectable, str, or a sqlite3 connection
            Using SQLAlchemy makes it possible to use any DB supported by that
            library. If a DBAPI2 object, only sqlite3 is supported. The user is
            responsible for engine disposal and connection closure for the
            SQLAlchemy connectable; str connections are closed automatically.
        params: Parameters to pass to execute method.
        output: A dictionary mapping column names to types, used to override the
            schema inferred from the query results.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        infer_schema_length:
            The maximum number of rows to scan for inferring schema.
            If set to `None`, the full data may be scanned.
            The rows used for schema inference are stored in memory,
            so large values can lead to high memory usage.
            Only applies if the `output` parameter is not set for the given column.

    Examples:
        Reading from a SQL query against a user-supplied connection:
        ```python
        query = "SELECT key, value FROM tbl"
        chain = dc.read_database(query, connection, output={"value": float})
        ```

        Load data from a SQLAlchemy driver/engine:
        ```python
        from sqlalchemy import create_engine
        engine = create_engine("postgresql+psycopg://myuser:mypassword@localhost:5432/mydb")
        chain = dc.read_database("select * from tbl", engine)
        ```

        Load data from a parameterized SQLAlchemy query:
        ```python
        query = "SELECT key, value FROM tbl WHERE value > :value"
        dc.read_database(query, engine, params={"value": 50})
        ```

    Notes:
        - This function works with a variety of databases — including,
        but not limited to, SQLite, DuckDB, PostgreSQL, and Snowflake,
        provided the appropriate driver is installed.
        - This call is blocking, and will execute the query and return once the
          results are saved.
    """
    from datachain.lib.dc.records import read_records

    output = output or {}
    if isinstance(query, str):
        query = sqlalchemy.text(query)
    kw = {"execution_options": {"stream_results": True}}  # use server-side cursors
    with _connect(connection) as conn, conn.execute(query, params, **kw) as result:
        cols = result.keys()
        to_infer = [k for k in cols if k not in output]  # preserve the order
        rows, inferred_schema = _infer_schema(result, to_infer, infer_schema_length)
        records = (row._asdict() for row in itertools.chain(rows, result))
        return read_records(
            records,
            session=session,
            settings=settings,
            in_memory=in_memory,
            schema=inferred_schema | output,
        )

ConnectionType module-attribute 

ConnectionType = Union[
    str,
    URL,
    Connectable,
    Engine,
    Connection,
    Session,
    Connection,
]

DataChain 

DataChain(
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: Optional[dict] = None,
    _sys: bool = False,
)

DataChain - a data structure for batch data processing and evaluation.

It represents a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data.

Data in DataChain is presented as Python classes with arbitrary set of fields, including nested classes. The data classes have to inherit from DataModel class. The supported set of field types include: majority of the type supported by the underlyind library Pydantic.

See Also

read_storage("s3://my-bucket/my-dir/") - reading unstructured data files from storages such as S3, gs or Azure ADLS.

DataChain.save("name") - saving to a dataset.

read_dataset("name") - reading from a dataset.

read_values(fib=[1, 2, 3, 5, 8]) - generating from values.

read_pandas(pd.DataFrame(...)) - generating from pandas.

read_json("file.json") - generating from json.

read_csv("file.csv") - generating from csv.

read_parquet("file.parquet") - generating from parquet.

Example 
import os

from mistralai.client import MistralClient
from mistralai.models.chat_completion import ChatMessage
import datachain as dc

PROMPT = (
    "Was this bot dialog successful? "
    "Describe the 'result' as 'Yes' or 'No' in a short JSON"
)

model = "mistral-large-latest"
api_key = os.environ["MISTRAL_API_KEY"]

chain = (
    dc.read_storage("gs://datachain-demo/chatbot-KiT/")
    .limit(5)
    .settings(cache=True, parallel=5)
    .map(
        mistral_response=lambda file: MistralClient(api_key=api_key)
        .chat(
            model=model,
            response_format={"type": "json_object"},
            messages=[
                ChatMessage(role="user", content=f"{PROMPT}: {file.read()}")
            ],
        )
        .choices[0]
        .message.content,
    )
    .persist()
)

try:
    print(chain.select("mistral_response").results())
except Exception as e:
    print(f"do you have the right Mistral API key? {e}")
Source code in datachain/lib/dc/datachain.py 
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
def __init__(
    self,
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: Optional[dict] = None,
    _sys: bool = False,
) -> None:
    """Don't instantiate this directly, use one of the from_XXX constructors."""
    self._query = query
    self._settings = settings
    self.signals_schema = signal_schema
    self._setup: dict = setup or {}
    self._sys = _sys
    self._delta = False
    self._delta_unsafe = False
    self._delta_on: Optional[Union[str, Sequence[str]]] = None
    self._delta_result_on: Optional[Union[str, Sequence[str]]] = None
    self._delta_compare: Optional[Union[str, Sequence[str]]] = None
    self._delta_retry: Optional[Union[bool, str]] = None

dataset property 

dataset: Optional[DatasetRecord]

Underlying dataset, if there is one.

delta property 

delta: bool

Returns True if this chain is ran in "delta" update mode

empty property 

empty: bool

Returns True if chain has zero number of rows

name property 

name: Optional[str]

Name of the underlying dataset, if there is one.

namespace_name property 

namespace_name: str

Current namespace name in which the chain is running

project_name property 

project_name: str

Current project name in which the chain is running

schema property 

schema: dict[str, DataType]

Get schema of the chain.

session property 

session: Session

Session of the chain.

version property 

version: Optional[str]

Version of the underlying dataset, if there is one.

__iter__ 

__iter__() -> Iterator[tuple[DataValue, ...]]

Make DataChain objects iterable.

Yields:

  • tuple[DataValue, ...] –

    Yields tuples of all column values for each row.

Example 
for row in chain:
    print(row)
Source code in datachain/lib/dc/datachain.py 
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
def __iter__(self) -> Iterator[tuple[DataValue, ...]]:
    """Make DataChain objects iterable.

    Yields:
        (tuple[DataValue, ...]): Yields tuples of all column values for each row.

    Example:
        ```py
        for row in chain:
            print(row)
        ```
    """
    return self.to_iter()

__or__ 

__or__(other: Self) -> Self

Return self.union(other).

Source code in datachain/lib/dc/datachain.py 
297
298
299
def __or__(self, other: "Self") -> "Self":
    """Return `self.union(other)`."""
    return self.union(other)

__repr__ 

__repr__() -> str

Return a string representation of the chain.

Source code in datachain/lib/dc/datachain.py 
198
199
200
201
202
203
204
205
206
207
208
def __repr__(self) -> str:
    """Return a string representation of the chain."""
    classname = self.__class__.__name__
    if not self._effective_signals_schema.values:
        return f"Empty {classname}"

    import io

    file = io.StringIO()
    self.print_schema(file=file)
    return file.getvalue()

agg 

agg(
    func: Optional[Callable] = None,
    partition_by: Optional[PartitionByType] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map: Callable
) -> Self

Aggregate rows using partition_by statement and apply a function to the groups of aggregated rows. The function needs to return new objects for each group of the new rows. It returns a chain itself with new signals.

Input-output relationship: N:M

This method bears similarity to gen() and map(), employing a comparable set of parameters, yet differs in two crucial aspects:

  1. The partition_by parameter: This specifies the column name or a list of column names that determine the grouping criteria for aggregation.
  2. Group-based UDF function input: Instead of individual rows, the function receives a list of all rows within each group defined by partition_by.

If partition_by is not set or is an empty list, all rows will be placed into a single group.

Parameters:

  • func (Optional[Callable], default: None ) –

    Function applied to each group of rows.

  • partition_by (Optional[PartitionByType], default: None ) –

    Column name(s) to group by. If None, all rows go into one group.

  • params (Union[None, str, Sequence[str]], default: None ) –

    List of column names used as input for the function. Default is taken from function signature.

  • output (OutputType, default: None ) –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature.

  • **signal_map (Callable, default: {} ) –

    kwargs can be used to define func together with its return signal name in format of agg(result_column=my_func).

Examples:

Basic aggregation with lambda function: 

chain = chain.agg(
    total=lambda category, amount: [sum(amount)],
    output=float,
    partition_by="category",
)
chain.save("new_dataset")

An alternative syntax, when you need to specify a more complex function: 

# It automatically resolves which columns to pass to the function
# by looking at the function signature.
def agg_sum(
    file: list[File], amount: list[float]
) -> Iterator[tuple[File, float]]:
    yield file[0], sum(amount)

chain = chain.agg(
    agg_sum,
    output={"file": File, "total": float},
    # Alternative syntax is to use `C` (short for Column) to specify
    # a column name or a nested column, e.g. C("file.path").
    partition_by=C("category"),
)
chain.save("new_dataset")

Using complex signals for partitioning (File or any Pydantic BaseModel): 

def my_agg(files: list[File]) -> Iterator[tuple[File, int]]:
    yield files[0], sum(f.size for f in files)

chain = chain.agg(
    my_agg,
    params=("file",),
    output={"file": File, "total": int},
    partition_by="file",  # Column referring to all sub-columns of File
)
chain.save("new_dataset")

Aggregating all rows into a single group (when partition_by is not set): 

chain = chain.agg(
    total_size=lambda file, size: [sum(size)],
    output=int,
    # No partition_by specified - all rows go into one group
)
chain.save("new_dataset")

Multiple partition columns: 

chain = chain.agg(
    total=lambda category, subcategory, amount: [sum(amount)],
    output=float,
    partition_by=["category", "subcategory"],
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
@delta_disabled
def agg(
    self,
    /,
    func: Optional[Callable] = None,
    partition_by: Optional[PartitionByType] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map: Callable,
) -> "Self":
    """Aggregate rows using `partition_by` statement and apply a function to the
    groups of aggregated rows. The function needs to return new objects for each
    group of the new rows. It returns a chain itself with new signals.

    Input-output relationship: N:M

    This method bears similarity to `gen()` and `map()`, employing a comparable set
    of parameters, yet differs in two crucial aspects:

    1. The `partition_by` parameter: This specifies the column name or a list of
       column names that determine the grouping criteria for aggregation.
    2. Group-based UDF function input: Instead of individual rows, the function
       receives a list of all rows within each group defined by `partition_by`.

    If `partition_by` is not set or is an empty list, all rows will be placed
    into a single group.

    Parameters:
        func: Function applied to each group of rows.
        partition_by: Column name(s) to group by. If None, all rows go
            into one group.
        params: List of column names used as input for the function. Default is
            taken from function signature.
        output: Dictionary defining new signals and their corresponding types.
            Default type is taken from function signature.
        **signal_map: kwargs can be used to define `func` together with its return
            signal name in format of `agg(result_column=my_func)`.

    Examples:
        Basic aggregation with lambda function:
        ```py
        chain = chain.agg(
            total=lambda category, amount: [sum(amount)],
            output=float,
            partition_by="category",
        )
        chain.save("new_dataset")
        ```

        An alternative syntax, when you need to specify a more complex function:
        ```py
        # It automatically resolves which columns to pass to the function
        # by looking at the function signature.
        def agg_sum(
            file: list[File], amount: list[float]
        ) -> Iterator[tuple[File, float]]:
            yield file[0], sum(amount)

        chain = chain.agg(
            agg_sum,
            output={"file": File, "total": float},
            # Alternative syntax is to use `C` (short for Column) to specify
            # a column name or a nested column, e.g. C("file.path").
            partition_by=C("category"),
        )
        chain.save("new_dataset")
        ```

        Using complex signals for partitioning (`File` or any Pydantic `BaseModel`):
        ```py
        def my_agg(files: list[File]) -> Iterator[tuple[File, int]]:
            yield files[0], sum(f.size for f in files)

        chain = chain.agg(
            my_agg,
            params=("file",),
            output={"file": File, "total": int},
            partition_by="file",  # Column referring to all sub-columns of File
        )
        chain.save("new_dataset")
        ```

        Aggregating all rows into a single group (when `partition_by` is not set):
        ```py
        chain = chain.agg(
            total_size=lambda file, size: [sum(size)],
            output=int,
            # No partition_by specified - all rows go into one group
        )
        chain.save("new_dataset")
        ```

        Multiple partition columns:
        ```py
        chain = chain.agg(
            total=lambda category, subcategory, amount: [sum(amount)],
            output=float,
            partition_by=["category", "subcategory"],
        )
        chain.save("new_dataset")
        ```
    """
    if partition_by is not None:
        # Convert string partition_by parameters to Column objects
        if isinstance(partition_by, (str, Function, ColumnElement)):
            list_partition_by = [partition_by]
        else:
            list_partition_by = list(partition_by)

        processed_partition_columns: list[ColumnElement] = []
        for col in list_partition_by:
            if isinstance(col, str):
                columns = self.signals_schema.db_signals(name=col, as_columns=True)
                if not columns:
                    raise SignalResolvingError([col], "is not found")
                processed_partition_columns.extend(cast("list[Column]", columns))
            elif isinstance(col, Function):
                column = col.get_column(self.signals_schema)
                processed_partition_columns.append(column)
            else:
                # Assume it's already a ColumnElement
                processed_partition_columns.append(col)

        processed_partition_by = processed_partition_columns
    else:
        processed_partition_by = []

    udf_obj = self._udf_to_obj(Aggregator, func, params, output, signal_map)
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(self._settings.batch_size),
            partition_by=processed_partition_by,
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

apply 

apply(func, *args, **kwargs)

Apply any function to the chain.

Useful for reusing in a chain of operations.

Example 
import datachain as dc
def parse_stem(chain):
    return chain.map(
        lambda file: file.get_file_stem()
        output={"stem": str}
    )

chain = (
    dc.read_storage("s3://my-bucket")
    .apply(parse_stem)
    .filter(C("stem").glob("*cat*"))
)
Source code in datachain/lib/dc/datachain.py 
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
def apply(self, func, *args, **kwargs):
    """Apply any function to the chain.

    Useful for reusing in a chain of operations.

    Example:
        ```py
        import datachain as dc
        def parse_stem(chain):
            return chain.map(
                lambda file: file.get_file_stem()
                output={"stem": str}
            )

        chain = (
            dc.read_storage("s3://my-bucket")
            .apply(parse_stem)
            .filter(C("stem").glob("*cat*"))
        )
        ```
    """
    return func(self, *args, **kwargs)

avg 

avg(col: str) -> StandardType

Compute the average of a column.

Parameters:

  • col (str) –

    The column to compute the average for.

Returns:

  • StandardType –

    The average of the column values.

Example 
average_size = chain.avg("file.size")
print(f"Average size: {average_size}")
Source code in datachain/lib/dc/datachain.py 
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
def avg(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the average of a column.

    Parameters:
        col: The column to compute the average for.

    Returns:
        The average of the column values.

    Example:
        ```py
        average_size = chain.avg("file.size")
        print(f"Average size: {average_size}")
        ```
    """
    return self._extend_to_data_model("avg", col)

batch_map 

batch_map(
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map
) -> Self

This is a batch version of map().

Input-output relationship: N:N

It accepts the same parameters plus an additional parameter:

batch: Size of each batch passed to `func`. Defaults to 1000.
Example 
chain = chain.batch_map(
    sqrt=lambda size: np.sqrt(size),
    output=float
)
chain.save("new_dataset")

.. deprecated:: 0.29.0 This method is deprecated and will be removed in a future version. Use agg() instead, which provides the similar functionality.

Source code in datachain/lib/dc/datachain.py 
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
def batch_map(
    self,
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map,
) -> "Self":
    """This is a batch version of `map()`.

    Input-output relationship: N:N

    It accepts the same parameters plus an
    additional parameter:

        batch: Size of each batch passed to `func`. Defaults to 1000.

    Example:
        ```py
        chain = chain.batch_map(
            sqrt=lambda size: np.sqrt(size),
            output=float
        )
        chain.save("new_dataset")
        ```

    .. deprecated:: 0.29.0
        This method is deprecated and will be removed in a future version.
        Use `agg()` instead, which provides the similar functionality.
    """
    import warnings

    warnings.warn(
        "batch_map() is deprecated and will be removed in a future version. "
        "Use agg() instead, which provides the similar functionality.",
        DeprecationWarning,
        stacklevel=2,
    )
    udf_obj = self._udf_to_obj(BatchMapper, func, params, output, signal_map)

    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(self._settings.batch_size, batch=batch),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

c 

c(column: Union[str, Column]) -> Column

Returns Column instance attached to the current chain.

Source code in datachain/lib/dc/datachain.py 
265
266
267
268
269
def c(self, column: Union[str, Column]) -> Column:
    """Returns Column instance attached to the current chain."""
    c = self.column(column) if isinstance(column, str) else self.column(column.name)
    c.table = self._query.table
    return c

chunk 

chunk(index: int, total: int) -> Self

Split a chain into smaller chunks for e.g. parallelization.

Parameters:

  • index (int) –

    The index of the chunk (0-indexed).

  • total (int) –

    The total number of chunks.

Example 
import datachain as dc

chain = dc.read_storage(...)
chunk_1 = query._chunk(0, 2)
chunk_2 = query._chunk(1, 2)
Note

Bear in mind that index is 0-indexed but total isn't. Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.

Source code in datachain/lib/dc/datachain.py 
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
def chunk(self, index: int, total: int) -> "Self":
    """Split a chain into smaller chunks for e.g. parallelization.

    Parameters:
        index: The index of the chunk (0-indexed).
        total: The total number of chunks.

    Example:
        ```py
        import datachain as dc

        chain = dc.read_storage(...)
        chunk_1 = query._chunk(0, 2)
        chunk_2 = query._chunk(1, 2)
        ```

    Note:
        Bear in mind that `index` is 0-indexed but `total` isn't.
        Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.
    """
    return self._evolve(query=self._query.chunk(index, total))

clone 

clone() -> Self

Make a copy of the chain in a new table.

Source code in datachain/lib/dc/datachain.py 
305
306
307
def clone(self) -> "Self":
    """Make a copy of the chain in a new table."""
    return self._evolve(query=self._query.clone(new_table=True))

collect 

collect() -> Iterator[tuple[DataValue, ...]]

collect(col: str) -> Iterator[DataValue]

collect(*cols: str) -> Iterator[tuple[DataValue, ...]]

collect(
    *cols: str,
) -> Iterator[Union[DataValue, tuple[DataValue, ...]]]

Deprecated. Use to_iter method instead.

Source code in datachain/lib/dc/datachain.py 
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
def collect(self, *cols: str) -> Iterator[Union[DataValue, tuple[DataValue, ...]]]:  # type: ignore[overload-overlap,misc]
    """
    Deprecated. Use `to_iter` method instead.
    """
    warnings.warn(
        "Method `collect` is deprecated. Use `to_iter` method instead.",
        DeprecationWarning,
        stacklevel=2,
    )

    if len(cols) == 1:
        yield from [item[0] for item in self.to_iter(*cols)]
    else:
        yield from self.to_iter(*cols)

column 

column(name: str) -> Column

Returns Column instance with a type if name is found in current schema, otherwise raises an exception.

Source code in datachain/lib/dc/datachain.py 
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
def column(self, name: str) -> Column:
    """Returns Column instance with a type if name is found in current schema,
    otherwise raises an exception.
    """
    if "." in name:
        name_path = name.split(".")
    elif DEFAULT_DELIMITER in name:
        name_path = name.split(DEFAULT_DELIMITER)
    else:
        name_path = [name]
    for path, type_, _, _ in self.signals_schema.get_flat_tree():
        if path == name_path:
            return Column(name, python_to_sql(type_))

    raise ValueError(f"Column with name {name} not found in the schema")

count 

count() -> int

Return the number of rows in the chain.

Source code in datachain/lib/dc/datachain.py 
2734
2735
2736
def count(self) -> int:
    """Return the number of rows in the chain."""
    return self._query.count()

diff 

diff(
    other: DataChain,
    on: Union[str, Sequence[str]],
    right_on: Optional[Union[str, Sequence[str]]] = None,
    compare: Optional[Union[str, Sequence[str]]] = None,
    right_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: Optional[str] = None,
) -> DataChain

Calculate differences between two chains.

This method identifies records that are added, deleted, modified, or unchanged between two chains. It adds a status column with values: A=added, D=deleted, M=modified, S=same.

Parameters:

  • other (DataChain) –

    Chain to compare against.

  • on (Union[str, Sequence[str]]) –

    Column(s) to match records between chains.

  • right_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Column(s) in the other chain to match against. Defaults to on.

  • compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Column(s) to check for changes. If not specified,all columns are used.

  • right_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Column(s) in the other chain to compare against. Defaults to values of compare.

  • added (bool, default: True ) –

    Include records that exist in this chain but not in the other.

  • deleted (bool, default: True ) –

    Include records that exist only in the other chain.

  • modified (bool, default: True ) –

    Include records that exist in both but have different values.

  • same (bool, default: False ) –

    Include records that are identical in both chains.

  • status_col (str, default: None ) –

    Name for the status column showing differences.

Default behavior: By default, shows added, deleted, and modified records, but excludes unchanged records (same=False). Status column is not created.

Example 
res = persons.diff(
    new_persons,
    on=["id"],
    right_on=["other_id"],
    compare=["name"],
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
def diff(
    self,
    other: "DataChain",
    on: Union[str, Sequence[str]],
    right_on: Optional[Union[str, Sequence[str]]] = None,
    compare: Optional[Union[str, Sequence[str]]] = None,
    right_compare: Optional[Union[str, Sequence[str]]] = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: Optional[str] = None,
) -> "DataChain":
    """Calculate differences between two chains.

    This method identifies records that are added, deleted, modified, or unchanged
    between two chains. It adds a status column with values: A=added, D=deleted,
    M=modified, S=same.

    Parameters:
        other: Chain to compare against.
        on: Column(s) to match records between chains.
        right_on: Column(s) in the other chain to match against. Defaults to `on`.
        compare: Column(s) to check for changes.
                 If not specified,all columns are used.
        right_compare: Column(s) in the other chain to compare against.
                 Defaults to values of `compare`.
        added (bool): Include records that exist in this chain but not in the other.
        deleted (bool): Include records that exist only in the other chain.
        modified (bool): Include records that exist in both
                 but have different values.
        same (bool): Include records that are identical in both chains.
        status_col (str): Name for the status column showing differences.

    Default behavior: By default, shows added, deleted, and modified records,
    but excludes unchanged records (same=False). Status column is not created.

    Example:
        ```py
        res = persons.diff(
            new_persons,
            on=["id"],
            right_on=["other_id"],
            compare=["name"],
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    from datachain.diff import _compare

    return _compare(
        self,
        other,
        on,
        right_on=right_on,
        compare=compare,
        right_compare=right_compare,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

distinct 

distinct(arg: str, *args: str) -> Self

Removes duplicate rows based on uniqueness of some input column(s) i.e if rows are found with the same value of input column(s), only one row is left in the result set.

Example 
dc.distinct("file.path")
Source code in datachain/lib/dc/datachain.py 
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
@delta_disabled
def distinct(self, arg: str, *args: str) -> "Self":  # type: ignore[override]
    """Removes duplicate rows based on uniqueness of some input column(s)
    i.e if rows are found with the same value of input column(s), only one
    row is left in the result set.

    Example:
        ```py
        dc.distinct("file.path")
        ```
    """
    return self._evolve(
        query=self._query.distinct(
            *self.signals_schema.resolve(arg, *args).db_signals()
        )
    )

exec 

exec() -> Self

Execute the chain.

Source code in datachain/lib/dc/datachain.py 
2738
2739
2740
def exec(self) -> "Self":
    """Execute the chain."""
    return self._evolve(query=self._query.exec())

explode 

explode(
    col: str,
    model_name: Optional[str] = None,
    column: Optional[str] = None,
    schema_sample_size: int = 1,
) -> DataChain

Explodes a column containing JSON objects (dict or str DataChain type) into individual columns based on the schema of the JSON. Schema is inferred from the first row of the column.

Parameters:

  • col (str) –

    the name of the column containing JSON to be exploded.

  • model_name (Optional[str], default: None ) –

    optional generated model name. By default generates the name automatically.

  • column (Optional[str], default: None ) –

    optional generated column name. By default generates the name automatically.

  • schema_sample_size (int, default: 1 ) –

    the number of rows to use for inferring the schema of the JSON (in case some fields are optional and it's not enough to analyze a single row).

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance with the new set of columns.

Source code in datachain/lib/dc/datachain.py 
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
def explode(
    self,
    col: str,
    model_name: Optional[str] = None,
    column: Optional[str] = None,
    schema_sample_size: int = 1,
) -> "DataChain":
    """Explodes a column containing JSON objects (dict or str DataChain type) into
       individual columns based on the schema of the JSON. Schema is inferred from
       the first row of the column.

    Args:
        col: the name of the column containing JSON to be exploded.
        model_name: optional generated model name.  By default generates the name
            automatically.
        column: optional generated column name. By default generates the
            name automatically.
        schema_sample_size: the number of rows to use for inferring the schema of
            the JSON (in case some fields are optional and it's not enough to
            analyze a single row).

    Returns:
        DataChain: A new DataChain instance with the new set of columns.
    """
    import pyarrow as pa

    from datachain.lib.arrow import schema_to_output

    json_values = self.limit(schema_sample_size).to_list(col)
    json_dicts = [
        json.loads(json_value) if isinstance(json_value, str) else json_value
        for (json_value,) in json_values
    ]

    if any(not isinstance(json_dict, dict) for json_dict in json_dicts):
        raise TypeError(f"Column {col} should be a string or dict type with JSON")

    schema = pa.Table.from_pylist(json_dicts).schema
    output, original_names = schema_to_output(schema, None)

    if not model_name:
        model_name = f"{col.title()}ExplodedModel"

    model = dict_to_data_model(model_name, output, original_names)

    def json_to_model(json_value: Union[str, dict]):
        json_dict = (
            json.loads(json_value) if isinstance(json_value, str) else json_value
        )
        return model.model_validate(json_dict)

    if not column:
        column = f"{col}_expl"

    return self.map(json_to_model, params=col, output={column: model})

file_diff 

file_diff(
    other: DataChain,
    on: str = "file",
    right_on: Optional[str] = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: Optional[str] = None,
) -> DataChain

Calculate differences between two chains containing files.

This method is specifically designed for file chains. It uses file source and path to match files, and file version and etag to detect changes.

Parameters:

  • other (DataChain) –

    Chain to compare against.

  • on (str, default: 'file' ) –

    File column name in this chain. Default is "file".

  • right_on (Optional[str], default: None ) –

    File column name in the other chain. Defaults to on.

  • added (bool, default: True ) –

    Include files that exist in this chain but not in the other.

  • deleted (bool, default: False ) –

    Include files that exist only in the other chain.

  • modified (bool, default: True ) –

    Include files that exist in both but have different versions/etags.

  • same (bool, default: False ) –

    Include files that are identical in both chains.

  • status_col (str, default: None ) –

    Name for the status column showing differences (A=added, D=deleted, M=modified, S=same).

Default behavior: By default, includes only new files (added=True and modified=True). This is useful for incremental processing.

Example 
diff = images.file_diff(
    new_images,
    on="file",
    right_on="other_file",
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
def file_diff(
    self,
    other: "DataChain",
    on: str = "file",
    right_on: Optional[str] = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: Optional[str] = None,
) -> "DataChain":
    """Calculate differences between two chains containing files.

    This method is specifically designed for file chains. It uses file `source`
    and `path` to match files, and file `version` and `etag` to detect changes.

    Parameters:
        other: Chain to compare against.
        on: File column name in this chain. Default is "file".
        right_on: File column name in the other chain. Defaults to `on`.
        added (bool): Include files that exist in this chain but not in the other.
        deleted (bool): Include files that exist only in the other chain.
        modified (bool): Include files that exist in both but have different
                         versions/etags.
        same (bool): Include files that are identical in both chains.
        status_col (str): Name for the status column showing differences
                          (A=added, D=deleted, M=modified, S=same).

    Default behavior: By default, includes only new files (added=True and
    modified=True). This is useful for incremental processing.

    Example:
        ```py
        diff = images.file_diff(
            new_images,
            on="file",
            right_on="other_file",
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    on_file_signals = ["source", "path"]
    compare_file_signals = ["version", "etag"]

    def get_file_signals(file: str, signals):
        return [f"{file}.{c}" for c in signals]

    right_on = right_on or on

    on_cols = get_file_signals(on, on_file_signals)
    right_on_cols = get_file_signals(right_on, on_file_signals)
    compare_cols = get_file_signals(on, compare_file_signals)
    right_compare_cols = get_file_signals(right_on, compare_file_signals)

    return self.diff(
        other,
        on_cols,
        right_on=right_on_cols,
        compare=compare_cols,
        right_compare=right_compare_cols,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

filter 

filter(*args: Any) -> Self

Filter the chain according to conditions.

Example

Basic usage with built-in operators 

dc.filter(C("width") < 200)

Using glob to match patterns 

dc.filter(C("file.path").glob("*.jpg"))

Using in to match lists 

ids = [1,2,3]
dc.filter(C("experiment_id").in_(ids))

Using datachain.func 

from datachain.func import string
dc.filter(string.length(C("file.path")) > 5)

Combining filters with "or" 

dc.filter(
    C("file.path").glob("cat*") |
    C("file.path").glob("dog*")
)


dc.filter(dc.func.or_(
    C("file.path").glob("cat*"),
    C("file.path").glob("dog*")
))

Combining filters with "and" 

dc.filter(
    C("file.path").glob("*.jpg"),
    string.length(C("file.path")) > 5
)


dc.filter(
    C("file.path").glob("*.jpg") &
    (string.length(C("file.path")) > 5)
)

dc.filter(dc.func.and_(
    C("file.path").glob("*.jpg"),
    string.length(C("file.path")) > 5
))

Combining filters with "not" 

dc.filter(~(C("file.path").glob("*.jpg")))

Source code in datachain/lib/dc/datachain.py 
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
def filter(self, *args: Any) -> "Self":
    """Filter the chain according to conditions.

    Example:
        Basic usage with built-in operators
        ```py
        dc.filter(C("width") < 200)
        ```

        Using glob to match patterns
        ```py
        dc.filter(C("file.path").glob("*.jpg"))
        ```

        Using in to match lists
        ```py
        ids = [1,2,3]
        dc.filter(C("experiment_id").in_(ids))
        ```

        Using `datachain.func`
        ```py
        from datachain.func import string
        dc.filter(string.length(C("file.path")) > 5)
        ```

        Combining filters with "or"
        ```py
        dc.filter(
            C("file.path").glob("cat*") |
            C("file.path").glob("dog*")
        )
        ```

        ```py
        dc.filter(dc.func.or_(
            C("file.path").glob("cat*"),
            C("file.path").glob("dog*")
        ))
        ```

        Combining filters with "and"
        ```py
        dc.filter(
            C("file.path").glob("*.jpg"),
            string.length(C("file.path")) > 5
        )
        ```

        ```py
        dc.filter(
            C("file.path").glob("*.jpg") &
            (string.length(C("file.path")) > 5)
        )
        ```

        ```py
        dc.filter(dc.func.and_(
            C("file.path").glob("*.jpg"),
            string.length(C("file.path")) > 5
        ))
        ```

        Combining filters with "not"
        ```py
        dc.filter(~(C("file.path").glob("*.jpg")))
        ```
    """
    return self._evolve(query=self._query.filter(*args))

gen 

gen(
    func: Optional[Union[Callable, Generator]] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map
) -> Self

Apply a function to each row to create new rows (with potentially new signals). The function needs to return a new objects for each of the new rows. It returns a chain itself with new signals.

Input-output relationship: 1:N

This method is similar to map(), uses the same list of parameters, but with one key differences: It produces a sequence of rows for each input row (like extracting multiple file records from a single tar file or bounding boxes from a single image file).

Example 
chain = chain.gen(
    line=lambda file: [l for l in file.read().split("\n")],
    output=str,
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
def gen(
    self,
    func: Optional[Union[Callable, Generator]] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    r"""Apply a function to each row to create new rows (with potentially new
    signals). The function needs to return a new objects for each of the new rows.
    It returns a chain itself with new signals.

    Input-output relationship: 1:N

    This method is similar to `map()`, uses the same list of parameters, but with
    one key differences: It produces a sequence of rows for each input row (like
    extracting multiple file records from a single tar file or bounding boxes from a
    single image file).

    Example:
        ```py
        chain = chain.gen(
            line=lambda file: [l for l in file.read().split("\n")],
            output=str,
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Generator, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(self._settings.batch_size),
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

group_by 

group_by(
    *,
    partition_by: Optional[
        Union[str, Func, Sequence[Union[str, Func]]]
    ] = None,
    **kwargs: Func
) -> Self

Group rows by specified set of signals and return new signals with aggregated values.

The supported functions

count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

Example 
chain = chain.group_by(
    cnt=func.count(),
    partition_by=("file_source", "file_ext"),
)

Using complex signals: 

chain = chain.group_by(
    total_size=func.sum("file.size"),
    count=func.count(),
    partition_by="file",  # Uses column name, expands to File's unique keys
)

Source code in datachain/lib/dc/datachain.py 
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
@delta_disabled  # type: ignore[arg-type]
def group_by(  # noqa: C901, PLR0912
    self,
    *,
    partition_by: Optional[Union[str, Func, Sequence[Union[str, Func]]]] = None,
    **kwargs: Func,
) -> "Self":
    """Group rows by specified set of signals and return new signals
    with aggregated values.

    The supported functions:
       count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

    Example:
        ```py
        chain = chain.group_by(
            cnt=func.count(),
            partition_by=("file_source", "file_ext"),
        )
        ```

        Using complex signals:
        ```py
        chain = chain.group_by(
            total_size=func.sum("file.size"),
            count=func.count(),
            partition_by="file",  # Uses column name, expands to File's unique keys
        )
        ```
    """
    if partition_by is None:
        partition_by = []
    elif isinstance(partition_by, (str, Func)):
        partition_by = [partition_by]

    partition_by_columns: list[Column] = []
    signal_columns: list[Column] = []
    schema_fields: dict[str, DataType] = {}
    keep_columns: list[str] = []
    partial_fields: list[str] = []  # Track specific fields for partial creation
    schema_partition_by: list[str] = []

    for col in partition_by:
        if isinstance(col, str):
            columns = self.signals_schema.db_signals(name=col, as_columns=True)
            if not columns:
                raise SignalResolvingError([col], "is not found")
            partition_by_columns.extend(cast("list[Column]", columns))

            # For nested field references (e.g., "nested.level1.name"),
            # we need to distinguish between:
            # 1. References to fields within a complex signal (create partials)
            # 2. Deep nested references that should be flattened
            if "." in col:
                # Split the column reference to analyze it
                parts = col.split(".")
                parent_signal = parts[0]
                parent_type = self.signals_schema.values.get(parent_signal)

                if ModelStore.is_partial(parent_type):
                    if parent_signal not in keep_columns:
                        keep_columns.append(parent_signal)
                    partial_fields.append(col)
                    schema_partition_by.append(col)
                else:
                    # BaseModel or other - add flattened columns directly
                    for column in cast("list[Column]", columns):
                        col_type = self.signals_schema.get_column_type(column.name)
                        schema_fields[column.name] = col_type
                    schema_partition_by.append(col)
            else:
                # simple signal - but we need to check if it's a complex signal
                # complex signal - only include the columns used for partitioning
                col_type = self.signals_schema.get_column_type(
                    col, with_subtree=True
                )
                if isinstance(col_type, type) and issubclass(col_type, BaseModel):
                    # Complex signal - add only the partitioning columns
                    for column in cast("list[Column]", columns):
                        col_type = self.signals_schema.get_column_type(column.name)
                        schema_fields[column.name] = col_type
                    schema_partition_by.append(col)
                # Simple signal - keep the entire signal
                else:
                    if col not in keep_columns:
                        keep_columns.append(col)
                    schema_partition_by.append(col)
        elif isinstance(col, Function):
            column = col.get_column(self.signals_schema)
            col_db_name = column.name
            col_type = column.type.python_type
            schema_fields[col_db_name] = col_type
            partition_by_columns.append(column)
            signal_columns.append(column)
        else:
            raise DataChainColumnError(
                col,
                (
                    f"partition_by column {col} has type {type(col)}"
                    " but expected str or Function"
                ),
            )

    if not kwargs:
        raise ValueError("At least one column should be provided for group_by")
    for col_name, func in kwargs.items():
        if not isinstance(func, Func):
            raise DataChainColumnError(
                col_name,
                f"Column {col_name} has type {type(func)} but expected Func object",
            )
        column = func.get_column(self.signals_schema, label=col_name)
        signal_columns.append(column)
        schema_fields[col_name] = func.get_result_type(self.signals_schema)

    signal_schema = self.signals_schema.group_by(
        schema_partition_by, signal_columns
    )

    return self._evolve(
        query=self._query.group_by(signal_columns, partition_by_columns),
        signal_schema=signal_schema,
    )

limit 

limit(n: int) -> Self

Return the first n rows of the chain.

If the chain is unordered, which rows are returned is undefined. If the chain has less than n rows, the whole chain is returned.

Parameters:

  • n (int) –

    Number of rows to return.

Source code in datachain/lib/dc/datachain.py 
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
def limit(self, n: int) -> "Self":
    """Return the first `n` rows of the chain.

    If the chain is unordered, which rows are returned is undefined.
    If the chain has less than `n` rows, the whole chain is returned.

    Parameters:
        n (int): Number of rows to return.
    """
    return self._evolve(query=self._query.limit(n))

map 

map(
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map: Any
) -> Self

Apply a function to each row to create new signals. The function should return a new object for each row. It returns a chain itself with new signals.

Input-output relationship: 1:1

Parameters:

  • func (Optional[Callable], default: None ) –

    Function applied to each row.

  • params (Union[None, str, Sequence[str]], default: None ) –

    List of column names used as input for the function. Default is taken from function signature.

  • output (OutputType, default: None ) –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature. Default can be also taken from kwargs - **signal_map (see below). If signal name is defined using signal_map (see below) only a single type value can be used.

  • **signal_map (Any, default: {} ) –

    kwargs can be used to define func together with its return signal name in format of map(my_sign=my_func). This helps define signal names and functions in a nicer way.

Example

Using signal_map and single type in output: 

chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
chain.save("new_dataset")

Using func and output as a map: 

chain = chain.map(
    lambda name: name.split("."), output={"stem": str, "ext": str}
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
def map(
    self,
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map: Any,
) -> "Self":
    """Apply a function to each row to create new signals. The function should
    return a new object for each row. It returns a chain itself with new signals.

    Input-output relationship: 1:1

    Parameters:
        func: Function applied to each row.
        params: List of column names used as input for the function. Default
                is taken from function signature.
        output: Dictionary defining new signals and their corresponding types.
                Default type is taken from function signature. Default can be also
                taken from kwargs - **signal_map (see below).
                If signal name is defined using signal_map (see below) only a single
                type value can be used.
        **signal_map: kwargs can be used to define `func` together with its return
                signal name in format of `map(my_sign=my_func)`. This helps define
                signal names and functions in a nicer way.

    Example:
        Using signal_map and single type in output:
        ```py
        chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
        chain.save("new_dataset")
        ```

        Using func and output as a map:
        ```py
        chain = chain.map(
            lambda name: name.split("."), output={"stem": str, "ext": str}
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Mapper, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch

    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(self._settings.batch_size),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

max 

max(col: str) -> StandardType

Compute the maximum of a column.

Parameters:

  • col (str) –

    The column to compute the maximum for.

Returns:

  • StandardType –

    The maximum value in the column.

Example 
max_size = chain.max("file.size")
print(f"Maximum size: {max_size}")
Source code in datachain/lib/dc/datachain.py 
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
def max(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the maximum of a column.

    Parameters:
        col: The column to compute the maximum for.

    Returns:
        The maximum value in the column.

    Example:
        ```py
        max_size = chain.max("file.size")
        print(f"Maximum size: {max_size}")
        ```
    """
    return self._extend_to_data_model("max", col)

merge 

merge(
    right_ds: DataChain,
    on: Union[MergeColType, Sequence[MergeColType]],
    right_on: Optional[
        Union[MergeColType, Sequence[MergeColType]]
    ] = None,
    inner=False,
    full=False,
    rname="right_",
) -> Self

Merge two chains based on the specified criteria.

Parameters:

  • right_ds (DataChain) –

    Chain to join with.

  • on (Union[MergeColType, Sequence[MergeColType]]) –

    Predicate ("column.name", C("column.name"), or Func) or list of Predicates to join on. If both chains have the same predicates then this predicate is enough for the join. Otherwise, right_on parameter has to specify the predicates for the other chain.

  • right_on (Optional[Union[MergeColType, Sequence[MergeColType]]], default: None ) –

    Optional predicate or list of Predicates for the right_ds to join.

  • inner (bool, default: False ) –

    Whether to run inner join or outer join.

  • full (bool, default: False ) –

    Whether to run full outer join.

  • rname (str, default: 'right_' ) –

    Name prefix for conflicting signal names.

Examples:

meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                      right_on=(C.name, C.pq__index))

imgs.merge(captions,
           on=func.path.file_stem(imgs.c("file.path")),
           right_on=func.path.file_stem(captions.c("file.path"))

)

Source code in datachain/lib/dc/datachain.py 
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
@delta_disabled
def merge(
    self,
    right_ds: "DataChain",
    on: Union[MergeColType, Sequence[MergeColType]],
    right_on: Optional[Union[MergeColType, Sequence[MergeColType]]] = None,
    inner=False,
    full=False,
    rname="right_",
) -> "Self":
    """Merge two chains based on the specified criteria.

    Parameters:
        right_ds: Chain to join with.
        on: Predicate ("column.name", C("column.name"), or Func) or list of
            Predicates to join on. If both chains have the same predicates then
            this predicate is enough for the join. Otherwise, `right_on` parameter
            has to specify the predicates for the other chain.
        right_on: Optional predicate or list of Predicates for the `right_ds`
            to join.
        inner (bool): Whether to run inner join or outer join.
        full (bool): Whether to run full outer join.
        rname (str): Name prefix for conflicting signal names.

    Examples:
        ```py
        meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                              right_on=(C.name, C.pq__index))
        ```

        ```py
        imgs.merge(captions,
                   on=func.path.file_stem(imgs.c("file.path")),
                   right_on=func.path.file_stem(captions.c("file.path"))
        ```
    )
    """
    if on is None:
        raise DatasetMergeError(["None"], None, "'on' must be specified")

    on = _validate_merge_on(on, self)
    if not on:
        raise DatasetMergeError(
            on,
            right_on,
            (
                "'on' must be 'str', 'Func' or 'Sequence' object "
                f"but got type '{type(on)}'"
            ),
        )

    if right_on is not None:
        right_on = _validate_merge_on(right_on, right_ds)
        if not right_on:
            raise DatasetMergeError(
                on,
                right_on,
                "'right_on' must be 'str', 'Func' or 'Sequence' object"
                f" but got type '{type(right_on)}'",
            )

        if len(right_on) != len(on):
            raise DatasetMergeError(
                on, right_on, "'on' and 'right_on' must have the same length'"
            )

    if self == right_ds:
        right_ds = right_ds.clone()

    errors = []

    def _resolve(
        ds: DataChain,
        col: Union[str, Function, sqlalchemy.ColumnElement],
        side: Union[str, None],
    ):
        try:
            if isinstance(col, Function):
                return ds.c(col.get_column())
            return ds.c(col) if isinstance(col, (str, C)) else col
        except ValueError:
            if side:
                errors.append(f"{_get_merge_error_str(col)} in {side}")

    ops = [
        _resolve(self, left, "left")
        == _resolve(right_ds, right, "right" if right_on else None)
        for left, right in zip(on, right_on or on)
    ]

    if errors:
        raise DatasetMergeError(
            on, right_on, f"Could not resolve {', '.join(errors)}"
        )

    query = self._query.join(
        right_ds._query, sqlalchemy.and_(*ops), inner, full, rname + "{name}"
    )
    query.feature_schema = None
    ds = self._evolve(query=query)

    signals_schema = self.signals_schema.clone_without_sys_signals()
    right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
    ds.signals_schema = SignalSchema({"sys": Sys}) | signals_schema.merge(
        right_signals_schema, rname
    )

    return ds

min 

min(col: str) -> StandardType

Compute the minimum of a column.

Parameters:

  • col (str) –

    The column to compute the minimum for.

Returns:

  • StandardType –

    The minimum value in the column.

Example 
min_size = chain.min("file.size")
print(f"Minimum size: {min_size}")
Source code in datachain/lib/dc/datachain.py 
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
def min(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the minimum of a column.

    Parameters:
        col: The column to compute the minimum for.

    Returns:
        The minimum value in the column.

    Example:
        ```py
        min_size = chain.min("file.size")
        print(f"Minimum size: {min_size}")
        ```
    """
    return self._extend_to_data_model("min", col)

mutate 

mutate(**kwargs) -> Self

Create or modify signals based on existing signals.

This method is vectorized and more efficient compared to map(), and it does not extract or download any data from the internal database. However, it can only utilize predefined built-in functions and their combinations.

Supported functions

Numerical: +, -, *, /, rand(), avg(), count(), func(), greatest(), least(), max(), min(), sum() String: length(), split(), replace(), regexp_replace() Filename: name(), parent(), file_stem(), file_ext() Array: length(), sip_hash_64(), euclidean_distance(), cosine_distance() Window: row_number(), rank(), dense_rank(), first()

Example: 

 dc.mutate(
    area=Column("image.height") * Column("image.width"),
    extension=file_ext(Column("file.path")),
    dist=cosine_distance(embedding_text, embedding_image)
)

Window function example: 

window = func.window(partition_by="file.parent", order_by="file.size")
dc.mutate(
    row_number=func.row_number().over(window),
)

This method can be also used to rename signals. If the Column("name") provided as value for the new signal - the old signal will be dropped. Otherwise a new signal is created. Exception, if the old signal is nested one (e.g. C("file.path")), it will be kept to keep the object intact.

Example: 

 dc.mutate(
    newkey=Column("oldkey") # drops oldkey
)


 dc.mutate(
    size=Column("file.size") # keeps `file.size`
)
Source code in datachain/lib/dc/datachain.py 
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
def mutate(self, **kwargs) -> "Self":
    """Create or modify signals based on existing signals.

    This method is vectorized and more efficient compared to map(), and it does not
    extract or download any data from the internal database. However, it can only
    utilize predefined built-in functions and their combinations.

    Supported functions:
       Numerical:   +, -, *, /, rand(), avg(), count(), func(),
                    greatest(), least(), max(), min(), sum()
       String:      length(), split(), replace(), regexp_replace()
       Filename:    name(), parent(), file_stem(), file_ext()
       Array:       length(), sip_hash_64(), euclidean_distance(),
                    cosine_distance()
       Window:      row_number(), rank(), dense_rank(), first()

    Example:
    ```py
     dc.mutate(
        area=Column("image.height") * Column("image.width"),
        extension=file_ext(Column("file.path")),
        dist=cosine_distance(embedding_text, embedding_image)
    )
    ```

    Window function example:
    ```py
    window = func.window(partition_by="file.parent", order_by="file.size")
    dc.mutate(
        row_number=func.row_number().over(window),
    )
    ```

    This method can be also used to rename signals. If the Column("name") provided
    as value for the new signal - the old signal will be dropped. Otherwise a new
    signal is created. Exception, if the old signal is nested one (e.g.
    `C("file.path")`), it will be kept to keep the object intact.

    Example:
    ```py
     dc.mutate(
        newkey=Column("oldkey") # drops oldkey
    )
    ```

    ```py
     dc.mutate(
        size=Column("file.size") # keeps `file.size`
    )
    ```
    """
    from sqlalchemy.sql.sqltypes import NullType

    primitives = (bool, str, int, float)

    for col_name, expr in kwargs.items():
        if not isinstance(expr, (*primitives, Column, Func)) and isinstance(
            expr.type, NullType
        ):
            raise DataChainColumnError(
                col_name, f"Cannot infer type with expression {expr}"
            )

    mutated = {}
    schema = self.signals_schema
    for name, value in kwargs.items():
        if isinstance(value, Column):
            # renaming existing column
            for signal in schema.db_signals(name=value.name, as_columns=True):
                mutated[signal.name.replace(value.name, name, 1)] = signal  # type: ignore[union-attr]
        elif isinstance(value, Func):
            # adding new signal
            mutated[name] = value.get_column(schema)
        elif isinstance(value, primitives):
            # adding simple python constant primitives like str, int, float, bool
            val = literal(value)
            val.type = python_to_sql(type(value))()
            mutated[name] = val  # type: ignore[assignment]
        else:
            # adding new signal
            mutated[name] = value

    new_schema = schema.mutate(kwargs)
    return self._evolve(
        query=self._query.mutate(new_schema=new_schema, **mutated),
        signal_schema=new_schema,
    )

offset 

offset(offset: int) -> Self

Return the results starting with the offset row.

If the chain is unordered, which rows are skipped in undefined. If the chain has less than offset rows, the result is an empty chain.

Parameters:

  • offset (int) –

    Number of rows to skip.

Source code in datachain/lib/dc/datachain.py 
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
def offset(self, offset: int) -> "Self":
    """Return the results starting with the offset row.

    If the chain is unordered, which rows are skipped in undefined.
    If the chain has less than `offset` rows, the result is an empty chain.

    Parameters:
        offset (int): Number of rows to skip.
    """
    return self._evolve(query=self._query.offset(offset))

order_by 

order_by(*args, descending: bool = False) -> Self

Orders by specified set of columns.

Parameters:

  • descending (bool, default: False ) –

    Whether to sort in descending order or not.

Example 
dc.order_by("similarity_score", descending=True).limit(10)
Note

Order is not guaranteed when steps are added after an order_by statement. I.e. when using read_dataset an order_by statement should be used if the order of the records in the chain is important. Using order_by directly before limit, to_list and similar methods will give expected results. See https://github.com/iterative/datachain/issues/477 for further details.

Source code in datachain/lib/dc/datachain.py 
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
@resolve_columns
def order_by(self, *args, descending: bool = False) -> "Self":
    """Orders by specified set of columns.

    Parameters:
        descending (bool): Whether to sort in descending order or not.

    Example:
        ```py
        dc.order_by("similarity_score", descending=True).limit(10)
        ```

    Note:
        Order is not guaranteed when steps are added after an `order_by` statement.
        I.e. when using `read_dataset` an `order_by` statement should be used if
        the order of the records in the chain is important.
        Using `order_by` directly before `limit`, `to_list` and similar methods
        will give expected results.
        See https://github.com/iterative/datachain/issues/477 for further details.
    """
    if descending:
        args = tuple(sqlalchemy.desc(a) for a in args)

    return self._evolve(query=self._query.order_by(*args))

parse_tabular 

parse_tabular(
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    **kwargs: Any
) -> Self

Generate chain from list of tabular files.

Parameters:

  • output (OutputType, default: None ) –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • column (str, default: '' ) –

    Generated column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • source (bool, default: True ) –

    Whether to include info about the source file.

  • nrows (Optional[int], default: None ) –

    Optional row limit.

  • kwargs (Any, default: {} ) –

    Parameters to pass to pyarrow.dataset.dataset.

Example

Reading a json lines file: 

import datachain as dc
chain = dc.read_storage("s3://mybucket/file.jsonl")
chain = chain.parse_tabular(format="json")

Reading a filtered list of files as a dataset: 

import datachain as dc

chain = dc.read_storage("s3://mybucket")
chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
chain = chain.parse_tabular(format="json")

Source code in datachain/lib/dc/datachain.py 
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
def parse_tabular(
    self,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    **kwargs: Any,
) -> "Self":
    """Generate chain from list of tabular files.

    Parameters:
        output: Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        column: Generated column name.
        model_name: Generated model name.
        source: Whether to include info about the source file.
        nrows: Optional row limit.
        kwargs: Parameters to pass to pyarrow.dataset.dataset.

    Example:
        Reading a json lines file:
        ```py
        import datachain as dc
        chain = dc.read_storage("s3://mybucket/file.jsonl")
        chain = chain.parse_tabular(format="json")
        ```

        Reading a filtered list of files as a dataset:
        ```py
        import datachain as dc

        chain = dc.read_storage("s3://mybucket")
        chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
        chain = chain.parse_tabular(format="json")
        ```
    """
    from pyarrow.dataset import CsvFileFormat, JsonFileFormat

    from datachain.lib.arrow import (
        ArrowGenerator,
        fix_pyarrow_format,
        infer_schema,
        schema_to_output,
    )

    parse_options = kwargs.pop("parse_options", None)
    if format := kwargs.get("format"):
        kwargs["format"] = fix_pyarrow_format(format, parse_options)

    if (
        nrows
        and format not in ["csv", "json"]
        and not isinstance(format, (CsvFileFormat, JsonFileFormat))
    ):
        raise DatasetPrepareError(
            self.name,
            "error in `parse_tabular` - "
            "`nrows` only supported for csv and json formats.",
        )

    if "file" not in self.schema or not self.count():
        raise DatasetPrepareError(self.name, "no files to parse.")

    schema = None
    col_names = output if isinstance(output, Sequence) else None
    if col_names or not output:
        try:
            schema = infer_schema(self, **kwargs, parse_options=parse_options)
            output, _ = schema_to_output(schema, col_names)
        except ValueError as e:
            raise DatasetPrepareError(self.name, e) from e

    if isinstance(output, dict):
        model_name = model_name or column or ""
        model = dict_to_data_model(model_name, output)
        output = model
    else:
        model = output  # type: ignore[assignment]

    if column:
        output = {column: model}  # type: ignore[dict-item]
    elif isinstance(output, type(BaseModel)):
        output = {
            name: info.annotation  # type: ignore[misc]
            for name, info in output.model_fields.items()
        }

    if source:
        output = {"source": ArrowRow} | output  # type: ignore[assignment,operator]

    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}
    return self.settings(**settings).gen(  # type: ignore[arg-type]
        ArrowGenerator(
            schema,
            model,
            source,
            nrows,
            parse_options=parse_options,
            **kwargs,
        ),
        output=output,
    )

persist 

persist() -> Self

Saves temporary chain that will be removed after the process ends. Temporary datasets are useful for optimization, for example when we have multiple chains starting with identical sub-chain. We can then persist that common chain and use it to calculate other chains, to avoid re-calculation every time. It returns the chain itself.

Source code in datachain/lib/dc/datachain.py 
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
def persist(self) -> "Self":
    """Saves temporary chain that will be removed after the process ends.
    Temporary datasets are useful for optimization, for example when we have
    multiple chains starting with identical sub-chain. We can then persist that
    common chain and use it to calculate other chains, to avoid re-calculation
    every time.
    It returns the chain itself.
    """
    schema = self.signals_schema.clone_without_sys_signals().serialize()
    project = self.session.catalog.metastore.get_project(
        self.project_name,
        self.namespace_name,
        create=True,
    )
    return self._evolve(
        query=self._query.save(project=project, feature_schema=schema)
    )

print_schema 

print_schema(file: Optional[IO] = None) -> None

Print schema of the chain.

Source code in datachain/lib/dc/datachain.py 
301
302
303
def print_schema(self, file: Optional[IO] = None) -> None:
    """Print schema of the chain."""
    self._effective_signals_schema.print_tree(file=file)

reset_settings 

reset_settings(settings: Optional[Settings] = None) -> Self

Reset all chain settings to default values.

Source code in datachain/lib/dc/datachain.py 
400
401
402
403
def reset_settings(self, settings: Optional[Settings] = None) -> "Self":
    """Reset all chain settings to default values."""
    self._settings = settings if settings else Settings()
    return self

sample 

sample(n: int) -> Self

Return a random sample from the chain.

Parameters:

  • n (int) –

    Number of samples to draw.

Note

Samples are not deterministic, and streamed/paginated queries or multiple workers will draw samples with replacement.

Source code in datachain/lib/dc/datachain.py 
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
def sample(self, n: int) -> "Self":
    """Return a random sample from the chain.

    Parameters:
        n: Number of samples to draw.

    Note:
        Samples are not deterministic, and streamed/paginated queries or
        multiple workers will draw samples with replacement.
    """
    return self._evolve(query=self._query.sample(n))

save 

save(
    name: str,
    version: Optional[str] = None,
    description: Optional[str] = None,
    attrs: Optional[list[str]] = None,
    update_version: Optional[str] = "patch",
    **kwargs
) -> DataChain

Save to a Dataset. It returns the chain itself.

Parameters:

  • name (str) –

    dataset name. This can be either a fully qualified name, including the namespace and project, or just a regular dataset name. In the latter case, the namespace and project will be taken from the settings (if specified) or from the default values otherwise.

  • version (Optional[str], default: None ) –

    version of a dataset. If version is not specified and dataset already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.

  • description (Optional[str], default: None ) –

    description of a dataset.

  • attrs (Optional[list[str]], default: None ) –

    attributes of a dataset. They can be without value, e.g "NLP", or with a value, e.g "location=US".

  • update_version (Optional[str], default: 'patch' ) –

    which part of the dataset version to automatically increase. Available values: major, minor or patch. Default is patch.

Source code in datachain/lib/dc/datachain.py 
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
def save(  # type: ignore[override]
    self,
    name: str,
    version: Optional[str] = None,
    description: Optional[str] = None,
    attrs: Optional[list[str]] = None,
    update_version: Optional[str] = "patch",
    **kwargs,
) -> "DataChain":
    """Save to a Dataset. It returns the chain itself.

    Parameters:
        name: dataset name. This can be either a fully qualified name, including
            the namespace and project, or just a regular dataset name. In the latter
            case, the namespace and project will be taken from the settings
            (if specified) or from the default values otherwise.
        version: version of a dataset. If version is not specified and dataset
            already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.
        description: description of a dataset.
        attrs: attributes of a dataset. They can be without value, e.g "NLP",
            or with a value, e.g "location=US".
        update_version: which part of the dataset version to automatically increase.
            Available values: `major`, `minor` or `patch`. Default is `patch`.
    """
    catalog = self.session.catalog
    if version is not None:
        semver.validate(version)

    if update_version is not None and update_version not in [
        "patch",
        "major",
        "minor",
    ]:
        raise ValueError(
            "update_version can have one of the following values: major, minor or"
            " patch"
        )

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        namespace_name=self._settings.namespace,
        project_name=self._settings.project,
    )

    try:
        project = self.session.catalog.metastore.get_project(
            project_name,
            namespace_name,
            create=is_studio(),
        )
    except ProjectNotFoundError as e:
        # not being able to create it as creation is not allowed
        raise ProjectCreateNotAllowedError("Creating project is not allowed") from e

    schema = self.signals_schema.clone_without_sys_signals().serialize()

    # Handle retry and delta functionality
    if self.delta and name:
        from datachain.delta import delta_retry_update

        # Delta chains must have delta_on defined (ensured by _as_delta method)
        assert self._delta_on is not None, "Delta chain must have delta_on defined"

        result_ds, dependencies, has_changes = delta_retry_update(
            self,
            namespace_name,
            project_name,
            name,
            on=self._delta_on,
            right_on=self._delta_result_on,
            compare=self._delta_compare,
            delta_retry=self._delta_retry,
        )

        if result_ds:
            return self._evolve(
                query=result_ds._query.save(
                    name=name,
                    version=version,
                    project=project,
                    feature_schema=schema,
                    dependencies=dependencies,
                    **kwargs,
                )
            )

        if not has_changes:
            # sources have not been changed so new version of resulting dataset
            # would be the same as previous one. To avoid duplicating exact
            # datasets, we won't create new version of it and we will return
            # current latest version instead.
            from .datasets import read_dataset

            return read_dataset(
                name, namespace=namespace_name, project=project_name, **kwargs
            )

    return self._evolve(
        query=self._query.save(
            name=name,
            version=version,
            project=project,
            description=description,
            attrs=attrs,
            feature_schema=schema,
            update_version=update_version,
            **kwargs,
        )
    )

select 

select(*args: str, _sys: bool = True) -> Self

Select only a specified set of signals.

Source code in datachain/lib/dc/datachain.py 
1053
1054
1055
1056
1057
1058
1059
1060
1061
def select(self, *args: str, _sys: bool = True) -> "Self":
    """Select only a specified set of signals."""
    new_schema = self.signals_schema.resolve(*args)
    if self._sys and _sys:
        new_schema = SignalSchema({"sys": Sys}) | new_schema
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

select_except 

select_except(*args: str) -> Self

Select all the signals expect the specified signals.

Source code in datachain/lib/dc/datachain.py 
1063
1064
1065
1066
1067
1068
1069
def select_except(self, *args: str) -> "Self":
    """Select all the signals expect the specified signals."""
    new_schema = self.signals_schema.select_except_signals(*args)
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

settings 

settings(
    cache: Optional[bool] = None,
    prefetch: Optional[Union[bool, int]] = None,
    parallel: Optional[Union[bool, int]] = None,
    workers: Optional[int] = None,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    min_task_size: Optional[int] = None,
    batch_size: Optional[int] = None,
    sys: Optional[bool] = None,
) -> Self

Set chain execution parameters. Returns the chain itself, allowing method chaining for subsequent operations. To restore all settings to their default values, use reset_settings().

Parameters:

  • cache (Optional[bool], default: None ) –

    Enable files caching to speed up subsequent accesses to the same files from the same or different chains. Defaults to False.

  • prefetch (Optional[Union[bool, int]], default: None ) –

    Enable prefetching of files. This will download files in advance in parallel. If an integer is provided, it specifies the number of files to prefetch concurrently for each process on each worker. Defaults to 2. Set to 0 or False to disable prefetching.

  • parallel (Optional[Union[bool, int]], default: None ) –

    Number of processes to use for processing user-defined functions (UDFs) in parallel. If an integer is provided, it specifies the number of CPUs to use. If True, all available CPUs are used. Defaults to 1.

  • namespace (Optional[str], default: None ) –

    Namespace to use for the chain by default.

  • project (Optional[str], default: None ) –

    Project to use for the chain by default.

  • min_task_size (Optional[int], default: None ) –

    Minimum number of rows per worker/process for parallel processing by UDFs. Defaults to 1.

  • batch_size (Optional[int], default: None ) –

    Number of rows per insert by UDF to fine tune and balance speed and memory usage. This might be useful when processing large rows or when running into memory issues. Defaults to 2000.

Example 
chain = (
    chain
    .settings(cache=True, parallel=8, batch_size=300)
    .map(laion=process_webdataset(spec=WDSLaion), params="file")
)
Source code in datachain/lib/dc/datachain.py 
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
def settings(
    self,
    cache: Optional[bool] = None,
    prefetch: Optional[Union[bool, int]] = None,
    parallel: Optional[Union[bool, int]] = None,
    workers: Optional[int] = None,
    namespace: Optional[str] = None,
    project: Optional[str] = None,
    min_task_size: Optional[int] = None,
    batch_size: Optional[int] = None,
    sys: Optional[bool] = None,
) -> "Self":
    """
    Set chain execution parameters. Returns the chain itself, allowing method
    chaining for subsequent operations. To restore all settings to their default
    values, use `reset_settings()`.

    Parameters:
        cache: Enable files caching to speed up subsequent accesses to the same
            files from the same or different chains. Defaults to False.
        prefetch: Enable prefetching of files. This will download files in
            advance in parallel. If an integer is provided, it specifies the number
            of files to prefetch concurrently for each process on each worker.
            Defaults to 2. Set to 0 or False to disable prefetching.
        parallel: Number of processes to use for processing user-defined functions
            (UDFs) in parallel. If an integer is provided, it specifies the number
            of CPUs to use. If True, all available CPUs are used. Defaults to 1.
        namespace: Namespace to use for the chain by default.
        project: Project to use for the chain by default.
        min_task_size: Minimum number of rows per worker/process for parallel
            processing by UDFs. Defaults to 1.
        batch_size: Number of rows per insert by UDF to fine tune and balance speed
            and memory usage. This might be useful when processing large rows
            or when running into memory issues. Defaults to 2000.

    Example:
        ```py
        chain = (
            chain
            .settings(cache=True, parallel=8, batch_size=300)
            .map(laion=process_webdataset(spec=WDSLaion), params="file")
        )
        ```
    """
    if sys is None:
        sys = self._sys
    settings = copy.copy(self._settings)
    settings.add(
        Settings(
            cache=cache,
            prefetch=prefetch,
            parallel=parallel,
            workers=workers,
            namespace=namespace,
            project=project,
            min_task_size=min_task_size,
            batch_size=batch_size,
        )
    )
    return self._evolve(settings=settings, _sys=sys)

setup 

setup(**kwargs) -> Self

Setup variables to pass to UDF functions.

Use before running map/gen/agg to save an object and pass it as an argument to the UDF.

The value must be a callable (a lambda: <value> syntax can be used to quickly create one) that returns the object to be passed to the UDF. It is evaluated lazily when UDF is running, in case of multiple machines the callable is run on a worker machine.

Example 
import anthropic
from anthropic.types import Message
import datachain as dc

(
    dc.read_storage(DATA, type="text")
    .settings(parallel=4, cache=True)

    # Setup Anthropic client and pass it to the UDF below automatically
    # The value is callable (see the note above)
    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))

    .map(
        claude=lambda client, file: client.messages.create(
            model=MODEL,
            system=PROMPT,
            messages=[{"role": "user", "content": file.get_value()}],
        ),
        output=Message,
    )
)
Source code in datachain/lib/dc/datachain.py 
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
def setup(self, **kwargs) -> "Self":
    """Setup variables to pass to UDF functions.

    Use before running map/gen/agg to save an object and pass it as an
    argument to the UDF.

    The value must be a callable (a `lambda: <value>` syntax can be used to quickly
    create one) that returns the object to be passed to the UDF. It is evaluated
    lazily when UDF is running, in case of multiple machines the callable is run on
    a worker machine.

    Example:
        ```py
        import anthropic
        from anthropic.types import Message
        import datachain as dc

        (
            dc.read_storage(DATA, type="text")
            .settings(parallel=4, cache=True)

            # Setup Anthropic client and pass it to the UDF below automatically
            # The value is callable (see the note above)
            .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))

            .map(
                claude=lambda client, file: client.messages.create(
                    model=MODEL,
                    system=PROMPT,
                    messages=[{"role": "user", "content": file.get_value()}],
                ),
                output=Message,
            )
        )
        ```
    """
    intersection = set(self._setup.keys()) & set(kwargs.keys())
    if intersection:
        keys = ", ".join(intersection)
        raise DatasetPrepareError(self.name, f"this value(s) already setup: {keys}")

    self._setup = self._setup | kwargs
    return self

show 

show(
    limit: int = 20,
    flatten: bool = False,
    transpose: bool = False,
    truncate: bool = True,
    include_hidden: bool = False,
) -> None

Show a preview of the chain results.

Parameters:

  • limit (int, default: 20 ) –

    How many rows to show.

  • flatten (bool, default: False ) –

    Whether to use a multiindex or flatten column names.

  • transpose (bool, default: False ) –

    Whether to transpose rows and columns.

  • truncate (bool, default: True ) –

    Whether or not to truncate the contents of columns.

  • include_hidden (bool, default: False ) –

    Whether to include hidden columns.

Source code in datachain/lib/dc/datachain.py 
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
def show(
    self,
    limit: int = 20,
    flatten: bool = False,
    transpose: bool = False,
    truncate: bool = True,
    include_hidden: bool = False,
) -> None:
    """Show a preview of the chain results.

    Parameters:
        limit: How many rows to show.
        flatten: Whether to use a multiindex or flatten column names.
        transpose: Whether to transpose rows and columns.
        truncate: Whether or not to truncate the contents of columns.
        include_hidden: Whether to include hidden columns.
    """
    import pandas as pd

    dc = self.limit(limit) if limit > 0 else self  # type: ignore[misc]
    df = dc.to_pandas(flatten, include_hidden=include_hidden)

    if df.empty:
        print("Empty result")
        print(f"Columns: {list(df.columns)}")
        return

    if transpose:
        df = df.T

    options: list = [
        "display.max_columns",
        None,
        "display.multi_sparse",
        False,
    ]

    try:
        if columns := os.get_terminal_size().columns:
            options.extend(["display.width", columns])
    except OSError:
        pass

    if not truncate:
        options.extend(["display.max_colwidth", None])

    with pd.option_context(*options):
        if inside_notebook():
            from IPython.display import display

            display(df)
        else:
            print(df)

    if len(df) == limit:
        print(f"\n[Limited by {len(df)} rows]")

shuffle 

shuffle() -> Self

Shuffle the rows of the chain deterministically.

Source code in datachain/lib/dc/datachain.py 
2626
2627
2628
def shuffle(self) -> "Self":
    """Shuffle the rows of the chain deterministically."""
    return self.order_by("sys.rand")

subtract 

subtract(
    other: DataChain,
    on: Optional[Union[str, Sequence[str]]] = None,
    right_on: Optional[Union[str, Sequence[str]]] = None,
) -> Self

Remove rows that appear in another chain.

Parameters:

  • other (DataChain) –

    chain whose rows will be removed from self

  • on (Optional[Union[str, Sequence[str]]], default: None ) –

    columns to consider for determining row equality in self. If unspecified, defaults to all common columns between self and other.

  • right_on (Optional[Union[str, Sequence[str]]], default: None ) –

    columns to consider for determining row equality in other. If unspecified, defaults to the same values as on.

Source code in datachain/lib/dc/datachain.py 
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
def subtract(  # type: ignore[override]
    self,
    other: "DataChain",
    on: Optional[Union[str, Sequence[str]]] = None,
    right_on: Optional[Union[str, Sequence[str]]] = None,
) -> "Self":
    """Remove rows that appear in another chain.

    Parameters:
        other: chain whose rows will be removed from `self`
        on: columns to consider for determining row equality in `self`.
            If unspecified, defaults to all common columns
            between `self` and `other`.
        right_on: columns to consider for determining row equality in `other`.
            If unspecified, defaults to the same values as `on`.
    """
    if isinstance(on, str):
        if not on:
            raise DataChainParamsError("'on' cannot be an empty string")
        on = [on]
    elif isinstance(on, Sequence):
        if not on or any(not col for col in on):
            raise DataChainParamsError("'on' cannot contain empty strings")

    if isinstance(right_on, str):
        if not right_on:
            raise DataChainParamsError("'right_on' cannot be an empty string")
        right_on = [right_on]
    elif isinstance(right_on, Sequence):
        if not right_on or any(not col for col in right_on):
            raise DataChainParamsError("'right_on' cannot contain empty strings")

    if on is None and right_on is None:
        other_columns = set(other._effective_signals_schema.db_signals())
        signals = [
            c
            for c in self._effective_signals_schema.db_signals()
            if c in other_columns
        ]
        if not signals:
            raise DataChainParamsError("subtract(): no common columns")
    elif on is not None and right_on is None:
        right_on = on
        signals = list(self.signals_schema.resolve(*on).db_signals())
    elif on is None and right_on is not None:
        raise DataChainParamsError(
            "'on' must be specified when 'right_on' is provided"
        )
    else:
        if not isinstance(on, Sequence) or not isinstance(right_on, Sequence):
            raise TypeError(
                "'on' and 'right_on' must be 'str' or 'Sequence' object"
            )
        if len(on) != len(right_on):
            raise DataChainParamsError(
                "'on' and 'right_on' must have the same length"
            )
        signals = list(
            zip(
                self.signals_schema.resolve(*on).db_signals(),
                other.signals_schema.resolve(*right_on).db_signals(),
            )  # type: ignore[arg-type]
        )
    return self._evolve(query=self._query.subtract(other._query, signals))  # type: ignore[arg-type]

sum 

sum(col: str) -> StandardType

Compute the sum of a column.

Parameters:

  • col (str) –

    The column to compute the sum for.

Returns:

  • StandardType –

    The sum of the column values.

Example 
total_size = chain.sum("file.size")
print(f"Total size: {total_size}")
Source code in datachain/lib/dc/datachain.py 
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
def sum(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the sum of a column.

    Parameters:
        col: The column to compute the sum for.

    Returns:
        The sum of the column values.

    Example:
        ```py
        total_size = chain.sum("file.size")
        print(f"Total size: {total_size}")
        ```
    """
    return self._extend_to_data_model("sum", col)

to_columnar_data_with_names 

to_columnar_data_with_names(
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
) -> tuple[list[str], Iterator[list[list[Any]]]]

Returns column names and the results as an iterator that provides chunks, with each chunk containing a list of columns, where each column contains a list of the row values for that column in that chunk. Useful for columnar data formats, such as parquet or other OLAP databases.

Source code in datachain/lib/dc/datachain.py 
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
def to_columnar_data_with_names(
    self, chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE
) -> tuple[list[str], Iterator[list[list[Any]]]]:
    """Returns column names and the results as an iterator that provides chunks,
    with each chunk containing a list of columns, where each column contains a
    list of the row values for that column in that chunk. Useful for columnar data
    formats, such as parquet or other OLAP databases.
    """
    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self._leaf_values()

    def column_chunks() -> Iterator[list[list[Any]]]:
        for chunk_iter in batched_it(results_iter, chunk_size):
            columns: list[list[Any]] = [[] for _ in column_names]
            for row in chunk_iter:
                for i, col in enumerate(columns):
                    col.append(row[i])
            yield columns

    return column_names, column_chunks()

to_csv 

to_csv(
    path: Union[str, PathLike[str]],
    delimiter: str = ",",
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs
) -> None

Save chain to a csv (comma-separated values) file.

Parameters:

  • path (Union[str, PathLike[str]]) –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • delimiter (str, default: ',' ) –

    Delimiter to use for the resulting file.

  • fs_kwargs (Optional[dict[str, Any]], default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
def to_csv(
    self,
    path: Union[str, os.PathLike[str]],
    delimiter: str = ",",
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs,
) -> None:
    """Save chain to a csv (comma-separated values) file.

    Parameters:
        path: Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        delimiter: Delimiter to use for the resulting file.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import csv

    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self._leaf_values()

    with opener(path, "w", newline="") as f:
        writer = csv.writer(f, delimiter=delimiter, **kwargs)
        writer.writerow(column_names)

        for row in results_iter:
            writer.writerow(row)

to_database 

to_database(
    table_name: str,
    connection: ConnectionType,
    *,
    batch_size: int = DEFAULT_DATABASE_BATCH_SIZE,
    on_conflict: Optional[str] = None,
    conflict_columns: Optional[list[str]] = None,
    column_mapping: Optional[
        dict[str, Optional[str]]
    ] = None
) -> int

Save chain to a database table using a given database connection.

This method exports all DataChain records to a database table, creating the table if it doesn't exist and appending data if it does. The table schema is automatically inferred from the DataChain's signal schema.

For PostgreSQL, tables are created in the schema specified by the connection's search_path (defaults to 'public'). Use URL parameters to target specific schemas.

Parameters:

  • table_name (str) –

    Name of the database table to create/write to.

  • connection (ConnectionType) –

    SQLAlchemy connectable, str, or a sqlite3 connection Using SQLAlchemy makes it possible to use any DB supported by that library. If a DBAPI2 object, only sqlite3 is supported. The user is responsible for engine disposal and connection closure for the SQLAlchemy connectable; str connections are closed automatically.

  • batch_size (int, default: DEFAULT_DATABASE_BATCH_SIZE ) –

    Number of rows to insert per batch for optimal performance. Larger batches are faster but use more memory. Default: 10,000.

  • on_conflict (Optional[str], default: None ) –

    Strategy for handling duplicate rows (requires table constraints): - None: Raise error (sqlalchemy.exc.IntegrityError) on conflict (default) - "ignore": Skip duplicate rows silently - "update": Update existing rows with new values

  • conflict_columns (Optional[list[str]], default: None ) –

    List of column names that form a unique constraint for conflict resolution. Required when on_conflict='update' and using PostgreSQL.

  • column_mapping (Optional[dict[str, Optional[str]]], default: None ) –

    Optional mapping to rename or skip columns: - Dict mapping DataChain column names to database column names - Set values to None to skip columns entirely, or use defaultdict to skip all columns except those specified.

Returns:

  • int ( int ) –

    Number of rows affected (inserted/updated). -1 if DB driver doesn't support telemetry.

Examples:

Basic usage with PostgreSQL: 

import datachain as dc

rows_affected = (dc
  .read_storage("s3://my-bucket/")
  .to_database("files_table", "postgresql://user:pass@localhost/mydb")
)
print(f"Inserted/updated {rows_affected} rows")

Using SQLite with connection string: 

rows_affected = chain.to_database("my_table", "sqlite:///data.db")
print(f"Affected {rows_affected} rows")

Column mapping and renaming: 

mapping = {
    "user.id": "id",
    "user.name": "name",
    "user.password": None  # Skip this column
}
chain.to_database("users", engine, column_mapping=mapping)

Handling conflicts (requires PRIMARY KEY or UNIQUE constraints): 

# Skip duplicates
chain.to_database("my_table", engine, on_conflict="ignore")

# Update existing records
chain.to_database(
   "my_table", engine, on_conflict="update", conflict_columns=["id"]
)

Working with different databases: 

# MySQL
mysql_engine = sa.create_engine("mysql+pymysql://user:pass@host/db")
chain.to_database("mysql_table", mysql_engine)

# SQLite in-memory
chain.to_database("temp_table", "sqlite:///:memory:")

PostgreSQL with schema support: 

pg_url = "postgresql://user:pass@host/db?options=-c search_path=analytics"
chain.to_database("processed_data", pg_url)

Source code in datachain/lib/dc/datachain.py 
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
def to_database(
    self,
    table_name: str,
    connection: "ConnectionType",
    *,
    batch_size: int = DEFAULT_DATABASE_BATCH_SIZE,
    on_conflict: Optional[str] = None,
    conflict_columns: Optional[list[str]] = None,
    column_mapping: Optional[dict[str, Optional[str]]] = None,
) -> int:
    """Save chain to a database table using a given database connection.

    This method exports all DataChain records to a database table, creating the
    table if it doesn't exist and appending data if it does. The table schema
    is automatically inferred from the DataChain's signal schema.

    For PostgreSQL, tables are created in the schema specified by the connection's
    search_path (defaults to 'public'). Use URL parameters to target specific
    schemas.

    Parameters:
        table_name: Name of the database table to create/write to.
        connection: SQLAlchemy connectable, str, or a sqlite3 connection
            Using SQLAlchemy makes it possible to use any DB supported by that
            library. If a DBAPI2 object, only sqlite3 is supported. The user is
            responsible for engine disposal and connection closure for the
            SQLAlchemy connectable; str connections are closed automatically.
        batch_size: Number of rows to insert per batch for optimal performance.
            Larger batches are faster but use more memory. Default: 10,000.
        on_conflict: Strategy for handling duplicate rows (requires table
            constraints):
            - None: Raise error (`sqlalchemy.exc.IntegrityError`) on conflict
              (default)
            - "ignore": Skip duplicate rows silently
            - "update": Update existing rows with new values
        conflict_columns: List of column names that form a unique constraint
            for conflict resolution. Required when on_conflict='update' and
            using PostgreSQL.
        column_mapping: Optional mapping to rename or skip columns:
            - Dict mapping DataChain column names to database column names
            - Set values to None to skip columns entirely, or use `defaultdict` to
              skip all columns except those specified.

    Returns:
        int: Number of rows affected (inserted/updated). -1 if DB driver doesn't
             support telemetry.

    Examples:
        Basic usage with PostgreSQL:
        ```py
        import datachain as dc

        rows_affected = (dc
          .read_storage("s3://my-bucket/")
          .to_database("files_table", "postgresql://user:pass@localhost/mydb")
        )
        print(f"Inserted/updated {rows_affected} rows")
        ```

        Using SQLite with connection string:
        ```py
        rows_affected = chain.to_database("my_table", "sqlite:///data.db")
        print(f"Affected {rows_affected} rows")
        ```

        Column mapping and renaming:
        ```py
        mapping = {
            "user.id": "id",
            "user.name": "name",
            "user.password": None  # Skip this column
        }
        chain.to_database("users", engine, column_mapping=mapping)
        ```

        Handling conflicts (requires PRIMARY KEY or UNIQUE constraints):
        ```py
        # Skip duplicates
        chain.to_database("my_table", engine, on_conflict="ignore")

        # Update existing records
        chain.to_database(
           "my_table", engine, on_conflict="update", conflict_columns=["id"]
        )
        ```

        Working with different databases:
        ```py
        # MySQL
        mysql_engine = sa.create_engine("mysql+pymysql://user:pass@host/db")
        chain.to_database("mysql_table", mysql_engine)

        # SQLite in-memory
        chain.to_database("temp_table", "sqlite:///:memory:")
        ```

        PostgreSQL with schema support:
        ```py
        pg_url = "postgresql://user:pass@host/db?options=-c search_path=analytics"
        chain.to_database("processed_data", pg_url)
        ```
    """
    from .database import to_database

    return to_database(
        self,
        table_name,
        connection,
        batch_size=batch_size,
        on_conflict=on_conflict,
        conflict_columns=conflict_columns,
        column_mapping=column_mapping,
    )

to_iter 

to_iter(*cols: str) -> Iterator[tuple[DataValue, ...]]

Yields rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Yields:

Example

Iterating over all rows: 

for row in ds.to_iter():
    print(row)

DataChain is iterable and can be used in a for loop directly which is equivalent to ds.to_iter(): 

for row in ds:
    print(row)

Iterating over all rows with selected columns: 

for name, size in ds.to_iter("file.path", "file.size"):
    print(name, size)

Iterating over a single column: 

for (file,) in ds.to_iter("file.path"):
    print(file)

Source code in datachain/lib/dc/datachain.py 
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
def to_iter(self, *cols: str) -> Iterator[tuple[DataValue, ...]]:
    """Yields rows of values, optionally limited to the specified columns.

    Args:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Yields:
        (tuple[DataType, ...]): Yields a tuple of items for each row.

    Example:
        Iterating over all rows:
        ```py
        for row in ds.to_iter():
            print(row)
        ```

        DataChain is iterable and can be used in a for loop directly which is
        equivalent to `ds.to_iter()`:
        ```py
        for row in ds:
            print(row)
        ```

        Iterating over all rows with selected columns:
        ```py
        for name, size in ds.to_iter("file.path", "file.size"):
            print(name, size)
        ```

        Iterating over a single column:
        ```py
        for (file,) in ds.to_iter("file.path"):
            print(file)
        ```
    """
    chain = self.select(*cols) if cols else self
    signals_schema = chain._effective_signals_schema
    db_signals = signals_schema.db_signals()
    with self._query.ordered_select(*db_signals).as_iterable() as rows:
        for row in rows:
            ret = signals_schema.row_to_features(
                row, catalog=chain.session.catalog, cache=chain._settings.cache
            )
            yield tuple(ret)

to_json 

to_json(
    path: Union[str, PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
    include_outer_list: bool = True,
) -> None

Save chain to a JSON file.

Parameters:

  • path (Union[str, PathLike[str]]) –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs (Optional[dict[str, Any]], default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

  • include_outer_list (bool, default: True ) –

    Sets whether to include an outer list for all rows. Setting this to True makes the file valid JSON, while False instead writes in the JSON lines format.

Source code in datachain/lib/dc/datachain.py 
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
def to_json(
    self,
    path: Union[str, os.PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
    include_outer_list: bool = True,
) -> None:
    """Save chain to a JSON file.

    Parameters:
        path: Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
        include_outer_list: Sets whether to include an outer list for all rows.
            Setting this to True makes the file valid JSON, while False instead
            writes in the JSON lines format.
    """
    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    headers = [list(filter(None, header)) for header in headers]

    is_first = True

    with opener(path, "wb") as f:
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"[\n")
        for row in self._leaf_values():
            if not is_first:
                if include_outer_list:
                    # This makes the file JSON instead of JSON lines.
                    f.write(b",\n")
                else:
                    f.write(b"\n")
            else:
                is_first = False
            f.write(
                json.dumps(
                    row_to_nested_dict(headers, row), ensure_ascii=False
                ).encode("utf-8")
            )
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"\n]\n")

to_jsonl 

to_jsonl(
    path: Union[str, PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
) -> None

Save chain to a JSON lines file.

Parameters:

  • path (Union[str, PathLike[str]]) –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs (Optional[dict[str, Any]], default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
def to_jsonl(
    self,
    path: Union[str, os.PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
) -> None:
    """Save chain to a JSON lines file.

    Parameters:
        path: Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    self.to_json(path, fs_kwargs, include_outer_list=False)

to_list 

to_list(*cols: str) -> list[tuple[DataValue, ...]]

Returns a list of rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Returns:

  • list[tuple[DataValue, ...]] –

    list[tuple[DataType, ...]]: Returns a list of tuples of items for each row.

Example

Getting all rows as a list: 

rows = dc.to_list()
print(rows)

Getting all rows with selected columns as a list: 

name_size_pairs = dc.to_list("file.path", "file.size")
print(name_size_pairs)

Getting a single column as a list: 

files = dc.to_list("file.path")
print(files)  # Returns list of 1-tuples

Source code in datachain/lib/dc/datachain.py 
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
def to_list(self, *cols: str) -> list[tuple[DataValue, ...]]:
    """Returns a list of rows of values, optionally limited to the specified
    columns.

    Parameters:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Returns:
        list[tuple[DataType, ...]]: Returns a list of tuples of items for each row.

    Example:
        Getting all rows as a list:
        ```py
        rows = dc.to_list()
        print(rows)
        ```

        Getting all rows with selected columns as a list:
        ```py
        name_size_pairs = dc.to_list("file.path", "file.size")
        print(name_size_pairs)
        ```

        Getting a single column as a list:
        ```py
        files = dc.to_list("file.path")
        print(files)  # Returns list of 1-tuples
        ```
    """
    return list(self.to_iter(*cols))

to_pandas 

to_pandas(
    flatten: bool = False, include_hidden: bool = True
) -> DataFrame

Return a pandas DataFrame from the chain.

Parameters:

  • flatten (bool, default: False ) –

    Whether to use a multiindex or flatten column names.

  • include_hidden (bool, default: True ) –

    Whether to include hidden columns.

Returns:

  • DataFrame –

    pd.DataFrame: A pandas DataFrame representation of the chain.

Source code in datachain/lib/dc/datachain.py 
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
def to_pandas(
    self,
    flatten: bool = False,
    include_hidden: bool = True,
) -> "pd.DataFrame":
    """Return a pandas DataFrame from the chain.

    Parameters:
        flatten: Whether to use a multiindex or flatten column names.
        include_hidden: Whether to include hidden columns.

    Returns:
        pd.DataFrame: A pandas DataFrame representation of the chain.
    """
    import pandas as pd

    headers, max_length = self._effective_signals_schema.get_headers_with_length(
        include_hidden=include_hidden
    )
    if flatten or max_length < 2:
        columns = [".".join(filter(None, header)) for header in headers]
    else:
        columns = pd.MultiIndex.from_tuples(map(tuple, headers))

    results = self.results(include_hidden=include_hidden)
    return pd.DataFrame.from_records(results, columns=columns)

to_parquet 

to_parquet(
    path: Union[str, PathLike[str], BinaryIO],
    partition_cols: Optional[Sequence[str]] = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs
) -> None

Save chain to parquet file with SignalSchema metadata.

Parameters:

  • path (Union[str, PathLike[str], BinaryIO]) –

    Path or a file-like binary object to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • partition_cols (Optional[Sequence[str]], default: None ) –

    Column names by which to partition the dataset.

  • chunk_size (int, default: DEFAULT_PARQUET_CHUNK_SIZE ) –

    The chunk size of results to read and convert to columnar data, to avoid running out of memory.

  • fs_kwargs (Optional[dict[str, Any]], default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
def to_parquet(
    self,
    path: Union[str, os.PathLike[str], BinaryIO],
    partition_cols: Optional[Sequence[str]] = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs,
) -> None:
    """Save chain to parquet file with SignalSchema metadata.

    Parameters:
        path: Path or a file-like binary object to save the file. This supports
            local paths as well as remote paths, such as s3:// or hf:// with fsspec.
        partition_cols: Column names by which to partition the dataset.
        chunk_size: The chunk size of results to read and convert to columnar
            data, to avoid running out of memory.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import pyarrow as pa
    import pyarrow.parquet as pq

    from datachain.lib.arrow import DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY

    fsspec_fs = None

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        if path.startswith("file://"):
            # pyarrow does not handle file:// uris, and needs a direct path instead.
            from urllib.parse import urlparse

            path = urlparse(path).path
            if sys.platform == "win32":
                path = os.path.normpath(path.lstrip("/"))

        fsspec_fs = client.create_fs(**fs_kwargs)

    _partition_cols = list(partition_cols) if partition_cols else None
    signal_schema_metadata = json.dumps(
        self._effective_signals_schema.serialize(), ensure_ascii=False
    ).encode("utf-8")

    column_names, column_chunks = self.to_columnar_data_with_names(chunk_size)

    parquet_schema = None
    parquet_writer = None
    first_chunk = True

    for chunk in column_chunks:
        # pyarrow infers the best parquet schema from the python types of
        # the input data.
        table = pa.Table.from_pydict(
            dict(zip(column_names, chunk)),
            schema=parquet_schema,
        )

        # Preserve any existing metadata, and add the DataChain SignalSchema.
        existing_metadata = table.schema.metadata or {}
        merged_metadata = {
            **existing_metadata,
            DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY: signal_schema_metadata,
        }
        table = table.replace_schema_metadata(merged_metadata)
        parquet_schema = table.schema

        if _partition_cols:
            # Write to a partitioned parquet dataset.
            pq.write_to_dataset(
                table,
                root_path=path,
                partition_cols=_partition_cols,
                filesystem=fsspec_fs,
                **kwargs,
            )
        else:
            if first_chunk:
                # Write to a single parquet file.
                parquet_writer = pq.ParquetWriter(
                    path, parquet_schema, filesystem=fsspec_fs, **kwargs
                )
                first_chunk = False

            assert parquet_writer
            parquet_writer.write_table(table)

    if parquet_writer:
        parquet_writer.close()

to_pytorch 

to_pytorch(
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
)

Convert to pytorch dataset format.

Parameters:

  • transform (Transform, default: None ) –

    Torchvision transforms to apply to the dataset.

  • tokenizer (Callable, default: None ) –

    Tokenizer to use to tokenize text values.

  • tokenizer_kwargs (dict, default: None ) –

    Additional kwargs to pass when calling tokenizer.

  • num_samples (int, default: 0 ) –

    Number of random samples to draw for each epoch. This argument is ignored if num_samples=0 (the default).

  • remove_prefetched (bool, default: False ) –

    Whether to remove prefetched files after reading.

Example 
from torch.utils.data import DataLoader
loader = DataLoader(
    chain.select("file", "label").to_pytorch(),
    batch_size=16
)
Source code in datachain/lib/dc/datachain.py 
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
def to_pytorch(
    self,
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
):
    """Convert to pytorch dataset format.

    Args:
        transform (Transform): Torchvision transforms to apply to the dataset.
        tokenizer (Callable): Tokenizer to use to tokenize text values.
        tokenizer_kwargs (dict): Additional kwargs to pass when calling tokenizer.
        num_samples (int): Number of random samples to draw for each epoch.
            This argument is ignored if `num_samples=0` (the default).
        remove_prefetched (bool): Whether to remove prefetched files after reading.

    Example:
        ```py
        from torch.utils.data import DataLoader
        loader = DataLoader(
            chain.select("file", "label").to_pytorch(),
            batch_size=16
        )
        ```
    """
    from datachain.torch import PytorchDataset

    if self._query.attached:
        chain = self
    else:
        chain = self.persist()
    assert chain.name is not None  # for mypy
    return PytorchDataset(
        chain.name,
        chain.version,
        catalog=self.session.catalog,
        transform=transform,
        tokenizer=tokenizer,
        tokenizer_kwargs=tokenizer_kwargs,
        num_samples=num_samples,
        dc_settings=chain._settings,
        remove_prefetched=remove_prefetched,
    )

to_records 

to_records() -> list[dict[str, Any]]

Convert every row to a dictionary.

Source code in datachain/lib/dc/datachain.py 
1377
1378
1379
1380
1381
1382
1383
def to_records(self) -> list[dict[str, Any]]:
    """Convert every row to a dictionary."""

    def to_dict(cols: list[str], row: tuple[Any, ...]) -> dict[str, Any]:
        return dict(zip(cols, row))

    return self.results(row_factory=to_dict)

to_storage 

to_storage(
    output: Union[str, PathLike[str]],
    signal: str = "file",
    placement: ExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
    anon: Optional[bool] = None,
    client_config: Optional[dict] = None,
) -> None

Export files from a specified signal to a directory. Files can be exported to a local or cloud directory.

Parameters:

  • output (Union[str, PathLike[str]]) –

    Path to the target directory for exporting files.

  • signal (str, default: 'file' ) –

    Name of the signal to export files from.

  • placement (ExportPlacement, default: 'fullpath' ) –

    The method to use for naming exported files. The possible values are: "filename", "etag", "fullpath", and "checksum".

  • link_type (Literal['copy', 'symlink'], default: 'copy' ) –

    Method to use for exporting files. Falls back to 'copy' if symlinking fails.

  • num_threads (Optional[int], default: EXPORT_FILES_MAX_THREADS ) –

    number of threads to use for exporting files. By default, it uses 5 threads.

  • anon (Optional[bool], default: None ) –

    If True, we will treat cloud bucket as a public one. Default behavior depends on the previous session configuration (e.g. happens in the initial read_storage) and particular cloud storage client implementation (e.g. S3 fallbacks to anonymous access if no credentials were found).

  • client_config (Optional[dict], default: None ) –

    Optional configuration for the destination storage client

Example

Cross cloud transfer 

import datachain as dc

ds = dc.read_storage("s3://mybucket")
ds.to_storage("gs://mybucket", placement="filename")

Source code in datachain/lib/dc/datachain.py 
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
def to_storage(
    self,
    output: Union[str, os.PathLike[str]],
    signal: str = "file",
    placement: FileExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
    anon: Optional[bool] = None,
    client_config: Optional[dict] = None,
) -> None:
    """Export files from a specified signal to a directory. Files can be
    exported to a local or cloud directory.

    Args:
        output: Path to the target directory for exporting files.
        signal: Name of the signal to export files from.
        placement: The method to use for naming exported files.
            The possible values are: "filename", "etag", "fullpath", and "checksum".
        link_type: Method to use for exporting files.
            Falls back to `'copy'` if symlinking fails.
        num_threads: number of threads to use for exporting files.
            By default, it uses 5 threads.
        anon: If True, we will treat cloud bucket as a public one. Default behavior
            depends on the previous session configuration (e.g. happens in the
            initial `read_storage`) and particular cloud storage client
            implementation (e.g. S3 fallbacks to anonymous access if no credentials
            were found).
        client_config: Optional configuration for the destination storage client

    Example:
        Cross cloud transfer
        ```py
        import datachain as dc

        ds = dc.read_storage("s3://mybucket")
        ds.to_storage("gs://mybucket", placement="filename")
        ```
    """
    chain = self.persist()
    count = chain.count()

    if placement == "filename" and (
        chain._query.distinct(pathfunc.name(C(f"{signal}__path"))).count() != count
    ):
        raise ValueError("Files with the same name found")

    if anon is not None:
        client_config = (client_config or {}) | {"anon": anon}

    progress_bar = tqdm(
        desc=f"Exporting files to {output}: ",
        unit=" files",
        unit_scale=True,
        unit_divisor=10,
        total=count,
        leave=False,
    )
    file_exporter = FileExporter(
        output,
        placement,
        self._settings.cache if self._settings else False,
        link_type,
        max_threads=num_threads or 1,
        client_config=client_config,
    )
    file_exporter.run(
        (rows[0] for rows in chain.to_iter(signal)),
        progress_bar,
    )

to_values 

to_values(col: str) -> list[DataValue]

Returns a flat list of values from a single column.

Parameters:

  • col (str) –

    The name of the column to extract values from.

Returns:

  • list[DataValue] –

    list[DataValue]: Returns a flat list of values from the specified column.

Example

Getting all values from a single column: 

file_paths = dc.to_values("file.path")
print(file_paths)  # Returns list of strings

Getting all file sizes: 

sizes = dc.to_values("file.size")
print(sizes)  # Returns list of integers

Source code in datachain/lib/dc/datachain.py 
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
def to_values(self, col: str) -> list[DataValue]:
    """Returns a flat list of values from a single column.

    Parameters:
        col: The name of the column to extract values from.

    Returns:
        list[DataValue]: Returns a flat list of values from the specified column.

    Example:
        Getting all values from a single column:
        ```py
        file_paths = dc.to_values("file.path")
        print(file_paths)  # Returns list of strings
        ```

        Getting all file sizes:
        ```py
        sizes = dc.to_values("file.size")
        print(sizes)  # Returns list of integers
        ```
    """
    return [row[0] for row in self.to_list(col)]

union 

union(other: Self) -> Self

Return the set union of the two datasets.

Parameters:

  • other (Self) –

    chain whose rows will be added to self.

Source code in datachain/lib/dc/datachain.py 
1609
1610
1611
1612
1613
1614
1615
1616
@delta_disabled
def union(self, other: "Self") -> "Self":
    """Return the set union of the two datasets.

    Parameters:
        other: chain whose rows will be added to `self`.
    """
    return self._evolve(query=self._query.union(other._query))

DataChainError

Bases: Exception

Session 

Session(
    name="",
    catalog: Optional[Catalog] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
)

Session is a context that keeps track of temporary DataChain datasets for a proper cleanup. By default, a global session is created.

Temporary or ephemeral datasets are the ones created without specified name. They are useful for optimization purposes and should be automatically removed.

Temp dataset has specific name format

"session_"

The suffix is optional. Both s are auto-generated.

Temp dataset examples

session_myname_624b41_48e8b4 session_4b962d_2a5dff

Parameters:

name (str): The name of the session. Only latters and numbers are supported. It can be empty. catalog (Catalog): Catalog object.

Source code in datachain/query/session.py 
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def __init__(
    self,
    name="",
    catalog: Optional["Catalog"] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
):
    if re.match(r"^[0-9a-zA-Z]*$", name) is None:
        raise ValueError(
            f"Session name can contain only letters or numbers - '{name}' given."
        )

    if not name:
        name = self.GLOBAL_SESSION_NAME

    session_uuid = uuid4().hex[: self.SESSION_UUID_LEN]
    self.name = f"{name}_{session_uuid}"
    self.is_new_catalog = not catalog
    self.catalog = catalog or get_catalog(
        client_config=client_config, in_memory=in_memory
    )
    self.dataset_versions: list[tuple[DatasetRecord, str, bool]] = []

get classmethod 

get(
    session: Optional[Session] = None,
    catalog: Optional[Catalog] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
) -> Session

Creates a Session() object from a catalog.

Parameters:

  • session (Session, default: None ) –

    Optional Session(). If not provided a new session will be created. It's needed mostly for simple API purposes.

  • catalog (Catalog, default: None ) –

    Optional catalog. By default, a new catalog is created.

Source code in datachain/query/session.py 
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
@classmethod
def get(
    cls,
    session: Optional["Session"] = None,
    catalog: Optional["Catalog"] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
) -> "Session":
    """Creates a Session() object from a catalog.

    Parameters:
        session (Session): Optional Session(). If not provided a new session will
                be created. It's needed mostly for simple API purposes.
        catalog (Catalog): Optional catalog. By default, a new catalog is created.
    """
    if session:
        return session

    # Access the active (most recent) context from the stack
    if cls.SESSION_CONTEXTS:
        session = cls.SESSION_CONTEXTS[-1]

    elif cls.GLOBAL_SESSION_CTX is None:
        cls.GLOBAL_SESSION_CTX = Session(
            cls.GLOBAL_SESSION_NAME,
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session = cls.GLOBAL_SESSION_CTX

        atexit.register(cls._global_cleanup)
        cls.ORIGINAL_EXCEPT_HOOK = sys.excepthook
        sys.excepthook = cls.except_hook
    else:
        session = cls.GLOBAL_SESSION_CTX

    if client_config and session.catalog.client_config != client_config:
        session = Session(
            "session" + uuid4().hex[:4],
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session.__enter__()

    return session

Sys

Bases: DataModel

Model for internal DataChain signals id and rand.