Operations¶

Data synchronization operations.

sync_data ¶

sync_data(
    source_path,
    destination_path,
    source_path_prefix=None,
    filter_config=None,
    filter_root=None,
    max_concurrency=10,
    retain_source_data=True,
    delete=True,
    require_lock=False,
    force=False,
    size_only=False,
    fail_if_missing=True,
    remote_to_local_config=None,
    include_detailed_response=False,
)

Sync data from a source path to a destination path.

Parameters:

Name	Type	Description	Default
`source_path`	`S3Path \| LocalPath`	Path to sync data from.	required
`destination_path`	`S3Path \| LocalPath`	Path to sync data to.	required
`source_path_prefix`	`str \| None`	Optional S3 key prefix scoping the source.	`None`
`filter_config`	`DataSyncFilterConfig \| None`	Optional include/exclude filters describing what to move. Not supported for local -> local syncs, which warn and copy in full.	`None`
`filter_root`	`str \| None`	Root that filter patterns are matched relative to. Defaults to the source path; set explicitly when syncing a sub-prefix of the root the patterns were written against.	`None`
`max_concurrency`	`int`	Maximum number of concurrent transfer operations.	`10`
`retain_source_data`	`bool`	Whether to keep source data after syncing.	`True`
`delete`	`bool`	Whether to delete destination paths absent from the (filtered) source. Note that this combines destructively with `filter_config` -- see `DataSyncConfig.delete`.	`True`
`require_lock`	`bool`	Whether to acquire a lock on the destination path.	`False`
`force`	`bool`	Whether to transfer regardless of existing destination content.	`False`
`size_only`	`bool`	Whether to compare only file sizes when deciding to transfer.	`False`
`fail_if_missing`	`bool`	Whether to raise if the source path does not exist.	`True`
`remote_to_local_config`	`RemoteToLocalConfig \| None`	Options specific to remote-to-local syncs.	`None`
`include_detailed_response`	`bool`	Whether to compute detailed transfer metrics.	`False`

Returns:

Type	Description
	The sync result.

Source code in src/aibs_informatics_aws_utils/data_sync/operations.py

def sync_data(
    source_path: S3Path | LocalPath,
    destination_path: S3Path | LocalPath,
    source_path_prefix: str | None = None,
    filter_config: DataSyncFilterConfig | None = None,
    filter_root: str | None = None,
    max_concurrency: int = 10,
    retain_source_data: bool = True,
    delete: bool = True,
    require_lock: bool = False,
    force: bool = False,
    size_only: bool = False,
    fail_if_missing: bool = True,
    remote_to_local_config: RemoteToLocalConfig | None = None,
    include_detailed_response: bool = False,
):
    """Sync data from a source path to a destination path.

    Args:
        source_path: Path to sync data from.
        destination_path: Path to sync data to.
        source_path_prefix: Optional S3 key prefix scoping the source.
        filter_config: Optional include/exclude filters describing what to move.
            Not supported for local -> local syncs, which warn and copy in full.
        filter_root: Root that filter patterns are matched relative to. Defaults
            to the source path; set explicitly when syncing a sub-prefix of the
            root the patterns were written against.
        max_concurrency: Maximum number of concurrent transfer operations.
        retain_source_data: Whether to keep source data after syncing.
        delete: Whether to delete destination paths absent from the (filtered)
            source. Note that this combines destructively with ``filter_config``
            -- see ``DataSyncConfig.delete``.
        require_lock: Whether to acquire a lock on the destination path.
        force: Whether to transfer regardless of existing destination content.
        size_only: Whether to compare only file sizes when deciding to transfer.
        fail_if_missing: Whether to raise if the source path does not exist.
        remote_to_local_config: Options specific to remote-to-local syncs.
        include_detailed_response: Whether to compute detailed transfer metrics.

    Returns:
        The sync result.
    """
    request = DataSyncRequest(
        source_path=source_path,
        destination_path=destination_path,
        source_path_prefix=S3KeyPrefix(source_path_prefix) if source_path_prefix else None,
        filter_config=filter_config,
        filter_root=filter_root,
        max_concurrency=max_concurrency,
        retain_source_data=retain_source_data,
        delete=delete,
        require_lock=require_lock,
        force=force,
        size_only=size_only,
        fail_if_missing=fail_if_missing,
        remote_to_local_config=remote_to_local_config or RemoteToLocalConfig(),
        include_detailed_response=include_detailed_response,
    )
    return DataSyncOperations.sync_request(request=request)