Skip to content

IO

danling.utils.io

save

Python
save(obj: Any, file: PathStr, *args: Any, **kwargs: Any) -> File

Save an object using the backend selected by its filename extension.

NumPy suffixes are case-insensitive and write to the exact requested path. .npy and .numpy use NPY encoding. .npz uses numpy.savez: obj is stored as arr_0, additional positional arrays as arr_1, arr_2, etc., and keyword arguments follow NumPy’s native contract.

Parameters:

Name Type Description Default

obj

Any

Object accepted by the selected backend.

 required

file

PathStr

String, bytes or path-like filename. Bytes paths use the filesystem encoding.

 required

*args

Any

Positional arguments forwarded to the backend.

 ()

**kwargs

Any

Keyword arguments forwarded to the backend.

 {}

Returns:

Name Type Description
File File

The original filename object supplied by the caller.
Source code in danling/utils/io.py
Python
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
def save(obj: Any, file: PathStr, *args: Any, **kwargs: Any) -> File:
    r"""
    Save an object using the backend selected by its filename extension.

    NumPy suffixes are case-insensitive and write to the exact requested path.
    ``.npy`` and ``.numpy`` use NPY encoding. ``.npz`` uses ``numpy.savez``:
    ``obj`` is stored as ``arr_0``, additional positional arrays as ``arr_1``,
    ``arr_2``, etc., and keyword arguments follow NumPy's native contract.

    Args:
        obj: Object accepted by the selected backend.
        file: String, bytes or path-like filename. Bytes paths use the filesystem encoding.
        *args: Positional arguments forwarded to the backend.
        **kwargs: Keyword arguments forwarded to the backend.

    Returns:
        File: The original filename object supplied by the caller.
    """
    path = os.fsdecode(file)
    extension = os.path.splitext(path)[-1].lower()[1:]
    if extension in PYTORCH:
        if not TORCH_AVAILABLE:
            raise ImportError(f"Trying to save {obj} to {file!r} but torch is not installed.")
        torch.save(obj, path, *args, **kwargs)
    elif extension in NUMPY:
        if not NUMPY_AVAILABLE:
            raise ImportError(f"Trying to save {obj} to {file!r} but numpy is not installed.")
        with open(path, "wb") as array_file:
            if extension == "npz":
                numpy.savez(array_file, obj, *args, **kwargs)
            else:
                numpy.save(array_file, obj, *args, **kwargs)
    elif extension in PANDAS:
        if not PANDAS_AVAILABLE:
            raise ImportError(f"Trying to save {obj} to {file!r} but pandas is not installed.")
        pandas.to_pickle(obj, path, *args, **kwargs)
    elif extension in PARQUET:
        if isinstance(obj, pandas.DataFrame):
            obj.to_parquet(path, *args, **kwargs)
        elif not PYARROW_AVAILABLE:
            raise ImportError(f"Trying to save {obj} to {file!r} but pyarrow is not installed.")
        else:
            pyarrow.parquet.write_table(obj, path, *args, **kwargs)
    elif extension in CSV:
        if isinstance(obj, pandas.DataFrame):
            obj.to_csv(path, *args, **kwargs)
        else:
            raise NotImplementedError(f"Trying to save {obj} to {file!r} but is not supported")
    elif extension in JSON:
        if isinstance(obj, FlatDict):
            obj.json(path, *args, **kwargs)
        else:
            with open(path, "w") as fp:
                json.dump(obj, fp, *args, **kwargs)  # type: ignore[arg-type]
    elif extension in YAML:
        if isinstance(obj, FlatDict):
            obj.yaml(path, *args, **kwargs)
        else:
            with open(path, "w") as fp:
                yaml.dump(obj, fp, *args, **kwargs)  # type: ignore[arg-type, call-overload]
    elif extension in PICKLE:
        with open(path, "wb") as fp:
            pickle.dump(obj, fp, *args, **kwargs)  # type: ignore[arg-type]
    else:
        raise ValueError(f"Tying to save {obj} to {file!r} with unsupported extension={extension!r}")
    return file

load

Python
load(file: PathStr, *args: Any, **kwargs: Any) -> Any

Load an object using the backend selected by its filename extension.

NumPy files return the native numpy.load result. In particular, an NPZ archive returns an NpzFile that the caller must close, preferably with a with statement. NPY files return an array (or a memory map when requested).

Parameters:

Name Type Description Default

file

PathStr

String, bytes or path-like filename. Bytes paths use the filesystem encoding.

 required

*args

Any

Positional arguments forwarded to the backend.

 ()

**kwargs

Any

Keyword arguments forwarded to the backend.

 {}

Returns:

Name Type Description
Any Any

The object returned by the selected backend.

Raises:

Type Description
ValueError

The filename is not a file or has an unsupported extension.

Examples:

Python Console Session
1
2
3
4
5
6
7
8
9
>>> from pathlib import Path
>>> from tempfile import TemporaryDirectory
>>> import numpy as np
>>> with TemporaryDirectory() as directory:
...     path = Path(directory) / "values.npz"
...     _ = save(np.array([1, 2]), path)
...     with load(path) as archive:
...         print(archive["arr_0"].tolist())
[1, 2]
Source code in danling/utils/io.py
Python
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
def load(file: PathStr, *args: Any, **kwargs: Any) -> Any:
    r"""
    Load an object using the backend selected by its filename extension.

    NumPy files return the native ``numpy.load`` result. In particular, an NPZ
    archive returns an ``NpzFile`` that the caller must close, preferably with
    a ``with`` statement. NPY files return an array (or a memory map when requested).

    Args:
        file: String, bytes or path-like filename. Bytes paths use the filesystem encoding.
        *args: Positional arguments forwarded to the backend.
        **kwargs: Keyword arguments forwarded to the backend.

    Returns:
        Any: The object returned by the selected backend.

    Raises:
        ValueError: The filename is not a file or has an unsupported extension.

    Examples:
        >>> from pathlib import Path
        >>> from tempfile import TemporaryDirectory
        >>> import numpy as np
        >>> with TemporaryDirectory() as directory:
        ...     path = Path(directory) / "values.npz"
        ...     _ = save(np.array([1, 2]), path)
        ...     with load(path) as archive:
        ...         print(archive["arr_0"].tolist())
        [1, 2]
    """
    if not os.path.isfile(file):
        raise ValueError(f"Trying to load {file!r} but it is not a file.")
    path = os.fsdecode(file)
    extension = os.path.splitext(path)[-1].lower()[1:]
    if extension in PYTORCH:
        if not TORCH_AVAILABLE:
            raise ImportError(f"Trying to load {file!r} but torch is not installed.")
        return torch.load(path, *args, **kwargs)
    if extension in NUMPY:
        if not NUMPY_AVAILABLE:
            raise ImportError(f"Trying to load {file!r} but numpy is not installed.")
        return numpy.load(path, *args, **kwargs)
    if extension in JSON:
        with open(path) as fp:
            return json.load(fp, *args, **kwargs)  # type: ignore[arg-type]
    if extension in YAML:
        with open(path) as fp:
            kwargs.setdefault("Loader", yaml.FullLoader)  # type: ignore[arg-type]
            return yaml.load(fp, *args, **kwargs)  # type: ignore[arg-type]
    if extension in PICKLE:
        with open(path, "rb") as fp:
            return pickle.load(fp, *args, **kwargs)  # type: ignore[arg-type]
    if extension in PANDAS_SUPPORTED:
        return load_pandas(path, *args, **kwargs)
    raise ValueError(f"Tying to load {file!r} with unsupported extension={extension!r}")

load_pandas

Python
load_pandas(file: PathStr, *args: Any, **kwargs: Any) -> Any

Load any pandas data file with supported extensions.

Source code in danling/utils/io.py
Python
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def load_pandas(file: PathStr, *args: Any, **kwargs: Any) -> Any:
    r"""
    Load any pandas data file with supported extensions.
    """
    if not PANDAS_AVAILABLE:
        raise ImportError(f"Trying to load {file!r} but pandas is not installed.")
    if not os.path.isfile(file):
        raise ValueError(f"Trying to load {file!r} but it is not a file.")
    path = os.fsdecode(file)
    extension = os.path.splitext(path)[-1].lower()[1:]
    if extension in PANDAS or extension in PICKLE:
        return pandas.read_pickle(path, *args, **kwargs)
    if extension in PARQUET:
        return pandas.read_parquet(path, *args, **kwargs)
    if extension in H5:
        return pandas.read_hdf(path, *args, **kwargs)
    if extension in CSV:
        return pandas.read_csv(path, *args, **kwargs)
    if extension in JSON:
        return pandas.read_json(path, *args, **kwargs)
    if extension in EXCEL:
        return pandas.read_excel(path, *args, **kwargs)
    if extension in XML:
        return pandas.read_xml(path, *args, **kwargs)
    if extension in SQL:
        return pandas.read_sql(path, *args, **kwargs)
    raise ValueError(f"Tying to load {file!r} with unsupported extension={extension!r}")

is_json_serializable

Python
is_json_serializable(obj: Any) -> bool

Check if obj is JSON serializable.

Source code in danling/utils/io.py
Python
228
229
230
231
232
233
234
235
236
def is_json_serializable(obj: Any) -> bool:
    r"""
    Check if `obj` is JSON serializable.
    """
    try:
        json.dumps(obj)
        return True
    except (TypeError, OverflowError):
        return False