From e9fdb5b236403aa3f63e4aa81b4259f21d99071a Mon Sep 17 00:00:00 2001 From: Bryan Harter <41062454+bryan-harter@users.noreply.github.com> Date: Fri, 15 Mar 2024 00:25:55 +0000 Subject: [PATCH 1/2] Add a check for Epoch being monotonically increasing --- cdflib/xarray/cdf_to_xarray.py | 2 +- cdflib/xarray/xarray_to_cdf.py | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/cdflib/xarray/cdf_to_xarray.py b/cdflib/xarray/cdf_to_xarray.py index 8050d76..57813b9 100644 --- a/cdflib/xarray/cdf_to_xarray.py +++ b/cdflib/xarray/cdf_to_xarray.py @@ -616,7 +616,7 @@ def _generate_xarray_data_variables( var_data = _convert_fillvals_to_nan(var_data, var_atts, var_props) # Finally, create the new variable - created_vars[var_name] = xr.Variable(var_dims, var_data, attrs=var_atts) # type: ignore[no-untyped-call] + created_vars[var_name] = xr.Variable(var_dims, var_data, attrs=var_atts) return created_vars, depend_dimensions diff --git a/cdflib/xarray/xarray_to_cdf.py b/cdflib/xarray/xarray_to_cdf.py index f683fd7..08d3908 100644 --- a/cdflib/xarray/xarray_to_cdf.py +++ b/cdflib/xarray/xarray_to_cdf.py @@ -319,7 +319,14 @@ def _epoch_checker(dataset: xr.Dataset, dim_vars: List[str], terminate_on_warnin epoch_found = False for d in depend_0_list: if d.lower().startswith("epoch"): - epoch_found = True + monotonically_increasing = _verify_monotonically_increasing(dataset[d].data) + if monotonically_increasing: + epoch_found = True + else: + _warn_or_except( + f"Variable {d} was determined to be an ISTP 'Epoch' variable, but it is not monotonically increasing.", + terminate_on_warning, + ) if not epoch_found: _warn_or_except( @@ -330,6 +337,10 @@ def _epoch_checker(dataset: xr.Dataset, dim_vars: List[str], terminate_on_warnin return depend_0_list, time_varying_dimensions +def _verify_monotonically_increasing(epoch_data: npt.NDArray) -> np.bool_: + return np.all(epoch_data[1:] > epoch_data[:-1]) + + def _add_depend_variables_to_dataset( dataset: xr.Dataset, dim_vars: List[str], @@ -402,8 +413,10 @@ def _variable_attribute_checker(dataset: xr.Dataset, epoch_list: List[str], term # Ensure None of the attributes are given a type of "None" for key, value in d[var].attrs.items(): if value is None: - _warn_or_except(f"CDF Warning: {key} was given a type of None for variable {var}. CDF does not allow None types, so {key} will be skipped.", terminate_on_warning) - + _warn_or_except( + f"CDF Warning: {key} was given a type of None for variable {var}. CDF does not allow None types, so {key} will be skipped.", + terminate_on_warning, + ) # Check for VAR_TYPE if "VAR_TYPE" not in d[var].attrs: @@ -817,7 +830,10 @@ def xarray_to_cdf( var_data = _datetime_to_tt2000(d[var].data) elif datetime64_to_cdftt2000: if d[var].dtype.type != np.datetime64: - _warn_or_except(f"datetime64_to_cdftt2000 is set, but datetime64 is not used in the {var} variable", terminate_on_warning) + _warn_or_except( + f"datetime64_to_cdftt2000 is set, but datetime64 is not used in the {var} variable", + terminate_on_warning, + ) else: unixtime_from_datetime64 = d[var].data.astype("datetime64[ns]").astype("int64") / 1000000000 var_data = _unixtime_to_tt2000(unixtime_from_datetime64) From f782e0aeb906f9a5d9d56848c1cd668dea4ec86d Mon Sep 17 00:00:00 2001 From: Bryan Harter <41062454+bryan-harter@users.noreply.github.com> Date: Fri, 15 Mar 2024 16:39:21 +0000 Subject: [PATCH 2/2] Fixing python linting stuff --- cdflib/cdfwrite.py | 14 ++++++++------ tests/test_xarray_reader_writer.py | 26 +++++++++++++------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cdflib/cdfwrite.py b/cdflib/cdfwrite.py index 923c930..1e497af 100644 --- a/cdflib/cdfwrite.py +++ b/cdflib/cdfwrite.py @@ -858,7 +858,9 @@ def _write_var_attrs(self, f: io.BufferedWriter, varNum: int, var_attrs: Dict[st offset = self.attrsinfo[attrNum][2] if entry is None: - logger.warning(f"Attribute: {attr}" + " is None type, which does not have an equivalent in CDF... Skipping attribute.") + logger.warning( + f"Attribute: {attr}" + " is None type, which does not have an equivalent in CDF... Skipping attribute." + ) continue # Check if dataType was provided @@ -2542,14 +2544,14 @@ def _checklistofstrs(obj: Any) -> bool: @staticmethod def _checklistofNums(obj: Any) -> bool: - ''' + """ This method checks if a list is ready to be immediately converted to binary format, - or if any pre-processing needs to occur. Numbers and datetime64 objects can be immediately converted. - ''' + or if any pre-processing needs to occur. Numbers and datetime64 objects can be immediately converted. + """ if hasattr(obj, "__len__"): - return bool(all(obj)) and all((isinstance(elem, numbers.Number) or isinstance(elem, np.datetime64)) for elem in obj) + return bool(all(obj)) and all((isinstance(elem, numbers.Number) or isinstance(elem, np.datetime64)) for elem in obj) else: - return (isinstance(obj, numbers.Number) or isinstance(obj, np.datetime64)) + return isinstance(obj, numbers.Number) or isinstance(obj, np.datetime64) def _md5_compute(self, f: io.BufferedWriter) -> bytes: """ diff --git a/tests/test_xarray_reader_writer.py b/tests/test_xarray_reader_writer.py index 6c84778..e135607 100644 --- a/tests/test_xarray_reader_writer.py +++ b/tests/test_xarray_reader_writer.py @@ -314,10 +314,10 @@ def test_build_from_scratch(): pytest.importorskip("xarray") var_data = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] var_dims = ["epoch", "direction"] - data = xr.Variable(var_dims, var_data) # type: ignore[no-untyped-call] + data = xr.Variable(var_dims, var_data) epoch_data = [1, 2, 3] epoch_dims = ["epoch"] - epoch = xr.Variable(epoch_dims, epoch_data) # type: ignore[no-untyped-call] + epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf") os.remove("hello.cdf") @@ -337,14 +337,14 @@ def test_build_from_scratch(): "Logical_source": ":)", "Logical_source_description": ":(", } - data = xr.Variable(var_dims, var_data) # type: ignore[no-untyped-call] - epoch = xr.Variable(epoch_dims, epoch_data) # type: ignore[no-untyped-call] + data = xr.Variable(var_dims, var_data) + epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}, attrs=global_attributes) xarray_to_cdf(ds, "hello.cdf") os.remove("hello.cdf") dir_data = [1, 2, 3] dir_dims = ["direction"] - direction = xr.Variable(dir_dims, dir_data) # type: ignore[no-untyped-call] + direction = xr.Variable(dir_dims, dir_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch, "direction": direction}, attrs=global_attributes) xarray_to_cdf(ds, "hello.cdf") os.remove("hello.cdf") @@ -361,10 +361,10 @@ def test_datetime64_conversion(): pytest.importorskip("xarray") var_data = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] var_dims = ["epoch", "direction"] - data = xr.Variable(var_dims, var_data) # type: ignore[no-untyped-call] + data = xr.Variable(var_dims, var_data) epoch_data = [np.datetime64(1, "s"), np.datetime64(2, "s"), np.datetime64(3, "s")] epoch_dims = ["epoch"] - epoch = xr.Variable(epoch_dims, epoch_data) # type: ignore[no-untyped-call] + epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf", datetime64_to_cdftt2000=True) x = cdf_to_xarray("hello.cdf", to_datetime=True) @@ -378,10 +378,10 @@ def test_datetime64_no_conversion(): pytest.importorskip("xarray") var_data = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] var_dims = ["epoch", "direction"] - data = xr.Variable(var_dims, var_data) # type: ignore[no-untyped-call] + data = xr.Variable(var_dims, var_data) epoch_data = [np.datetime64(1, "s"), np.datetime64(2, "s"), np.datetime64(3, "s")] epoch_dims = ["epoch"] - epoch = xr.Variable(epoch_dims, epoch_data) # type: ignore[no-untyped-call] + epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf") x = cdf_to_xarray("hello.cdf") @@ -396,10 +396,10 @@ def test_datetime64_conversion_odd_units(): pytest.importorskip("xarray") var_data = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] var_dims = ["epoch", "direction"] - data = xr.Variable(var_dims, var_data) # type: ignore[no-untyped-call] + data = xr.Variable(var_dims, var_data) epoch_data = [np.datetime64("2000-01-01"), np.datetime64("2000-01-02"), np.datetime64("2000-01-03")] epoch_dims = ["epoch"] - epoch = xr.Variable(epoch_dims, epoch_data) # type: ignore[no-untyped-call] + epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf", datetime64_to_cdftt2000=True) x = cdf_to_xarray("hello.cdf", to_datetime=True) @@ -413,10 +413,10 @@ def test_numpy_string_array(): pytest.importorskip("xarray") var_data = ["a", "b", "c"] var_dims = ["epoch"] - data = xr.Variable(var_dims, var_data) # type: ignore[no-untyped-call] + data = xr.Variable(var_dims, var_data) epoch_data = [np.datetime64("2000-01-01"), np.datetime64("2000-01-02"), np.datetime64("2000-01-03")] epoch_dims = ["epoch"] - epoch = xr.Variable(epoch_dims, epoch_data) # type: ignore[no-untyped-call] + epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf", datetime64_to_cdftt2000=True) x = cdf_to_xarray("hello.cdf", to_datetime=True)