Skip to content

Fix pandas SettingWithCopyWarning, which really shouldn't be ignored. #2447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rapids-bot merged 1 commit into from
Jul 27, 2022
Merged

Fix pandas SettingWithCopyWarning, which really shouldn't be ignored. #2447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions python/cugraph/cugraph/structure/property_graph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ def add_vertex_data(self,
# column in the incoming dataframe, since the initial merge may not
# result in the same dtype. (see
# https://github.com/rapidsai/cudf/issues/9981)
self.__update_dataframe_dtypes(
self.__vertex_prop_dataframe = self.__update_dataframe_dtypes(
self.__vertex_prop_dataframe,
{self.vertex_col_name: dataframe[vertex_col_name].dtype})

Expand Down Expand Up @@ -430,7 +430,7 @@ def add_edge_data(self,
# column in the incoming dataframe, since the initial merge may not
# result in the same dtype. (see
# https://github.com/rapidsai/cudf/issues/9981)
self.__update_dataframe_dtypes(
self.__edge_prop_dataframe = self.__update_dataframe_dtypes(
self.__edge_prop_dataframe,
{self.src_col_name: dataframe[vertex_col_names[0]].dtype,
self.dst_col_name: dataframe[vertex_col_names[1]].dtype,
Expand Down Expand Up @@ -659,7 +659,7 @@ def extract_subgraph(self,
# possibly had their dtypes converted in order to accommodate NaN
# values. Restore the original dtypes in the resulting edges df prior
# to creating a Graph.
self.__update_dataframe_dtypes(edges, self.__edge_prop_dtypes)
edges = self.__update_dataframe_dtypes(edges, self.__edge_prop_dtypes)

return self.edge_props_to_graph(
edges,
Expand Down Expand Up @@ -727,7 +727,9 @@ def annotate_dataframe(self, df, G, edge_vertex_col_names):
inplace=True)

# restore the original dtypes
self.__update_dataframe_dtypes(new_df, self.__edge_prop_dtypes)
new_df = self.__update_dataframe_dtypes(
new_df, self.__edge_prop_dtypes
)
for col in df.columns:
new_df[col] = new_df[col].astype(df[col].dtype)

Expand Down Expand Up @@ -906,6 +908,7 @@ def __update_dataframe_dtypes(df, column_dtype_dict):
This also handles converting standard integer dtypes to nullable
integer dtypes, needed to accommodate NA values in columns.
"""
update_cols = {}
for (col, dtype) in column_dtype_dict.items():
# If the DataFrame is Pandas and the dtype is an integer type,
# ensure a nullable integer array is used by specifying the correct
Expand All @@ -919,7 +922,11 @@ def __update_dataframe_dtypes(df, column_dtype_dict):
# Assigning to df[col] produces a (false?) warning with Pandas,
# but assigning to df.loc[:,col] does not update the df in
# cudf, so do one or the other based on type.
if type(df) is cudf.DataFrame:
df[col] = df[col].astype(dtype_str)
else:
df.loc[:, col] = df[col].astype(dtype_str)
update_cols[col] = df[col].astype(dtype_str)
if not update_cols:
return df
# Use df.assign to avoid assignment into df in case df is a view:
# https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html
# #returning-a-view-versus-a-copy
# Note that this requires all column names to be strings.
return df.assign(**update_cols)
15 changes: 15 additions & 0 deletions python/cugraph/cugraph/tests/test_property_graph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,21 @@ def setup_function():
# =============================================================================
# Pytest fixtures
# =============================================================================
@pytest.fixture(scope="function", autouse=True)
def raise_on_pandas_warning():
"""Raise when pandas gives SettingWithCopyWarning warning"""
# Perhaps we should put this in pytest.ini, pyproject.toml, or conftest.py
import warnings

filters = list(warnings.filters)
warnings.filterwarnings(
"error",
category=pd.core.common.SettingWithCopyWarning
)
yield
warnings.filters = filters


Comment on lines +121 to +135
Copy link
Contributor Author

@eriknw eriknw Jul 26, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pytest does some weird things with warnings between each test, but this fixture works and is only applied to this file (and I don't know if we need to reset the filters). If we want to make this warning raise everywhere, there are better ways to achieve it.

Without this fixture, a way to make pytest raise on this warning is:

pytest -Werror::pandas.core.common.SettingWithCopyWarning cugraph/cugraph/tests/test_property_graph.py

df_types = [cudf.DataFrame, pd.DataFrame]


Expand Down