From 930ed44b8ba80c2a8b794aabdbd532d6a3bfe1f8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 17 Jul 2026 12:42:16 +0100 Subject: [PATCH] migrate: convert missing-values.ipynb from pandas to polars --- missing-values.ipynb | 181 ++++++++++++++++++++++++++----------------- 1 file changed, 110 insertions(+), 71 deletions(-) diff --git a/missing-values.ipynb b/missing-values.ipynb index 67e95ef..37d46e8 100644 --- a/missing-values.ipynb +++ b/missing-values.ipynb @@ -9,7 +9,7 @@ "\n", "## Introduction\n", "\n", - "In this chapter, we'll look at the tools and tricks for dealing with missing values. We'll start by discussing some general tools for working with missing values recorded as `NA`s. We'll then explore the idea of implicitly missing values, values are that are simply absent from your data, and show some tools you can use to make them explicit.\n", + "In this chapter, we'll look at the tools and tricks for dealing with missing values. We'll start by discussing some general tools for working with missing values recorded as `null` values. We'll then explore the idea of implicitly missing values, values are that are simply absent from your data, and show some tools you can use to make them explicit.\n", "We'll finish off with a related discussion of empty groups, caused by categories that don't appear in the data." ] }, @@ -40,7 +40,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **pandas** data analysis package." + "This chapter will use the **polars** data analysis package." ] }, { @@ -50,13 +50,11 @@ "source": [ "## Explicit Missing Values\n", "\n", - "To begin, let's explore a few handy tools for creating or eliminating missing explicit values, i.e. cells where you see an `NA` or `nan`.\n", + "To begin, let's explore a few handy tools for creating or eliminating missing explicit values, i.e. cells where you see an `null` value.\n", "\n", "### Types of Missing Values\n", "\n", - "One thing to note about missing values in **pandas** is that they are not all created alike!\n", - "\n", - "For example, real numbers in **pandas** (such as the `float64` dtype) uses a 'nan' (aka, Not a Number):" + "In polars, missing values are represented by `null`:" ] }, { @@ -67,9 +65,9 @@ "outputs": [], "source": [ "import numpy as np\n", - "import pandas as pd\n", + "import polars as pl\n", "\n", - "df = pd.DataFrame([5, 27.3, np.nan, -16], columns=[\"numbers\"])\n", + "df = pl.DataFrame({\"numbers\": [5.0, 27.3, None, -16.0]})\n", "df" ] }, @@ -78,7 +76,7 @@ "id": "4b1c9ea7", "metadata": {}, "source": [ - "But this isn't the only way! We can also use Python's built-in `None` value (which, here, gets converted to a NaN because the valid values are all floating point numbers) and **pandas**' `pd.NA`:" + "Polars also handles Python's built-in None values seamlessly:" ] }, { @@ -88,7 +86,7 @@ "metadata": {}, "outputs": [], "source": [ - "numbers = pd.DataFrame([pd.NA, 27.3, np.nan, -16, None], columns=[\"numbers\"])\n", + "numbers = pl.DataFrame({\"numbers\": [None, 27.3, np.nan, -16.0, None]})\n", "numbers" ] }, @@ -97,7 +95,7 @@ "id": "b2dda456", "metadata": {}, "source": [ - "However, with the object data type (the default for strings), the types can co-exist:" + "For string columns, missing values are also represented by `null`:" ] }, { @@ -107,9 +105,7 @@ "metadata": {}, "outputs": [], "source": [ - "fruits = pd.DataFrame(\n", - " [\"orange\", np.nan, \"apple\", None, \"banana\", pd.NA], columns=[\"fruit\"]\n", - ")\n", + "fruits = pl.DataFrame({\"fruit\": [\"orange\", None, \"apple\", None, \"banana\", None]})\n", "fruits" ] }, @@ -118,7 +114,7 @@ "id": "8b5a2849", "metadata": {}, "source": [ - "Both of these types of missing value can be found using **pandas** `.isna()` function. This returns a new column of boolean values that are `True` if the value is any kind of missing value." + "Both types of missing value can be found using the `.is_null()` method, which returns a new column of boolean values that are `True` if the value is missing:" ] }, { @@ -128,7 +124,7 @@ "metadata": {}, "outputs": [], "source": [ - "fruits.isna()" + "fruits.select(pl.col(\"fruit\").is_null())" ] }, { @@ -136,7 +132,17 @@ "id": "0fd6764d", "metadata": {}, "source": [ - "As a convenience, there is also a `notna()` function." + "As a convenience, there is also an .is_not_null() method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3fb8082", + "metadata": {}, + "outputs": [], + "source": [ + "fruits.select(pl.col(\"fruit\").is_not_null())" ] }, { @@ -146,7 +152,7 @@ "source": [ "### Dealing with Explicit Missing Values\n", "\n", - "There are various options for dealing with missing values. The `fillna()` function achieves this. Let's take a look at it with some test data:" + "There are various options for dealing with missing values. The `fill_null()` method achieves this. Let's take a look at it with some test data:" ] }, { @@ -156,17 +162,15 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df = pd.DataFrame(\n", - " [\n", - " [np.nan, 2, None, 0],\n", - " [3, 4, np.nan, 1],\n", - " [5, np.nan, np.nan, pd.NA],\n", - " [np.nan, 3, np.nan, 4],\n", - " ],\n", - " columns=list(\"ABCD\"),\n", + "null_df = pl.DataFrame(\n", + " {\n", + " \"A\": [None, 3, 5, None],\n", + " \"B\": [2, 4, None, 3],\n", + " \"C\": [None, None, None, None],\n", + " \"D\": [0, 1, None, 4],\n", + " }\n", ")\n", - "\n", - "nan_df" + "null_df" ] }, { @@ -184,7 +188,7 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df.fillna(0)" + "null_df.fill_null(0)" ] }, { @@ -192,7 +196,7 @@ "id": "af8e2cb7", "metadata": {}, "source": [ - "This can be done on a by-column basis; here we replace all NaN elements in column ‘A’, ‘B’, ‘C’, and ‘D’, with 0, 1, 2, and 3 respectively." + "This can be done on a by-column basis by passing a dictionary:" ] }, { @@ -202,7 +206,14 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3})" + "null_df.with_columns(\n", + " [\n", + " pl.col(\"A\").fill_null(0),\n", + " pl.col(\"B\").fill_null(1),\n", + " pl.col(\"C\").fill_null(2),\n", + " pl.col(\"D\").fill_null(3),\n", + " ]\n", + ")" ] }, { @@ -210,7 +221,7 @@ "id": "2544fe5a", "metadata": {}, "source": [ - "We can also propagate non-null values forward or backward (relative to the index)" + "We can also propagate non-null values forward or backward:" ] }, { @@ -220,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df.fillna(method=\"ffill\")" + "null_df.fill_null(strategy=\"forward\")" ] }, { @@ -230,7 +241,7 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df.fillna(method=\"bfill\")" + "null_df.fill_null(strategy=\"backward\")" ] }, { @@ -246,7 +257,7 @@ "id": "ca7e4ca9", "metadata": {}, "source": [ - "Another feature of all of these functions is that you can limit the number of NaNs that get replaced using the `limit=` keyword argument." + "Another feature of all of these functions is that you can limit the number of nulls that get replaced using the `limit=` keyword argument." ] }, { @@ -256,7 +267,8 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3}, limit=1)" + "# Forward fill with limit of 1\n", + "null_df.fill_null(strategy=\"forward\", limit=2)" ] }, { @@ -264,7 +276,7 @@ "id": "99b94a01", "metadata": {}, "source": [ - "Of course, another option might be just to filter out the missing values altogether. There are a couple of ways to do this depending if you want to remove entire rows (`axis=0`) or columns (`axis=1`—but in this case, as there is at least one NaN in each column though, there will be no data left!)" + "Of course, another option might be just to filter out the missing values altogether. The `.drop_nulls()` method removes rows with any missing values:" ] }, { @@ -274,7 +286,15 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df[\"A\"].dropna(axis=0) # on a single column" + "null_df.drop_nulls()" + ] + }, + { + "cell_type": "markdown", + "id": "710bcb2c", + "metadata": {}, + "source": [ + "You can also drop rows where all values are null using a custom filter:" ] }, { @@ -284,25 +304,34 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df.dropna(axis=1)" + "null_exp_df = pl.DataFrame(\n", + " {\n", + " \"A\": [None, 3, 5, None, None],\n", + " \"B\": [2, 4, None, 3, None],\n", + " \"C\": [None, None, None, None, None],\n", + " \"D\": [0, 1, None, 4, None],\n", + " }\n", + ")\n", + "\n", + "null_exp_df" ] }, { - "cell_type": "markdown", - "id": "9bc57652", + "cell_type": "code", + "execution_count": null, + "id": "f7b38107", "metadata": {}, + "outputs": [], "source": [ - "`dropna()` takes some keyword arguments too; for example `how=\"all\"` only drops a column or row if *all* of the values are NA." + "null_exp_df.filter(~pl.all_horizontal(pl.all().is_null()))" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "3296ea35", + "cell_type": "markdown", + "id": "2b058955", "metadata": {}, - "outputs": [], "source": [ - "nan_df.dropna(how=\"all\")" + "Notice that the only the last all null column was removed." ] }, { @@ -318,7 +347,7 @@ "id": "095f53a5", "metadata": {}, "source": [ - "Another way to filter out nans is to use the same filtering methods you would use normally, via boolean columns, in combination with the `.notna()` function. In the below example, we see all columns for the rows for which A is not NA." + "Another way to filter out nulls is to use the same filtering methods you would use normally, via boolean columns, in combination with the `.is_not_null()` method. In the below example, we see the rows for which column A is not null:" ] }, { @@ -328,7 +357,7 @@ "metadata": {}, "outputs": [], "source": [ - "nan_df[nan_df[\"A\"].notna()]" + "null_df.filter(pl.col(\"A\").is_not_null())" ] }, { @@ -340,7 +369,7 @@ "\n", "Sometimes you'll hit the opposite problem where some concrete value actually represents a missing value. This typically arises in data generated by older software that doesn't have a proper way to represent missing values, so it must instead use some special value like 99 or -999.\n", "\n", - "If possible, handle this when reading in the data, for example, by using the `na_values=` keyword argument when calling `pd.read_csv()`. If you discover the problem later, or your data source doesn't provide a way to handle it on reading the file, you can use a range of options to replace the given data:" + "If possible, handle this when reading in the data, for example, by using the `null_values=` keyword argument when calling `pl.read_csv()`. If you discover the problem later, or your data source doesn't provide a way to handle it on reading the file, you can use a range of options to replace the given data:" ] }, { @@ -350,8 +379,13 @@ "metadata": {}, "outputs": [], "source": [ - "stata_df = pd.DataFrame([[3, 4, 5], [-7, 4, -99], [-99, 6, 5]], columns=list(\"ABC\"))\n", - "\n", + "stata_df = pl.DataFrame(\n", + " {\n", + " \"A\": [3, -7, -99],\n", + " \"B\": [4, 4, 6],\n", + " \"C\": [5, -99, 5],\n", + " }\n", + ")\n", "stata_df" ] }, @@ -360,7 +394,7 @@ "id": "81685807", "metadata": {}, "source": [ - "The easiest option is probably `.replace()`:" + "The easiest option is probably the expression-level `replace()` method:" ] }, { @@ -370,7 +404,7 @@ "metadata": {}, "outputs": [], "source": [ - "stata_df.replace({-99: pd.NA})" + "stata_df.with_columns(pl.all().replace(-99, None))" ] }, { @@ -378,7 +412,7 @@ "id": "9d4d3d3d", "metadata": {}, "source": [ - "Because `.replace()` accepts a dictionary, it's possible to replace several values at once:" + "Because `replace()` accepts a dictionary, it's possible to replace several values at once:" ] }, { @@ -388,7 +422,7 @@ "metadata": {}, "outputs": [], "source": [ - "stata_df.replace({-99: pd.NA, -7: pd.NA})" + "stata_df.with_columns(pl.all().replace({-99: None, -7: None}))" ] }, { @@ -418,11 +452,11 @@ "metadata": {}, "outputs": [], "source": [ - "stocks = pd.DataFrame(\n", + "stocks = pl.DataFrame(\n", " {\n", " \"year\": [2020, 2020, 2020, 2020, 2021, 2021, 2021],\n", " \"qtr\": [1, 2, 3, 4, 2, 3, 4],\n", - " \"price\": [1.88, 0.59, 0.35, np.nan, 0.92, 0.17, 2.66],\n", + " \"price\": [1.88, 0.59, 0.35, None, 0.92, 0.17, 2.66],\n", " }\n", ")\n", "stocks" @@ -467,7 +501,7 @@ "metadata": {}, "outputs": [], "source": [ - "stocks.pivot(columns=\"qtr\", values=\"price\", index=\"year\")" + "stocks.pivot(index=\"year\", on=\"qtr\", values=\"price\", aggregate_function=\"first\")" ] }, { @@ -497,14 +531,16 @@ "metadata": {}, "outputs": [], "source": [ - "health = pd.DataFrame(\n", + "health = pl.DataFrame(\n", " {\n", " \"name\": [\"Ikaia\", \"Oletta\", \"Leriah\", \"Dashay\", \"Tresaun\"],\n", " \"smoker\": [\"no\", \"no\", \"previously\", \"no\", \"yes\"],\n", " \"age\": [34, 88, 75, 47, 56],\n", " }\n", ")\n", - "health[\"smoker\"] = health[\"smoker\"].astype(\"category\")" + "health = health.with_columns(\n", + " pl.col(\"smoker\").cast(pl.Enum([\"no\", \"previously\", \"yes\"]))\n", + ")" ] }, { @@ -522,7 +558,7 @@ "metadata": {}, "outputs": [], "source": [ - "health_cut = health.iloc[:-1, :]\n", + "health_cut = health[:-1]\n", "health_cut" ] }, @@ -531,7 +567,7 @@ "id": "7ae95b5f", "metadata": {}, "source": [ - "The value 'yes' for smoker now doesn't (seem to) appear anywhere in our data frame. But if we run `value_counts()` to get a count of the number of each type of category, you'll see that the data frame 'remembers' that there's a 'yes' category that isn't currently present:" + "The value 'yes' for smoker now doesn't (seem to) appear anywhere in our data frame. Because we used an `Enum` type, the column still remembers that 'yes' is a valid category. We can inspect the categories of the column's datatype:" ] }, { @@ -541,7 +577,7 @@ "metadata": {}, "outputs": [], "source": [ - "health_cut[\"smoker\"].value_counts()" + "health_cut[\"smoker\"].dtype.categories" ] }, { @@ -549,7 +585,7 @@ "id": "bde48398", "metadata": {}, "source": [ - "You'll see the same thing happen with a `groupby()` operation:" + "If we perform aggregation operations, Polars will only return results for groups present in the data by default. If we need to include all categories (even those with no observations), we can join the categories list back to the aggregated result:" ] }, { @@ -559,7 +595,13 @@ "metadata": {}, "outputs": [], "source": [ - "health_cut.groupby(\"smoker\")[\"age\"].mean()" + "all_smokers = pl.DataFrame({\"smoker\": health_cut[\"smoker\"].dtype.categories})\n", + "means = (\n", + " health_cut.group_by(\"smoker\")\n", + " .agg(pl.col(\"age\").mean())\n", + " .with_columns(pl.col(\"smoker\").cast(pl.Utf8))\n", + ")\n", + "all_smokers.join(means, on=\"smoker\", how=\"left\")" ] }, { @@ -567,14 +609,11 @@ "id": "5a2af19e", "metadata": {}, "source": [ - "You can see here that, because we took the mean of a number that doesn't exist, we got a NaN in place of a real value for the yes row (but there is a 'yes' row)." + "You can see here that, because we took the mean of a number that doesn't exist, we got a null in place of a real value for the yes row (but there is a 'yes' row)." ] } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -582,7 +621,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" },