diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py b/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py
index edd8e2c71255..1abaf6cfed5e 100644
--- a/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py
+++ b/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py
@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-import json
+
 import os
 import logging
 import requests
@@ -11,7 +11,6 @@
 import sys
 import time
 import signal
-from typing import TYPE_CHECKING
 
 import pytest
 import subprocess
@@ -20,9 +19,6 @@
 from .helpers import is_live_and_not_recording
 from .sanitizers import add_remove_header_sanitizer, set_custom_default_matcher
 
-if TYPE_CHECKING:
-    from typing import Optional
-
 
 _LOGGER = logging.getLogger()
 
@@ -61,23 +57,11 @@ def get_image_tag() -> str:
     return image_tag
 
 
-def get_container_info() -> "Optional[dict]":
-    """Returns a dictionary containing the test proxy container's information, or None if the container isn't present"""
-    proc = subprocess.Popen(
-        shlex.split("docker container ls -a --format '{{json .}}' --filter name=" + CONTAINER_NAME),
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        stdin=subprocess.DEVNULL,
-    )
-
+def delete_container() -> None:
+    """Delete container if it remained"""
+    proc = subprocess.Popen(shlex.split(f"docker rm -f {CONTAINER_NAME}"))
     output, stderr = proc.communicate()
-    try:
-        # This will succeed if we found a container with CONTAINER_NAME
-        return json.loads(output)
-    # We'll get a JSONDecodeError on Py3 (ValueError on Py2) if output is empty (i.e. there's no proxy container)
-    except ValueError:
-        # Didn't find a container with CONTAINER_NAME
-        return None
+    return None
 
 
 def check_availability() -> None:
@@ -109,6 +93,8 @@ def create_container() -> None:
     # Most of the time, running this script on a Windows machine will work just fine, as Docker defaults to Linux
     # containers. However, in CI, Windows images default to _Windows_ containers. We cannot swap them. We can tell
     # if we're in a CI build by checking for the environment variable TF_BUILD.
+    delete_container()
+
     if sys.platform.startswith("win") and os.environ.get("TF_BUILD"):
         image_prefix = WINDOWS_IMAGE_SOURCE_PREFIX
         path_prefix = "C:"
@@ -119,19 +105,12 @@ def create_container() -> None:
         linux_container_args = "--add-host=host.docker.internal:host-gateway"
 
     image_tag = get_image_tag()
-    proc = subprocess.Popen(
+    subprocess.Popen(
         shlex.split(
-            "docker container create -v '{}:{}/srv/testproxy' {} -p 5001:5001 -p 5000:5000 --name {} {}:{}".format(
-                REPO_ROOT,
-                path_prefix,
-                linux_container_args,
-                CONTAINER_NAME,
-                image_prefix,
-                image_tag,
-            )
+            f"docker run --rm --name {CONTAINER_NAME} -v '{REPO_ROOT}:{path_prefix}/srv/testproxy' "
+            f"{linux_container_args} -p 5001:5001 -p 5000:5000 {image_prefix}:{image_tag}"
         )
     )
-    proc.communicate()
 
 
 def start_test_proxy() -> None:
@@ -158,23 +137,7 @@ def start_test_proxy() -> None:
                 os.environ[TOOL_ENV_VAR] = str(proc.pid)
         else:
             _LOGGER.info("Starting the test proxy container...")
-
-            container_info = get_container_info()
-            if container_info:
-                _LOGGER.debug("Found an existing instance of the test proxy container.")
-
-                if container_info["State"] == "running":
-                    _LOGGER.debug("Proxy container is already running. Exiting...")
-                    return
-
-            else:
-                _LOGGER.debug("No instance of the test proxy container found. Attempting creation...")
-                create_container()
-
-            _LOGGER.debug("Attempting to start the test proxy container...")
-
-            proc = subprocess.Popen(shlex.split("docker container start " + CONTAINER_NAME))
-            proc.communicate()
+            create_container()
 
     # Wait for the proxy server to become available
     check_proxy_availability()
@@ -199,15 +162,7 @@ def stop_test_proxy() -> None:
 
         else:
             _LOGGER.info("Stopping the test proxy container...")
-            container_info = get_container_info()
-            if container_info:
-                if container_info["State"] == "running":
-                    _LOGGER.debug("Found a running instance of the test proxy container; shutting it down...")
-
-                    proc = subprocess.Popen(shlex.split("docker container stop " + CONTAINER_NAME))
-                    proc.communicate()
-            else:
-                _LOGGER.debug("No running instance of the test proxy container found. Exiting...")
+            subprocess.Popen(shlex.split("docker stop " + CONTAINER_NAME))
 
 
 @pytest.fixture(scope="session")